Add itSkipOnFail util

Part of #139076

test/smoke/src/areas/terminal/terminal-input.test.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { Application, Terminal, SettingsEditor } from '../../../../automation';
+import { itSkipOnFail } from '../../utils';
 
 export function setup() {
 	describe('Terminal Input', () => {
@@ -24,13 +25,13 @@ export function setup() {
 				await terminal.runCommandInTerminal(`"\r${text}`, true);
 			}
 
-			it('should automatically reply to default "Terminate batch job (Y/N)"', async () => {
+			itSkipOnFail('should automatically reply to default "Terminate batch job (Y/N)"', async () => {
 				await terminal.createTerminal();
 				await writeTextForAutoReply('Terminate batch job (Y/N)?');
 				await terminal.waitForTerminalText(buffer => buffer.some(line => line.includes('Terminate batch job (Y/N)?Y')));
 			});
 
-			it('should automatically reply to a custom entry', async () => {
+			itSkipOnFail('should automatically reply to a custom entry', async () => {
 				await settingsEditor.addUserSetting('terminal.integrated.autoReplies', '{ "foo": "bar" }');
 				await terminal.createTerminal();
 				await writeTextForAutoReply('foo');

test/smoke/src/utils.ts

@@ -18,6 +18,25 @@ export function itRepeat(n: number, description: string, callback: (this: Contex
 	}
 }
 
+/**
+ * Defines a test-case that will run but will be skips it if it throws an exception. This is useful
+ * to get some runs in CI when trying to stabilize a flaky test, without failing the build. Note
+ * that this only works if something inside the test throws, so a test's overall timeout won't work
+ * but throwing due to a polling timeout will.
+ * @param title The test-case title.
+ * @param callback The test-case callback.
+ */
+export function itSkipOnFail(title: string, callback: (this: Context) => any): void {
+	it(title, function () {
+		return Promise.resolve().then(() => {
+			return callback.apply(this, arguments);
+		}).catch(() => {
+			console.warn(`Test "${title}" failed but was marks as skip on fail`);
+			this.skip();
+		});
+	});
+}
+
 export function installAllHandlers(logger: Logger, optionsTransform?: (opts: ApplicationOptions) => ApplicationOptions) {
 	installDiagnosticsHandler(logger);
 	installAppBeforeHandler(optionsTransform);