Merge pull request #148918 from microsoft/tyriar/146700_2

Fix, re-enable or disable skipped terminal smoke tests

build/azure-pipelines/darwin/product-build-darwin-test.yml

@@ -213,7 +213,7 @@ steps:
       set -e
       VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-darwin-$(VSCODE_ARCH)" \
       yarn smoketest-no-compile --web --tracing --headless
-    timeoutInMinutes: 10
+    timeoutInMinutes: 20
     displayName: Run smoke tests (Browser, Chromium)
 
   - script: |

build/azure-pipelines/linux/product-build-linux-client.yml

@@ -269,7 +269,7 @@ steps:
       set -e
       VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-web-linux-$(VSCODE_ARCH)" \
       yarn smoketest-no-compile --web --tracing --headless --electronArgs="--disable-dev-shm-usage"
-    timeoutInMinutes: 10
+    timeoutInMinutes: 20
     displayName: Run smoke tests (Browser, Chromium)
     condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
 

build/azure-pipelines/win32/product-build-win32.yml

@@ -229,7 +229,7 @@ steps:
       $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)"
       exec { yarn smoketest-no-compile --web --tracing --headless }
     displayName: Run smoke tests (Browser, Chromium)
-    timeoutInMinutes: 10
+    timeoutInMinutes: 20
     condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
 
   - powershell: |

test/automation/src/application.ts

@@ -107,7 +107,7 @@ export class Application {
 			extraArgs: [...(this.options.extraArgs || []), ...extraArgs],
 		});
 
-		this._workbench = new Workbench(this._code, this.userDataPath);
+		this._workbench = new Workbench(this._code);
 
 		return code;
 	}

test/automation/src/settings.ts

@@ -3,8 +3,6 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import * as fs from 'fs';
-import * as path from 'path';
 import { Editor } from './editor';
 import { Editors } from './editors';
 import { Code } from './code';
@@ -12,7 +10,7 @@ import { QuickAccess } from './quickaccess';
 
 export class SettingsEditor {
 
-	constructor(private code: Code, private userDataPath: string, private editors: Editors, private editor: Editor, private quickaccess: QuickAccess) { }
+	constructor(private code: Code, private editors: Editors, private editor: Editor, private quickaccess: QuickAccess) { }
 
 	async addUserSetting(setting: string, value: string): Promise<void> {
 		await this.openUserSettingsFile();
@@ -23,11 +21,12 @@ export class SettingsEditor {
 	}
 
 	async clearUserSettings(): Promise<void> {
-		const settingsPath = path.join(this.userDataPath, 'User', 'settings.json');
-		await new Promise<void>((c, e) => fs.writeFile(settingsPath, '{\n}', 'utf8', err => err ? e(err) : c()));
-
 		await this.openUserSettingsFile();
-		await this.editor.waitForEditorContents('settings.json', c => c === '{}');
+		await this.quickaccess.runCommand('editor.action.selectAll');
+		await this.code.dispatchKeybinding('Delete');
+		await this.editor.waitForTypeInEditor('settings.json', `{`); // will auto close }
+		await this.editors.saveOpenedFile();
+		await this.quickaccess.runCommand('workbench.action.closeActiveEditor');
 	}
 
 	async openUserSettingsFile(): Promise<void> {

test/automation/src/terminal.ts

@@ -6,6 +6,7 @@
 import { QuickInput } from './quickinput';
 import { Code } from './code';
 import { QuickAccess } from './quickaccess';
+import { IElement } from './driver';
 
 export enum Selector {
 	TerminalView = `#terminal`,
@@ -75,7 +76,7 @@ export class Terminal {
 
 	constructor(private code: Code, private quickaccess: QuickAccess, private quickinput: QuickInput) { }
 
-	async runCommand(commandId: TerminalCommandId): Promise<void> {
+	async runCommand(commandId: TerminalCommandId, expectedLocation?: 'editor' | 'panel'): Promise<void> {
 		const keepOpen = commandId === TerminalCommandId.Join;
 		await this.quickaccess.runCommand(commandId, keepOpen);
 		if (keepOpen) {
@@ -83,7 +84,7 @@ export class Terminal {
 			await this.quickinput.waitForQuickInputClosed();
 		}
 		if (commandId === TerminalCommandId.Show || commandId === TerminalCommandId.CreateNewEditor || commandId === TerminalCommandId.CreateNew || commandId === TerminalCommandId.NewWithProfile) {
-			return await this._waitForTerminal(commandId === TerminalCommandId.CreateNewEditor ? 'editor' : 'panel');
+			return await this._waitForTerminal(expectedLocation === 'editor' || commandId === TerminalCommandId.CreateNewEditor ? 'editor' : 'panel');
 		}
 	}
 
@@ -114,11 +115,11 @@ export class Terminal {
 
 	/**
 	 * Creates a terminal using the new terminal command.
-	 * @param location The location to check the terminal for, defaults to panel.
+	 * @param expectedLocation The location to check the terminal for, defaults to panel.
 	 */
-	async createTerminal(location?: 'editor' | 'panel'): Promise<void> {
+	async createTerminal(expectedLocation?: 'editor' | 'panel'): Promise<void> {
-		await this.runCommand(TerminalCommandId.CreateNew);
+		await this.runCommand(TerminalCommandId.CreateNew, expectedLocation);
-		await this._waitForTerminal(location);
+		await this._waitForTerminal(expectedLocation);
 	}
 
 	async assertEditorGroupCount(count: number): Promise<void> {
@@ -160,11 +161,11 @@ export class Terminal {
 		const groups: TerminalGroup[] = [];
 		for (let i = 0; i < tabCount; i++) {
 			const title = await this.code.waitForElement(`${Selector.Tabs}[data-index="${i}"] ${Selector.TabsEntry}`, e => e?.textContent?.length ? e?.textContent?.length > 1 : false);
-			const description = await this.code.waitForElement(`${Selector.Tabs}[data-index="${i}"] ${Selector.TabsEntry} ${Selector.Description}`, e => e?.textContent?.length ? e?.textContent?.length > 1 : false);
+			const description: IElement | undefined = await this.code.waitForElement(`${Selector.Tabs}[data-index="${i}"] ${Selector.TabsEntry} ${Selector.Description}`, () => true);
 
 			const label: TerminalLabel = {
 				name: title.textContent.replace(/^[├┌└]\s*/, ''),
-				description: description.textContent
+				description: description?.textContent
 			};
 			// It's a new group if the the tab does not start with ├ or └
 			if (title.textContent.match(/^[├└]/)) {
@@ -257,10 +258,10 @@ export class Terminal {
 
 	/**
 	 * Waits for the terminal to be focused and to contain content.
-	 * @param location The location to check the terminal for, defaults to panel.
+	 * @param expectedLocation The location to check the terminal for, defaults to panel.
 	 */
-	private async _waitForTerminal(location?: 'editor' | 'panel'): Promise<void> {
+	private async _waitForTerminal(expectedLocation?: 'editor' | 'panel'): Promise<void> {
 		await this.code.waitForElement(Selector.XtermFocused);
-		await this.code.waitForTerminalBuffer(location === 'editor' ? Selector.XtermEditor : Selector.Xterm, lines => lines.some(line => line.length > 0));
+		await this.code.waitForTerminalBuffer(expectedLocation === 'editor' ? Selector.XtermEditor : Selector.Xterm, lines => lines.some(line => line.length > 0));
 	}
 }

test/automation/src/workbench.ts

@@ -46,7 +46,7 @@ export class Workbench {
 	readonly notebook: Notebook;
 	readonly localization: Localization;
 
-	constructor(code: Code, userDataPath: string) {
+	constructor(code: Code) {
 		this.editors = new Editors(code);
 		this.quickinput = new QuickInput(code);
 		this.quickaccess = new QuickAccess(code, this.editors, this.quickinput);
@@ -59,7 +59,7 @@ export class Workbench {
 		this.debug = new Debug(code, this.quickaccess, this.editors, this.editor);
 		this.statusbar = new StatusBar(code);
 		this.problems = new Problems(code, this.quickaccess);
-		this.settingsEditor = new SettingsEditor(code, userDataPath, this.editors, this.editor, this.quickaccess);
+		this.settingsEditor = new SettingsEditor(code, this.editors, this.editor, this.quickaccess);
 		this.keybindingsEditor = new KeybindingsEditor(code);
 		this.terminal = new Terminal(code, this.quickaccess, this.quickinput);
 		this.notebook = new Notebook(this.quickaccess, code);

test/smoke/src/areas/terminal/terminal-editors.test.ts

@@ -3,16 +3,25 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { Application, Terminal, TerminalCommandId, TerminalCommandIdWithValue } from '../../../../automation';
+import { Application, Terminal, TerminalCommandId, TerminalCommandIdWithValue, SettingsEditor } from '../../../../automation';
+import { setTerminalTestSettings } from './terminal-helpers';
 
 export function setup() {
 	describe('Terminal Editors', () => {
-		let terminal: Terminal;
 		let app: Application;
+		let terminal: Terminal;
+		let settingsEditor: SettingsEditor;
+
 		// Acquire automation API
 		before(async function () {
 			app = this.app as Application;
 			terminal = app.workbench.terminal;
+			settingsEditor = app.workbench.settingsEditor;
+			await setTerminalTestSettings(app);
+		});
+
+		after(async function () {
+			await settingsEditor.clearUserSettings();
 		});
 
 		it('should update color of the tab', async () => {
@@ -66,13 +75,14 @@ export function setup() {
 			await terminal.assertEditorGroupCount(1);
 		});
 
-		it.skip('should create a terminal in the editor area by default', async () => {
+		it('should create a terminal in the editor area by default', async () => {
 			await app.workbench.settingsEditor.addUserSetting('terminal.integrated.defaultLocation', '"editor"');
 			// Close the settings editor
 			await app.workbench.quickaccess.runCommand('workbench.action.closeAllEditors');
 			await terminal.createTerminal('editor');
 			await terminal.assertEditorGroupCount(1);
 			await terminal.assertTerminalViewHidden();
+			await app.workbench.settingsEditor.clearUserSettings();
 		});
 	});
 }

test/smoke/src/areas/terminal/terminal-helpers.ts

@@ -0,0 +1,17 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Application } from '../../../../automation';
+
+export async function setTerminalTestSettings(app: Application) {
+	// Always show tabs to make getting terminal groups easier
+	await app.workbench.settingsEditor.addUserSetting('terminal.integrated.tabs.hideCondition', '"never"');
+	// Use the DOM renderer for smoke tests so they can be inspected in the playwright trace
+	// viewer
+	await app.workbench.settingsEditor.addUserSetting('terminal.integrated.gpuAcceleration', '"off"');
+
+	// Close the settings editor
+	await app.workbench.quickaccess.runCommand('workbench.action.closeAllEditors');
+}

test/smoke/src/areas/terminal/terminal-input.test.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { Application, Terminal, SettingsEditor } from '../../../../automation';
+import { setTerminalTestSettings } from './terminal-helpers';
 
 export function setup() {
 	describe('Terminal Input', () => {
@@ -15,6 +16,11 @@ export function setup() {
 			const app = this.app as Application;
 			terminal = app.workbench.terminal;
 			settingsEditor = app.workbench.settingsEditor;
+			await setTerminalTestSettings(app);
+		});
+
+		after(async function () {
+			await settingsEditor.clearUserSettings();
 		});
 
 		describe('Auto replies', function () {
@@ -31,12 +37,6 @@ export function setup() {
 				await terminal.runCommandInTerminal(`"\r${text}`, true);
 			}
 
-			it.skip('should automatically reply to default "Terminate batch job (Y/N)"', async () => { // TODO: #139076
-				await terminal.createTerminal();
-				await writeTextForAutoReply('Terminate batch job (Y/N)?');
-				await terminal.waitForTerminalText(buffer => buffer.some(line => line.match(/\?.*Y/)));
-			});
-
 			it('should automatically reply to a custom entry', async () => {
 				await settingsEditor.addUserSetting('terminal.integrated.autoReplies', '{ "foo": "bar" }');
 				await terminal.createTerminal();

test/smoke/src/areas/terminal/terminal-persistence.test.ts

@@ -3,20 +3,29 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { Application, Terminal, TerminalCommandId, TerminalCommandIdWithValue } from '../../../../automation';
+import { Application, Terminal, TerminalCommandId, TerminalCommandIdWithValue, SettingsEditor } from '../../../../automation';
+import { setTerminalTestSettings } from './terminal-helpers';
 
 export function setup() {
 	describe('Terminal Persistence', () => {
 		// Acquire automation API
 		let terminal: Terminal;
-		before(function () {
+		let settingsEditor: SettingsEditor;
+
+		before(async function () {
 			const app = this.app as Application;
 			terminal = app.workbench.terminal;
+			settingsEditor = app.workbench.settingsEditor;
+			await setTerminalTestSettings(app);
+		});
+
+		after(async function () {
+			await settingsEditor.clearUserSettings();
 		});
 
 		describe('detach/attach', () => {
 			// https://github.com/microsoft/vscode/issues/137799
-			it.skip('should support basic reconnection', async () => {
+			it('should support basic reconnection', async () => {
 				await terminal.createTerminal();
 				// TODO: Handle passing in an actual regex, not string
 				await terminal.assertTerminalGroups([
@@ -40,7 +49,7 @@ export function setup() {
 				]);
 			});
 
-			it.skip('should persist buffer content', async () => {
+			it('should persist buffer content', async () => {
 				await terminal.createTerminal();
 				// TODO: Handle passing in an actual regex, not string
 				await terminal.assertTerminalGroups([
@@ -68,34 +77,6 @@ export function setup() {
 				]);
 				await terminal.waitForTerminalText(buffer => buffer.some(e => e.includes('terminal_test_content')));
 			});
-
-			// TODO: This is currently flaky because it takes time to send over the new icon to the backend
-			it.skip('should persist terminal icon', async () => {
-				await terminal.createTerminal();
-				// TODO: Handle passing in an actual regex, not string
-				await terminal.assertTerminalGroups([
-					[{ name: '.*' }]
-				]);
-
-				// Get the terminal name
-				const name = (await terminal.getTerminalGroups())[0][0].name!;
-
-				// Set the icon
-				await terminal.runCommandWithValue(TerminalCommandIdWithValue.ChangeIcon, 'symbol-method');
-				await terminal.assertSingleTab({ icon: 'symbol-method' });
-
-				// Detach
-				await terminal.runCommand(TerminalCommandId.DetachSession);
-				await terminal.assertTerminalViewHidden();
-
-				// Attach
-				await terminal.runCommandWithValue(TerminalCommandIdWithValue.AttachToSession, name);
-				await terminal.assertTerminalGroups([
-					[{ name }]
-				]);
-				// TODO: This fails due to a bug
-				await terminal.assertSingleTab({ icon: 'symbol-method' });
-			});
 		});
 	});
 }

test/smoke/src/areas/terminal/terminal-profiles.test.ts

@@ -3,7 +3,8 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { Application, Terminal, TerminalCommandId, TerminalCommandIdWithValue } from '../../../../automation';
+import { Application, Terminal, TerminalCommandId, TerminalCommandIdWithValue, SettingsEditor } from '../../../../automation';
+import { setTerminalTestSettings } from './terminal-helpers';
 
 const CONTRIBUTED_PROFILE_NAME = `JavaScript Debug Terminal`;
 const ANY_PROFILE_NAME = '^((?!JavaScript Debug Terminal).)*$';
@@ -12,9 +13,17 @@ export function setup() {
 	describe('Terminal Profiles', () => {
 		// Acquire automation API
 		let terminal: Terminal;
-		before(function () {
+		let settingsEditor: SettingsEditor;
+
+		before(async function () {
 			const app = this.app as Application;
 			terminal = app.workbench.terminal;
+			settingsEditor = app.workbench.settingsEditor;
+			await setTerminalTestSettings(app);
+		});
+
+		after(async function () {
+			await settingsEditor.clearUserSettings();
 		});
 
 		it('should launch the default profile', async () => {
@@ -22,13 +31,13 @@ export function setup() {
 			await terminal.assertSingleTab({ name: ANY_PROFILE_NAME });
 		});
 
-		it.skip('should set the default profile to a contributed one', async () => {
+		it('should set the default profile to a contributed one', async () => {
 			await terminal.runCommandWithValue(TerminalCommandIdWithValue.SelectDefaultProfile, CONTRIBUTED_PROFILE_NAME);
 			await terminal.createTerminal();
 			await terminal.assertSingleTab({ name: CONTRIBUTED_PROFILE_NAME });
 		});
 
-		it.skip('should use the default contributed profile on panel open and for splitting', async () => {
+		it('should use the default contributed profile on panel open and for splitting', async () => {
 			await terminal.runCommandWithValue(TerminalCommandIdWithValue.SelectDefaultProfile, CONTRIBUTED_PROFILE_NAME);
 			await terminal.runCommand(TerminalCommandId.Show);
 			await terminal.runCommand(TerminalCommandId.Split);
@@ -53,7 +62,7 @@ export function setup() {
 			await terminal.assertSingleTab({ name: ANY_PROFILE_NAME });
 		});
 
-		it.skip('createWithProfile command should create a terminal with a contributed profile', async () => {
+		it('createWithProfile command should create a terminal with a contributed profile', async () => {
 			await terminal.runCommandWithValue(TerminalCommandIdWithValue.NewWithProfile, CONTRIBUTED_PROFILE_NAME);
 			await terminal.assertSingleTab({ name: CONTRIBUTED_PROFILE_NAME });
 		});
@@ -64,7 +73,7 @@ export function setup() {
 			await terminal.assertTerminalGroups([[{}, {}]]);
 		});
 
-		it.skip('createWithProfile command should create a split terminal with a contributed profile', async () => {
+		it('createWithProfile command should create a split terminal with a contributed profile', async () => {
 			await terminal.runCommand(TerminalCommandId.Show);
 			await terminal.assertSingleTab({});
 			await terminal.runCommandWithValue(TerminalCommandIdWithValue.NewWithProfile, CONTRIBUTED_PROFILE_NAME, true);

test/smoke/src/areas/terminal/terminal-shellIntegration.test.ts

@@ -4,6 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { Application, Terminal, SettingsEditor } from '../../../../automation';
+import { setTerminalTestSettings } from './terminal-helpers';
 
 export function setup() {
 	describe('Terminal Shell Integration', () => {
@@ -16,6 +17,11 @@ export function setup() {
 			terminal = app.workbench.terminal;
 			settingsEditor = app.workbench.settingsEditor;
 			await settingsEditor.addUserSetting('terminal.integrated.shellIntegration.enabled', 'true');
+			await setTerminalTestSettings(app);
+		});
+
+		after(async function () {
+			await settingsEditor.clearUserSettings();
 		});
 
 		describe('Shell integration', function () {

test/smoke/src/areas/terminal/terminal-splitCwd.test.ts

@@ -3,17 +3,24 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { Application, Terminal } from '../../../../automation';
+import { Application, Terminal, SettingsEditor } from '../../../../automation';
+import { setTerminalTestSettings } from './terminal-helpers';
 
 export function setup() {
 	describe('Terminal splitCwd', () => {
 		// Acquire automation API
 		let terminal: Terminal;
+		let settingsEditor: SettingsEditor;
 		before(async function () {
 			const app = this.app as Application;
 			terminal = app.workbench.terminal;
-			await app.workbench.settingsEditor.addUserSetting('terminal.integrated.splitCwd', '"inherited"');
+			settingsEditor = app.workbench.settingsEditor;
-			await app.workbench.quickaccess.runCommand('workbench.action.closeAllEditors');
+			await settingsEditor.addUserSetting('terminal.integrated.splitCwd', '"inherited"');
+			await setTerminalTestSettings(app);
+		});
+
+		after(async function () {
+			await settingsEditor.clearUserSettings();
 		});
 
 		it('should inherit cwd when split and update the tab description - alt click', async () => {

test/smoke/src/areas/terminal/terminal-tabs.test.ts

@@ -3,15 +3,24 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { Application, Terminal, TerminalCommandId, TerminalCommandIdWithValue } from '../../../../automation';
+import { Application, Terminal, TerminalCommandId, TerminalCommandIdWithValue, SettingsEditor } from '../../../../automation';
+import { setTerminalTestSettings } from './terminal-helpers';
 
 export function setup() {
 	describe('Terminal Tabs', () => {
 		// Acquire automation API
 		let terminal: Terminal;
-		before(function () {
+		let settingsEditor: SettingsEditor;
+
+		before(async function () {
 			const app = this.app as Application;
 			terminal = app.workbench.terminal;
+			settingsEditor = app.workbench.settingsEditor;
+			await setTerminalTestSettings(app);
+		});
+
+		after(async function () {
+			await settingsEditor.clearUserSettings();
 		});
 
 		it('clicking the plus button should create a terminal and display the tabs view showing no split decorations', async () => {

test/smoke/src/areas/terminal/terminal.test.ts

@@ -22,20 +22,12 @@ export function setup(logger: Logger) {
 		// Shared before/after handling
 		installAllHandlers(logger);
 
+		let app: Application;
 		let terminal: Terminal;
 		before(async function () {
 			// Fetch terminal automation API
-			const app = this.app as Application;
+			app = this.app as Application;
 			terminal = app.workbench.terminal;
-
-			// Always show tabs to make getting terminal groups easier
-			await app.workbench.settingsEditor.addUserSetting('terminal.integrated.tabs.hideCondition', '"never"');
-			// Use the DOM renderer for smoke tests so they can be inspected in the playwright trace
-			// viewer
-			await app.workbench.settingsEditor.addUserSetting('terminal.integrated.gpuAcceleration', '"off"');
-
-			// Close the settings editor
-			await app.workbench.quickaccess.runCommand('workbench.action.closeAllEditors');
 		});
 
 		afterEach(async () => {