diff --git a/test/automation/src/application.ts b/test/automation/src/application.ts index 8f573355e44..800660efb36 100644 --- a/test/automation/src/application.ts +++ b/test/automation/src/application.ts @@ -28,6 +28,7 @@ export class Application { private _workbench: Workbench | undefined; constructor(private options: ApplicationOptions) { + this._userDataPath = options.userDataDir; this._workspacePathOrFolder = options.workspacePath; } @@ -64,8 +65,9 @@ export class Application { return this.options.extensionsPath; } + private _userDataPath: string; get userDataPath(): string { - return this.options.userDataDir; + return this._userDataPath; } async start(expectWalkthroughPart = true): Promise { @@ -123,7 +125,7 @@ export class Application { this._code = await spawn({ codePath: this.options.codePath, workspacePath: this.workspacePathOrFolder, - userDataDir: this.options.userDataDir, + userDataDir: this.userDataPath, extensionsPath: this.options.extensionsPath, logger: this.options.logger, verbose: this.options.verbose, diff --git a/test/automation/src/playwrightDriver.ts b/test/automation/src/playwrightDriver.ts index 03840dcb547..a5c16905915 100644 --- a/test/automation/src/playwrightDriver.ts +++ b/test/automation/src/playwrightDriver.ts @@ -37,7 +37,10 @@ function buildDriver(browser: playwright.Browser, page: playwright.Page): IDrive }, capturePage: () => Promise.resolve(''), reloadWindow: (windowId) => Promise.resolve(), - exitApplication: () => browser.close(), + exitApplication: async () => { + await browser.close(); + await teardown(); + }, dispatchKeybinding: async (windowId, keybinding) => { const chords = keybinding.split(' '); for (let i = 0; i < chords.length; i++) { @@ -123,9 +126,9 @@ export async function launch(userDataDir: string, _workspacePath: string, codeSe endpoint = await waitForEndpoint(); } -function teardown(): void { +async function teardown(): Promise { if (server) { - kill(server.pid); + await new Promise((c, e) => kill(server!.pid, error => error ? e(error) : c(null))); server = undefined; } } diff --git a/test/smoke/src/areas/editor/editor.test.ts b/test/smoke/src/areas/editor/editor.test.ts index 111569e8fac..4b50f67a035 100644 --- a/test/smoke/src/areas/editor/editor.test.ts +++ b/test/smoke/src/areas/editor/editor.test.ts @@ -3,10 +3,15 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import minimist = require('minimist'); import { Application } from '../../../../automation'; +import { afterSuite, beforeSuite } from '../../utils'; -export function setup() { +export function setup(opts: minimist.ParsedArgs) { describe('Editor', () => { + beforeSuite(opts); + afterSuite(); + it('shows correct quick outline', async function () { const app = this.app as Application; await app.workbench.quickaccess.openFile('www'); diff --git a/test/smoke/src/areas/extensions/extensions.test.ts b/test/smoke/src/areas/extensions/extensions.test.ts index 8154cc465f0..aeba9bdd70e 100644 --- a/test/smoke/src/areas/extensions/extensions.test.ts +++ b/test/smoke/src/areas/extensions/extensions.test.ts @@ -3,10 +3,15 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import minimist = require('minimist'); import { Application, Quality } from '../../../../automation'; +import { afterSuite, beforeSuite } from '../../utils'; -export function setup() { +export function setup(opts: minimist.ParsedArgs) { describe('Extensions', () => { + beforeSuite(opts); + afterSuite(); + it(`install and enable vscode-smoketest-check extension`, async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/languages/languages.test.ts b/test/smoke/src/areas/languages/languages.test.ts index ef1bf9455cf..3112c08c3e1 100644 --- a/test/smoke/src/areas/languages/languages.test.ts +++ b/test/smoke/src/areas/languages/languages.test.ts @@ -3,10 +3,15 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import minimist = require('minimist'); import { Application, ProblemSeverity, Problems } from '../../../../automation/out'; +import { afterSuite, beforeSuite } from '../../utils'; -export function setup() { +export function setup(opts: minimist.ParsedArgs) { describe('Language Features', () => { + beforeSuite(opts); + afterSuite(); + it('verifies quick outline', async function () { const app = this.app as Application; await app.workbench.quickaccess.openFile('style.css'); diff --git a/test/smoke/src/areas/multiroot/multiroot.test.ts b/test/smoke/src/areas/multiroot/multiroot.test.ts index 76c46b5fe56..3707047d808 100644 --- a/test/smoke/src/areas/multiroot/multiroot.test.ts +++ b/test/smoke/src/areas/multiroot/multiroot.test.ts @@ -4,8 +4,10 @@ *--------------------------------------------------------------------------------------------*/ import * as fs from 'fs'; +import minimist = require('minimist'); import * as path from 'path'; import { Application } from '../../../../automation'; +import { afterSuite, beforeSuite } from '../../utils'; function toUri(path: string): string { if (process.platform === 'win32') { @@ -34,8 +36,9 @@ async function createWorkspaceFile(workspacePath: string): Promise { return workspaceFilePath; } -export function setup() { +export function setup(opts: minimist.ParsedArgs) { describe('Multiroot', () => { + beforeSuite(opts); before(async function () { const app = this.app as Application; @@ -47,6 +50,8 @@ export function setup() { await app.restart({ workspaceOrFolder: workspaceFilePath }); }); + afterSuite(); + it('shows results from all folders', async function () { const app = this.app as Application; await app.workbench.quickaccess.openQuickAccess('*.*'); diff --git a/test/smoke/src/areas/notebook/notebook.test.ts b/test/smoke/src/areas/notebook/notebook.test.ts index 190bf10c498..65859d3004b 100644 --- a/test/smoke/src/areas/notebook/notebook.test.ts +++ b/test/smoke/src/areas/notebook/notebook.test.ts @@ -4,20 +4,18 @@ *--------------------------------------------------------------------------------------------*/ import * as cp from 'child_process'; +import minimist = require('minimist'); import { Application } from '../../../../automation'; +import { afterSuite, beforeSuite } from '../../utils'; // function wait(ms: number): Promise { // return new Promise(r => setTimeout(r, ms)); // } -export function setup() { +export function setup(opts: minimist.ParsedArgs) { describe('Notebooks', () => { - after(async function () { - const app = this.app as Application; - cp.execSync('git checkout . --quiet', { cwd: app.workspacePathOrFolder }); - cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder }); - }); + beforeSuite(opts); afterEach(async function () { const app = this.app as Application; @@ -25,6 +23,14 @@ export function setup() { await app.workbench.quickaccess.runCommand('workbench.action.closeActiveEditor'); }); + after(async function () { + const app = this.app as Application; + cp.execSync('git checkout . --quiet', { cwd: app.workspacePathOrFolder }); + cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder }); + }); + + afterSuite(); + // it('inserts/edits code cell', async function () { // const app = this.app as Application; // await app.workbench.notebook.openNotebook(); diff --git a/test/smoke/src/areas/preferences/preferences.test.ts b/test/smoke/src/areas/preferences/preferences.test.ts index 69652142369..1138e7da0ba 100644 --- a/test/smoke/src/areas/preferences/preferences.test.ts +++ b/test/smoke/src/areas/preferences/preferences.test.ts @@ -3,10 +3,15 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import minimist = require('minimist'); import { Application, ActivityBarPosition } from '../../../../automation'; +import { afterSuite, beforeSuite } from '../../utils'; -export function setup() { +export function setup(opts: minimist.ParsedArgs) { describe('Preferences', () => { + beforeSuite(opts); + afterSuite(); + it('turns off editor line numbers and verifies the live change', async function () { const app = this.app as Application; @@ -27,13 +32,5 @@ export function setup() { await app.code.dispatchKeybinding('ctrl+u'); await app.workbench.activitybar.waitForActivityBar(ActivityBarPosition.RIGHT); }); - - after(async function () { - const app = this.app as Application; - await app.workbench.settingsEditor.clearUserSettings(); - - // Wait for settings to be applied, which will happen after the settings file is empty - await app.workbench.activitybar.waitForActivityBar(ActivityBarPosition.LEFT); - }); }); } diff --git a/test/smoke/src/areas/search/search.test.ts b/test/smoke/src/areas/search/search.test.ts index 4eca396976e..274d96d7cb3 100644 --- a/test/smoke/src/areas/search/search.test.ts +++ b/test/smoke/src/areas/search/search.test.ts @@ -4,17 +4,23 @@ *--------------------------------------------------------------------------------------------*/ import * as cp from 'child_process'; +import minimist = require('minimist'); import { Application } from '../../../../automation'; +import { afterSuite, beforeSuite } from '../../utils'; -export function setup() { +export function setup(opts: minimist.ParsedArgs) { // https://github.com/microsoft/vscode/issues/115244 describe('Search', () => { + beforeSuite(opts); + after(function () { const app = this.app as Application; cp.execSync('git checkout . --quiet', { cwd: app.workspacePathOrFolder }); cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder }); }); + afterSuite(); + // https://github.com/microsoft/vscode/issues/124146 it.skip /* https://github.com/microsoft/vscode/issues/124335 */('has a tooltp with a keybinding', async function () { const app = this.app as Application; @@ -68,6 +74,9 @@ export function setup() { }); describe('Quick Access', () => { + beforeSuite(opts); + afterSuite(); + it('quick access search produces correct result', async function () { const app = this.app as Application; const expectedNames = [ diff --git a/test/smoke/src/areas/statusbar/statusbar.test.ts b/test/smoke/src/areas/statusbar/statusbar.test.ts index 4cbf56526b4..b0744f0a906 100644 --- a/test/smoke/src/areas/statusbar/statusbar.test.ts +++ b/test/smoke/src/areas/statusbar/statusbar.test.ts @@ -3,10 +3,15 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import minimist = require('minimist'); import { Application, Quality, StatusBarElement } from '../../../../automation'; +import { afterSuite, beforeSuite } from '../../utils'; -export function setup(isWeb) { +export function setup(opts: minimist.ParsedArgs) { describe('Statusbar', () => { + beforeSuite(opts); + afterSuite(); + it('verifies presence of all default status bar elements', async function () { const app = this.app as Application; @@ -18,7 +23,7 @@ export function setup(isWeb) { await app.workbench.statusbar.waitForStatusbarElement(StatusBarElement.PROBLEMS_STATUS); await app.workbench.quickaccess.openFile('app.js'); - if (!isWeb) { + if (!opts.web) { // Encoding picker currently hidden in web (only UTF-8 supported) await app.workbench.statusbar.waitForStatusbarElement(StatusBarElement.ENCODING_STATUS); } @@ -39,7 +44,7 @@ export function setup(isWeb) { await app.workbench.statusbar.clickOn(StatusBarElement.INDENTATION_STATUS); await app.workbench.quickinput.waitForQuickInputOpened(); await app.workbench.quickinput.closeQuickInput(); - if (!isWeb) { + if (!opts.web) { // Encoding picker currently hidden in web (only UTF-8 supported) await app.workbench.statusbar.clickOn(StatusBarElement.ENCODING_STATUS); await app.workbench.quickinput.waitForQuickInputOpened(); diff --git a/test/smoke/src/areas/workbench/data-loss.test.ts b/test/smoke/src/areas/workbench/data-loss.test.ts index d225304b569..51dda345fc8 100644 --- a/test/smoke/src/areas/workbench/data-loss.test.ts +++ b/test/smoke/src/areas/workbench/data-loss.test.ts @@ -3,10 +3,16 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import minimist = require('minimist'); import { Application } from '../../../../automation'; +import { afterSuite, beforeSuite } from '../../utils'; + +export function setup(opts: minimist.ParsedArgs) { -export function setup() { describe('Dataloss', () => { + beforeSuite(opts); + afterSuite(); + it(`verifies that 'hot exit' works for dirty files`, async function () { const app = this.app as Application; await app.workbench.editors.newUntitledFile(); diff --git a/test/smoke/src/areas/workbench/localization.test.ts b/test/smoke/src/areas/workbench/localization.test.ts index 6ffd530f088..9ae63024de8 100644 --- a/test/smoke/src/areas/workbench/localization.test.ts +++ b/test/smoke/src/areas/workbench/localization.test.ts @@ -3,10 +3,14 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import minimist = require('minimist'); import { Application, Quality } from '../../../../automation'; +import { afterSuite, beforeSuite } from '../../utils'; -export function setup() { +export function setup(opts: minimist.ParsedArgs) { describe('Localization', () => { + beforeSuite(opts); + before(async function () { const app = this.app as Application; @@ -21,6 +25,8 @@ export function setup() { await app.restart({ extraArgs: ['--locale=DE'] }); }); + afterSuite(); + it(`starts with 'DE' locale and verifies title and viewlets text is in German`, async function () { const app = this.app as Application; diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts index b0e2ccf3fd6..b5b41438632 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/main.ts @@ -207,7 +207,8 @@ async function setupRepository(): Promise { cp.spawnSync('git', ['clean', '-xdf'], { cwd: workspacePath }); } - // None of the test run the project + // None of the current smoke tests have a dependency on the packages. + // If new smoke tests are added that need the packages, uncomment this. // console.log('*** Running yarn...'); // cp.execSync('yarn', { cwd: workspacePath, stdio: 'inherit' }); } @@ -299,27 +300,16 @@ describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => { } describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => { - before(async function () { - const app = new Application(this.defaultOptions); - await app!.start(opts.web ? false : undefined); - this.app = app; - }); - - after(async function () { - await this.app.stop(); - }); - - if (!opts.web) { setupDataLossTests(); } - if (!opts.web) { setupDataPreferencesTests(); } - setupDataSearchTests(); - setupDataNotebookTests(); - setupDataLanguagesTests(); - setupDataEditorTests(); - setupDataStatusbarTests(!!opts.web); - setupDataExtensionTests(); - if (!opts.web) { setupDataMultirootTests(); } - if (!opts.web) { setupDataLocalizationTests(); } + if (!opts.web) { setupDataLossTests(opts); } + if (!opts.web) { setupDataPreferencesTests(opts); } + setupDataSearchTests(opts); + setupDataNotebookTests(opts); + setupDataLanguagesTests(opts); + setupDataEditorTests(opts); + setupDataStatusbarTests(opts); + setupDataExtensionTests(opts); + if (!opts.web) { setupDataMultirootTests(opts); } + if (!opts.web) { setupDataLocalizationTests(opts); } if (!opts.web) { setupLaunchTests(); } }); }); - diff --git a/test/smoke/src/utils.ts b/test/smoke/src/utils.ts index c987c19992c..72b928ae0cf 100644 --- a/test/smoke/src/utils.ts +++ b/test/smoke/src/utils.ts @@ -3,7 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import minimist = require('minimist'); import { Suite, Context } from 'mocha'; +import { Application, ApplicationOptions } from '../../automation'; export function describeRepeat(n: number, description: string, callback: (this: Suite) => void): void { for (let i = 0; i < n; i++) { @@ -16,3 +18,25 @@ export function itRepeat(n: number, description: string, callback: (this: Contex it(`${description} (iteration ${i})`, callback); } } + +export function beforeSuite(opts: minimist.ParsedArgs) { + before(async function () { + // https://github.com/microsoft/vscode/issues/34988 + const options = this.defaultOptions as ApplicationOptions; + const userDataPathSuffix = [...Array(8)].map(() => Math.random().toString(36)[3]).join(''); + const userDataDir = options.userDataDir.concat(`-${userDataPathSuffix}`); + + const app = new Application({ ...options, userDataDir }); + await app!.start(opts.web ? false : undefined); + this.app = app; + }); +} + +export function afterSuite() { + after(async function () { + const app = this.app as Application; + if (app) { + await app.stop(); + } + }); +}