diff --git a/build/azure-pipelines/win32/product-build-win32.yml b/build/azure-pipelines/win32/product-build-win32.yml index f3c53445950..f37e749c99f 100644 --- a/build/azure-pipelines/win32/product-build-win32.yml +++ b/build/azure-pipelines/win32/product-build-win32.yml @@ -229,7 +229,7 @@ steps: . build/azure-pipelines/win32/exec.ps1 $ErrorActionPreference = "Stop" $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-web-win32-$(VSCODE_ARCH)" - exec { yarn smoketest-no-compile --web --browser firefox --headless } + exec { yarn smoketest-no-compile --web --headless } displayName: Run smoke tests (Browser) timeoutInMinutes: 10 condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64')) diff --git a/src/vs/platform/driver/common/driver.ts b/src/vs/platform/driver/common/driver.ts index 41b7809f448..717e6c82307 100644 --- a/src/vs/platform/driver/common/driver.ts +++ b/src/vs/platform/driver/common/driver.ts @@ -41,6 +41,8 @@ export interface IDriver { getWindowIds(): Promise; capturePage(windowId: number): Promise; + startTracing(windowId: number, name: string): Promise; + stopTracing(windowId: number, name: string, persist: boolean): Promise; reloadWindow(windowId: number): Promise; exitApplication(): Promise; dispatchKeybinding(windowId: number, keybinding: string): Promise; diff --git a/src/vs/platform/driver/electron-main/driver.ts b/src/vs/platform/driver/electron-main/driver.ts index 34f68536dad..17d85fa4592 100644 --- a/src/vs/platform/driver/electron-main/driver.ts +++ b/src/vs/platform/driver/electron-main/driver.ts @@ -69,6 +69,14 @@ export class Driver implements IDriver, IWindowDriverRegistry { return image.toPNG().toString('base64'); } + async startTracing(windowId: number, name: string): Promise { + // ignore - tracing is not implemented yet + } + + async stopTracing(windowId: number, name: string, persist: boolean): Promise { + // ignore - tracing is not implemented yet + } + async reloadWindow(windowId: number): Promise { await this.whenUnfrozen(windowId); diff --git a/src/vs/platform/driver/node/driver.ts b/src/vs/platform/driver/node/driver.ts index 6b30de6beec..9f05cb20932 100644 --- a/src/vs/platform/driver/node/driver.ts +++ b/src/vs/platform/driver/node/driver.ts @@ -21,6 +21,8 @@ export class DriverChannel implements IServerChannel { switch (command) { case 'getWindowIds': return this.driver.getWindowIds(); case 'capturePage': return this.driver.capturePage(arg); + case 'startTracing': return this.driver.startTracing(arg[0], arg[1]); + case 'stopTracing': return this.driver.stopTracing(arg[0], arg[1], arg[2]); case 'reloadWindow': return this.driver.reloadWindow(arg); case 'exitApplication': return this.driver.exitApplication(); case 'dispatchKeybinding': return this.driver.dispatchKeybinding(arg[0], arg[1]); @@ -56,6 +58,14 @@ export class DriverChannelClient implements IDriver { return this.channel.call('capturePage', windowId); } + startTracing(windowId: number, name: string): Promise { + return this.channel.call('startTracing', [windowId, name]); + } + + stopTracing(windowId: number, name: string, persist: boolean): Promise { + return this.channel.call('stopTracing', [windowId, name, persist]); + } + reloadWindow(windowId: number): Promise { return this.channel.call('reloadWindow', windowId); } diff --git a/test/automation/src/application.ts b/test/automation/src/application.ts index d6e024dd548..436c2fd9073 100644 --- a/test/automation/src/application.ts +++ b/test/automation/src/application.ts @@ -104,6 +104,14 @@ export class Application { } } + async startTracing(name: string): Promise { + await this._code?.startTracing(name); + } + + async stopTracing(name: string, persist: boolean): Promise { + await this._code?.stopTracing(name, persist); + } + private async startApplication(extraArgs: string[] = []): Promise { this._code = await spawn({ ...this.options, diff --git a/test/automation/src/code.ts b/test/automation/src/code.ts index c59a03eda5f..acf35f9a823 100644 --- a/test/automation/src/code.ts +++ b/test/automation/src/code.ts @@ -27,7 +27,6 @@ export interface SpawnOptions { web?: boolean; headless?: boolean; browser?: 'chromium' | 'webkit' | 'firefox'; - suiteTitle?: string; testTitle?: string; } @@ -134,6 +133,16 @@ export class Code { return await this.driver.capturePage(windowId); } + async startTracing(name: string): Promise { + const windowId = await this.getActiveWindowId(); + return await this.driver.startTracing(windowId, name); + } + + async stopTracing(name: string, persist: boolean): Promise { + const windowId = await this.getActiveWindowId(); + return await this.driver.stopTracing(windowId, name, persist); + } + async waitForWindowIds(fn: (windowIds: number[]) => boolean): Promise { await poll(() => this.driver.getWindowIds(), fn, `get window ids`); } @@ -161,6 +170,10 @@ export class Code { while (!done) { retries++; + if (retries > 20) { + console.warn('Smoke test exit call did not terminate process after 10s, still trying...'); + } + if (retries > 40) { done = true; reject(new Error('Smoke test exit call did not terminate process after 20s, giving up')); diff --git a/test/automation/src/playwrightDriver.ts b/test/automation/src/playwrightDriver.ts index ac8807a963b..b1a5db3b508 100644 --- a/test/automation/src/playwrightDriver.ts +++ b/test/automation/src/playwrightDriver.ts @@ -43,8 +43,7 @@ class PlaywrightDriver implements IDriver { private readonly server: ChildProcess, private readonly browser: playwright.Browser, private readonly context: playwright.BrowserContext, - private readonly page: playwright.Page, - private readonly suiteTitle: string | undefined + private readonly page: playwright.Page ) { } @@ -56,20 +55,34 @@ class PlaywrightDriver implements IDriver { return ''; } + async startTracing(windowId: number, name: string): Promise { + try { + await this.warnAfter(this.context.tracing.startChunk({ title: name }), 5000, 'Starting playwright trace took more than 5 seconds'); + } catch (error) { + console.warn(`Failed to start playwright tracing (chunk): ${error}`); + } + } + + async stopTracing(windowId: number, name: string, persist: boolean): Promise { + try { + let persistPath: string | undefined = undefined; + if (persist) { + persistPath = join(logsPath, `playwright-trace-${traceCounter++}-${name.replace(/\s+/g, '-')}.zip`); + } + + await this.warnAfter(this.context.tracing.stopChunk({ path: persistPath }), 5000, 'Stopping playwright trace took more than 5 seconds'); + } catch (error) { + console.warn(`Failed to stop playwright tracing (chunk): ${error}`); + } + } + async reloadWindow(windowId: number) { throw new Error('Unsupported'); } async exitApplication() { try { - let traceFileName: string; - if (this.suiteTitle) { - traceFileName = `playwright-trace-${traceCounter++}-${this.suiteTitle.replace(/\s+/g, '-')}.zip`; - } else { - traceFileName = `playwright-trace-${traceCounter++}.zip`; - } - - await this.warnAfter(this.context.tracing.stop({ path: join(logsPath, traceFileName) }), 5000, 'Stopping playwright trace took >5seconds'); + await this.warnAfter(this.context.tracing.stop(), 5000, 'Stopping playwright trace took >5seconds'); } catch (error) { console.warn(`Failed to stop playwright tracing: ${error}`); } @@ -196,7 +209,6 @@ let port = 9000; export interface PlaywrightOptions { readonly browser?: 'chromium' | 'webkit' | 'firefox'; readonly headless?: boolean; - readonly suiteTitle?: string; } export async function launch(codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH, userDataDir: string, extensionsPath: string, workspacePath: string, verbose: boolean, options: PlaywrightOptions = {}): Promise<{ serverProcess: ChildProcess, client: IDisposable, driver: IDriver }> { @@ -212,7 +224,7 @@ export async function launch(codeServerPath = process.env.VSCODE_REMOTE_SERVER_P client: { dispose: () => { /* there is no client to dispose for browser, teardown is triggered via exitApplication call */ } }, - driver: new PlaywrightDriver(serverProcess, browser, context, page, options.suiteTitle) + driver: new PlaywrightDriver(serverProcess, browser, context, page) }; } @@ -275,9 +287,9 @@ async function launchBrowser(options: PlaywrightOptions, endpoint: string, works const context = await browser.newContext(); try { - await context.tracing.start({ screenshots: true, snapshots: true, sources: true, title: options.suiteTitle }); + await context.tracing.start({ screenshots: true, snapshots: true, sources: true }); } catch (error) { - console.warn(`Failed to start playwright tracing.`); // do not fail the build when this fails + console.warn(`Failed to start playwright tracing: ${error}`); // do not fail the build when this fails } const page = await context.newPage(); diff --git a/test/smoke/src/areas/editor/editor.test.ts b/test/smoke/src/areas/editor/editor.test.ts index f64f3f658f9..c57672a04ca 100644 --- a/test/smoke/src/areas/editor/editor.test.ts +++ b/test/smoke/src/areas/editor/editor.test.ts @@ -5,12 +5,13 @@ import minimist = require('minimist'); import { Application } from '../../../../automation'; -import { afterSuite, beforeSuite } from '../../utils'; +import { installCommonTestHandlers } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Editor', () => { - beforeSuite(opts); - afterSuite(opts); + + // Shared before/after handling + installCommonTestHandlers(opts); it('shows correct quick outline', async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/extensions/extensions.test.ts b/test/smoke/src/areas/extensions/extensions.test.ts index 5ee13542b00..47d327ce993 100644 --- a/test/smoke/src/areas/extensions/extensions.test.ts +++ b/test/smoke/src/areas/extensions/extensions.test.ts @@ -5,14 +5,15 @@ import minimist = require('minimist'); import { Application, Quality } from '../../../../automation'; -import { afterSuite, beforeSuite } from '../../utils'; +import { installCommonTestHandlers } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Extensions', () => { - beforeSuite(opts); - afterSuite(opts); - it(`install and enable vscode-smoketest-check extension`, async function () { + // Shared before/after handling + installCommonTestHandlers(opts); + + it('install and enable vscode-smoketest-check extension', async function () { const app = this.app as Application; if (app.quality === Quality.Dev) { @@ -29,6 +30,5 @@ export function setup(opts: minimist.ParsedArgs) { await app.workbench.quickaccess.runCommand('Smoke Test Check'); }); - }); } diff --git a/test/smoke/src/areas/languages/languages.test.ts b/test/smoke/src/areas/languages/languages.test.ts index b4d6f67f130..b17e8c776f4 100644 --- a/test/smoke/src/areas/languages/languages.test.ts +++ b/test/smoke/src/areas/languages/languages.test.ts @@ -5,12 +5,13 @@ import minimist = require('minimist'); import { Application, ProblemSeverity, Problems } from '../../../../automation/out'; -import { afterSuite, beforeSuite } from '../../utils'; +import { installCommonTestHandlers } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Language Features', () => { - beforeSuite(opts); - afterSuite(opts); + + // Shared before/after handling + installCommonTestHandlers(opts); it('verifies quick outline', async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/multiroot/multiroot.test.ts b/test/smoke/src/areas/multiroot/multiroot.test.ts index 62f3026d517..06cd52eb2d6 100644 --- a/test/smoke/src/areas/multiroot/multiroot.test.ts +++ b/test/smoke/src/areas/multiroot/multiroot.test.ts @@ -7,7 +7,7 @@ import * as fs from 'fs'; import minimist = require('minimist'); import * as path from 'path'; import { Application } from '../../../../automation'; -import { afterSuite, beforeSuite } from '../../utils'; +import { installCommonTestHandlers } from '../../utils'; function toUri(path: string): string { if (process.platform === 'win32') { @@ -38,13 +38,13 @@ function createWorkspaceFile(workspacePath: string): string { export function setup(opts: minimist.ParsedArgs) { describe('Multiroot', () => { - beforeSuite(opts, async opts => { + + // Shared before/after handling + installCommonTestHandlers(opts, async opts => { const workspacePath = createWorkspaceFile(opts.workspacePath); return { ...opts, workspacePath }; }); - afterSuite(opts); - it('shows results from all folders', async function () { const app = this.app as Application; await app.workbench.quickaccess.openQuickAccess('*.*'); diff --git a/test/smoke/src/areas/notebook/notebook.test.ts b/test/smoke/src/areas/notebook/notebook.test.ts index 55a471578b4..40941d8603b 100644 --- a/test/smoke/src/areas/notebook/notebook.test.ts +++ b/test/smoke/src/areas/notebook/notebook.test.ts @@ -6,11 +6,13 @@ import * as cp from 'child_process'; import minimist = require('minimist'); import { Application } from '../../../../automation'; -import { afterSuite, beforeSuite } from '../../utils'; +import { installCommonTestHandlers } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe.skip('Notebooks', () => { - beforeSuite(opts); + + // Shared before/after handling + installCommonTestHandlers(opts); afterEach(async function () { const app = this.app as Application; @@ -24,8 +26,6 @@ export function setup(opts: minimist.ParsedArgs) { cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder }); }); - afterSuite(opts); - it.skip('inserts/edits code cell', async function () { const app = this.app as Application; await app.workbench.notebook.openNotebook(); diff --git a/test/smoke/src/areas/preferences/preferences.test.ts b/test/smoke/src/areas/preferences/preferences.test.ts index af363b08661..6f2aee54287 100644 --- a/test/smoke/src/areas/preferences/preferences.test.ts +++ b/test/smoke/src/areas/preferences/preferences.test.ts @@ -5,12 +5,13 @@ import minimist = require('minimist'); import { Application, ActivityBarPosition } from '../../../../automation'; -import { afterSuite, beforeSuite } from '../../utils'; +import { installCommonTestHandlers } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Preferences', () => { - beforeSuite(opts); - afterSuite(opts); + + // Shared before/after handling + installCommonTestHandlers(opts); it('turns off editor line numbers and verifies the live change', async function () { const app = this.app as Application; @@ -23,7 +24,7 @@ export function setup(opts: minimist.ParsedArgs) { await app.code.waitForElements('.line-numbers', false, result => !result || result.length === 0); }); - it(`changes 'workbench.action.toggleSidebarPosition' command key binding and verifies it`, async function () { + it('changes "workbench.action.toggleSidebarPosition" command key binding and verifies it', async function () { const app = this.app as Application; await app.workbench.activitybar.waitForActivityBar(ActivityBarPosition.LEFT); diff --git a/test/smoke/src/areas/search/search.test.ts b/test/smoke/src/areas/search/search.test.ts index 121a7aea927..4e3d6741104 100644 --- a/test/smoke/src/areas/search/search.test.ts +++ b/test/smoke/src/areas/search/search.test.ts @@ -6,11 +6,13 @@ import * as cp from 'child_process'; import minimist = require('minimist'); import { Application } from '../../../../automation'; -import { afterSuite, beforeSuite, retry } from '../../utils'; +import { installCommonTestHandlers, retry } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Search', () => { - beforeSuite(opts); + + // Shared before/after handling + installCommonTestHandlers(opts); after(function () { const app = this.app as Application; @@ -18,8 +20,6 @@ export function setup(opts: minimist.ParsedArgs) { retry(async () => cp.execSync('git reset --hard HEAD --quiet', { cwd: app.workspacePathOrFolder }), 0, 5); }); - afterSuite(opts); - // https://github.com/microsoft/vscode/issues/124146 it.skip /* https://github.com/microsoft/vscode/issues/124335 */('has a tooltp with a keybinding', async function () { const app = this.app as Application; @@ -73,8 +73,9 @@ export function setup(opts: minimist.ParsedArgs) { }); describe('Quick Access', () => { - beforeSuite(opts); - afterSuite(opts); + + // Shared before/after handling + installCommonTestHandlers(opts); it('quick access search produces correct result', async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/statusbar/statusbar.test.ts b/test/smoke/src/areas/statusbar/statusbar.test.ts index b59d8c3ce50..89e5b6e6356 100644 --- a/test/smoke/src/areas/statusbar/statusbar.test.ts +++ b/test/smoke/src/areas/statusbar/statusbar.test.ts @@ -5,12 +5,13 @@ import minimist = require('minimist'); import { Application, Quality, StatusBarElement } from '../../../../automation'; -import { afterSuite, beforeSuite } from '../../utils'; +import { installCommonTestHandlers } from '../../utils'; export function setup(opts: minimist.ParsedArgs) { describe('Statusbar', () => { - beforeSuite(opts); - afterSuite(opts); + + // Shared before/after handling + installCommonTestHandlers(opts); it('verifies presence of all default status bar elements', async function () { const app = this.app as Application; diff --git a/test/smoke/src/areas/terminal/terminal.test.ts b/test/smoke/src/areas/terminal/terminal.test.ts index 64c078a1e45..ce9e3a0d705 100644 --- a/test/smoke/src/areas/terminal/terminal.test.ts +++ b/test/smoke/src/areas/terminal/terminal.test.ts @@ -5,7 +5,7 @@ import minimist = require('minimist'); import { Application, Terminal, TerminalCommandId } from '../../../../automation/out'; -import { afterSuite, beforeSuite } from '../../utils'; +import { installCommonTestHandlers } from '../../utils'; import { setup as setupTerminalEditorsTests } from './terminal-editors.test'; import { setup as setupTerminalPersistenceTests } from './terminal-persistence.test'; import { setup as setupTerminalProfileTests } from './terminal-profiles.test'; @@ -21,8 +21,8 @@ export function setup(opts: minimist.ParsedArgs) { // Retry tests 3 times to minimize build failures due to any flakiness this.retries(3); - beforeSuite(opts); - afterSuite(opts); + // Shared before/after handling + installCommonTestHandlers(opts); let terminal: Terminal; before(async function () { diff --git a/test/smoke/src/areas/workbench/data-loss.test.ts b/test/smoke/src/areas/workbench/data-loss.test.ts index 6ea0d9a5b37..24be2f5743e 100644 --- a/test/smoke/src/areas/workbench/data-loss.test.ts +++ b/test/smoke/src/areas/workbench/data-loss.test.ts @@ -5,15 +5,14 @@ import { Application, ApplicationOptions, Quality } from '../../../../automation/out'; import { ParsedArgs } from 'minimist'; -import { afterSuite, getRandomUserDataDir, startApp, timeout } from '../../utils'; +import { installCommonAfterHandlers, getRandomUserDataDir, startApp, timeout } from '../../utils'; export function setup(opts: ParsedArgs) { - describe('Data Loss (insiders -> insiders)', () => { let app: Application | undefined = undefined; - afterSuite(opts, () => app); + installCommonAfterHandlers(opts, () => app); it('verifies opened editors are restored', async function () { app = await startApp(opts, this.defaultOptions); @@ -98,7 +97,7 @@ export function setup(opts: ParsedArgs) { let insidersApp: Application | undefined = undefined; let stableApp: Application | undefined = undefined; - afterSuite(opts, () => insidersApp ?? stableApp, async () => stableApp?.stop()); + installCommonAfterHandlers(opts, () => insidersApp ?? stableApp, async () => stableApp?.stop()); it('verifies opened editors are restored', async function () { const stableCodePath = opts['stable-build']; diff --git a/test/smoke/src/areas/workbench/launch.test.ts b/test/smoke/src/areas/workbench/launch.test.ts index f46a3bdc994..dc17c9dc14c 100644 --- a/test/smoke/src/areas/workbench/launch.test.ts +++ b/test/smoke/src/areas/workbench/launch.test.ts @@ -6,15 +6,14 @@ import minimist = require('minimist'); import { join } from 'path'; import { Application } from '../../../../automation'; -import { afterSuite, startApp } from '../../utils'; +import { installCommonAfterHandlers, startApp } from '../../utils'; export function setup(args: minimist.ParsedArgs) { - describe('Launch', () => { let app: Application | undefined; - afterSuite(args, () => app); + installCommonAfterHandlers(args, () => app); it(`verifies that application launches when user data directory has non-ascii characters`, async function () { const massagedOptions = { ...this.defaultOptions, userDataDir: join(this.defaultOptions.userDataDir, 'ΓΈ') }; diff --git a/test/smoke/src/areas/workbench/localization.test.ts b/test/smoke/src/areas/workbench/localization.test.ts index 9e27c5ba405..27ce308a902 100644 --- a/test/smoke/src/areas/workbench/localization.test.ts +++ b/test/smoke/src/areas/workbench/localization.test.ts @@ -5,7 +5,7 @@ import minimist = require('minimist'); import { Application, Quality } from '../../../../automation'; -import { afterSuite, startApp } from '../../utils'; +import { installCommonAfterHandlers, startApp } from '../../utils'; export function setup(args: minimist.ParsedArgs) { @@ -13,7 +13,7 @@ export function setup(args: minimist.ParsedArgs) { let app: Application | undefined = undefined; - afterSuite(args, () => app); + installCommonAfterHandlers(args, () => app); it(`starts with 'DE' locale and verifies title and viewlets text is in German`, async function () { if (this.defaultOptions.quality === Quality.Dev || this.defaultOptions.remote) { diff --git a/test/smoke/src/utils.ts b/test/smoke/src/utils.ts index fa960486b96..134c9d236c0 100644 --- a/test/smoke/src/utils.ts +++ b/test/smoke/src/utils.ts @@ -19,13 +19,16 @@ export function itRepeat(n: number, description: string, callback: (this: Contex } } -export function beforeSuite(args: minimist.ParsedArgs, optionsTransform?: (opts: ApplicationOptions) => Promise) { +export function installCommonTestHandlers(args: minimist.ParsedArgs, optionsTransform?: (opts: ApplicationOptions) => Promise) { + installCommonBeforeHandlers(args, optionsTransform); + installCommonAfterHandlers(args); +} + +export function installCommonBeforeHandlers(args: minimist.ParsedArgs, optionsTransform?: (opts: ApplicationOptions) => Promise) { before(async function () { const testTitle = this.currentTest?.title; - const suiteTitle = this.currentTest?.parent?.title; this.app = await startApp(args, this.defaultOptions, async opts => { - opts.suiteTitle = suiteTitle; opts.testTitle = testTitle; if (optionsTransform) { @@ -35,6 +38,12 @@ export function beforeSuite(args: minimist.ParsedArgs, optionsTransform?: (opts: return opts; }); }); + + beforeEach(async function () { + if (this.app) { + await this.app.startTracing(this.currentTest?.title); + } + }); } export async function startApp(args: minimist.ParsedArgs, options: ApplicationOptions, optionsTransform?: (opts: ApplicationOptions) => Promise): Promise { @@ -66,7 +75,7 @@ export function getRandomUserDataDir(options: ApplicationOptions): string { return options.userDataDir.concat(`-${userDataPathSuffix}`); } -export function afterSuite(opts: minimist.ParsedArgs, appFn?: () => Application | undefined, joinFn?: () => Promise) { +export function installCommonAfterHandlers(opts: minimist.ParsedArgs, appFn?: () => Application | undefined, joinFn?: () => Promise) { after(async function () { const app: Application = appFn?.() ?? this.app; @@ -87,6 +96,12 @@ export function afterSuite(opts: minimist.ParsedArgs, appFn?: () => Application await joinFn(); } }); + + afterEach(async function () { + if (this.app) { + await this.app.stopTracing(this.currentTest?.title, this.currentTest?.state === 'failed'); + } + }); } export function timeout(i: number) {