diff --git a/build/azure-pipelines/darwin/product-build-darwin.yml b/build/azure-pipelines/darwin/product-build-darwin.yml index 2b4a19c42aa..475aa86b7ca 100644 --- a/build/azure-pipelines/darwin/product-build-darwin.yml +++ b/build/azure-pipelines/darwin/product-build-darwin.yml @@ -249,7 +249,7 @@ steps: # Increased timeout because this test downloads stable code timeoutInMinutes: 20 displayName: Run smoke tests (Electron) - continueOnError: true + # continueOnError: true condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - script: | @@ -260,28 +260,28 @@ steps: yarn smoketest-no-compile --tracing --remote --build "$APP_ROOT/$APP_NAME" timeoutInMinutes: 10 displayName: Run smoke tests (Remote) - continueOnError: true + # continueOnError: true condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - - script: | - set -e - APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH) - APP_NAME="`ls $APP_ROOT | head -n 1`" - yarn smoketest-no-compile --legacy --tracing --build "$APP_ROOT/$APP_NAME" - # Increased timeout because this test downloads stable code - timeoutInMinutes: 20 - displayName: Run smoke tests (Legacy, Electron) - condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) + # - script: | + # set -e + # APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH) + # APP_NAME="`ls $APP_ROOT | head -n 1`" + # yarn smoketest-no-compile --legacy --tracing --build "$APP_ROOT/$APP_NAME" + # # Increased timeout because this test downloads stable code + # timeoutInMinutes: 20 + # displayName: Run smoke tests (Legacy, Electron) + # condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - - script: | - set -e - APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH) - APP_NAME="`ls $APP_ROOT | head -n 1`" - VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \ - yarn smoketest-no-compile --legacy --tracing --remote --build "$APP_ROOT/$APP_NAME" - timeoutInMinutes: 10 - displayName: Run smoke tests (Legacy, Remote) - condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) + # - script: | + # set -e + # APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH) + # APP_NAME="`ls $APP_ROOT | head -n 1`" + # VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin-$(VSCODE_ARCH)" \ + # yarn smoketest-no-compile --legacy --tracing --remote --build "$APP_ROOT/$APP_NAME" + # timeoutInMinutes: 10 + # displayName: Run smoke tests (Legacy, Remote) + # condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - task: PublishPipelineArtifact@0 inputs: diff --git a/build/azure-pipelines/linux/product-build-linux-client.yml b/build/azure-pipelines/linux/product-build-linux-client.yml index 3e04ef34233..5995d0c5939 100644 --- a/build/azure-pipelines/linux/product-build-linux-client.yml +++ b/build/azure-pipelines/linux/product-build-linux-client.yml @@ -271,7 +271,7 @@ steps: # Increased timeout because this test downloads stable code timeoutInMinutes: 20 displayName: Run smoke tests (Electron) - continueOnError: true + # continueOnError: true condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - script: | @@ -281,26 +281,26 @@ steps: yarn smoketest-no-compile --tracing --remote --build "$APP_PATH" timeoutInMinutes: 10 displayName: Run smoke tests (Remote) - continueOnError: true + # continueOnError: true condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - - script: | - set -e - APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH) - yarn smoketest-no-compile --legacy --tracing --build "$APP_PATH" - # Increased timeout because this test downloads stable code - timeoutInMinutes: 20 - displayName: Run smoke tests (Legacy, Electron) - condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) + # - script: | + # set -e + # APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH) + # yarn smoketest-no-compile --legacy --tracing --build "$APP_PATH" + # # Increased timeout because this test downloads stable code + # timeoutInMinutes: 20 + # displayName: Run smoke tests (Legacy, Electron) + # condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - - script: | - set -e - APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH) - VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \ - yarn smoketest-no-compile --legacy --tracing --remote --build "$APP_PATH" - timeoutInMinutes: 10 - displayName: Run smoke tests (Legacy, Remote) - condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) + # - script: | + # set -e + # APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH) + # VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \ + # yarn smoketest-no-compile --legacy --tracing --remote --build "$APP_PATH" + # timeoutInMinutes: 10 + # displayName: Run smoke tests (Legacy, Remote) + # condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) - task: PublishPipelineArtifact@0 inputs: diff --git a/build/azure-pipelines/win32/product-build-win32.yml b/build/azure-pipelines/win32/product-build-win32.yml index b68783b2e2c..20c9efd4a60 100644 --- a/build/azure-pipelines/win32/product-build-win32.yml +++ b/build/azure-pipelines/win32/product-build-win32.yml @@ -229,7 +229,7 @@ steps: . build/azure-pipelines/win32/exec.ps1 $ErrorActionPreference = "Stop" $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)" - exec { yarn smoketest-no-compile --tracing --build "$AppRoot" } + exec { $env:DEBUG = "pw:browser"; yarn smoketest-no-compile --verbose --tracing --build "$AppRoot" } displayName: Run smoke tests (Electron) continueOnError: true # Increased timeout because this test downloads stable code @@ -241,7 +241,7 @@ steps: $ErrorActionPreference = "Stop" $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)" $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)" - exec { yarn smoketest-no-compile --tracing --remote --build "$AppRoot" } + exec { $env:DEBUG = "pw:browser"; yarn smoketest-no-compile --verbose --tracing --remote --build "$AppRoot" } displayName: Run smoke tests (Remote) continueOnError: true timeoutInMinutes: 10 diff --git a/test/automation/src/application.ts b/test/automation/src/application.ts index 252d809edcd..3a95dc3a32a 100644 --- a/test/automation/src/application.ts +++ b/test/automation/src/application.ts @@ -5,7 +5,7 @@ import { Workbench } from './workbench'; import { Code, launch, LaunchOptions } from './code'; -import { Logger } from './logger'; +import { Logger, measureAndLog } from './logger'; import { PlaywrightDriver } from './playwrightBrowserDriver'; export const enum Quality { @@ -71,29 +71,29 @@ export class Application { return this._userDataPath; } - async start(): Promise { + async start(): Promise { await this._start(); await this.code.waitForElement('.explorer-folders-view'); } - async restart(options?: { workspaceOrFolder?: string; extraArgs?: string[] }): Promise { + async restart(options?: { workspaceOrFolder?: string; extraArgs?: string[] }): Promise { await this.stop(); await this._start(options?.workspaceOrFolder, options?.extraArgs); } - private async _start(workspaceOrFolder = this.workspacePathOrFolder, extraArgs: string[] = []): Promise { + private async _start(workspaceOrFolder = this.workspacePathOrFolder, extraArgs: string[] = []): Promise { this._workspacePathOrFolder = workspaceOrFolder; // Launch Code... const code = await this.startApplication(extraArgs); // ...and make sure the window is ready to interact - const windowReady = this.checkWindowReady(code); + const windowReady = measureAndLog(this.checkWindowReady(code), 'Application#checkWindowReady()', this.logger); // Make sure to take a screenshot if waiting for window ready // takes unusually long to help diagnose issues when Code does // not seem to startup healthy. - const timeoutHandle = setTimeout(() => this.takeScreenshot(`checkWindowReady_instance_${Application.INSTANCES}`), 20000); + const timeoutHandle = setTimeout(() => this.takeScreenshot(`checkWindowReady_instance_${Application.INSTANCES}`), 10000); try { await windowReady; } finally { @@ -101,7 +101,7 @@ export class Application { } } - async stop(): Promise { + async stop(): Promise { if (this._code) { try { await this._code.exit(); @@ -120,12 +120,12 @@ export class Application { } private async takeScreenshot(name: string): Promise { - if (this.web || !this.legacy) { - return; // supported only on desktop (legacy) + const driver = this._code?.driver; + if (!(driver instanceof PlaywrightDriver)) { + return; } - // Desktop (legacy): call `stopTracing` to take a screenshot - return this._code?.stopTracing(name, true); + await driver.takeScreenshot(name); } private async startApplication(extraArgs: string[] = []): Promise { @@ -139,21 +139,17 @@ export class Application { return code; } - private async checkWindowReady(code: Code): Promise { - this.logger.log('checkWindowReady: begin'); + private async checkWindowReady(code: Code): Promise { + // This is legacy and will be removed when our old driver removes await code.waitForWindowIds(ids => ids.length > 0); - // TODO@bpasero productize this hack - if (code.driver instanceof PlaywrightDriver) { - await code.driver.page.locator('.monaco-workbench').waitFor({ timeout: 40000 }); - } else { - await code.waitForElement('.monaco-workbench'); - } + // We need a rendered workbench + await measureAndLog(code.waitForElement('.monaco-workbench', undefined, 300 /* 30s of retry */), 'Application#checkWindowReady: wait for .monaco-workbench element', this.logger); // Remote but not web: wait for a remote connection state change if (this.remote) { - await code.waitForTextContent('.monaco-workbench .statusbar-item[id="status.host"]', undefined, s => { + await measureAndLog(code.waitForTextContent('.monaco-workbench .statusbar-item[id="status.host"]', undefined, s => { this.logger.log(`checkWindowReady: remote indicator text is ${s}`); // The absence of "Opening Remote" is not a strict @@ -165,9 +161,7 @@ export class Application { // state changes away from the "Opening Remote..." one // we return. return !s.includes('Opening Remote'); - }, 300 /* = 30s of retry */); + }, 300 /* = 30s of retry */), 'Application#checkWindowReady: wait for remote indicator', this.logger); } - - this.logger.log('checkWindowReady: end'); } } diff --git a/test/automation/src/playwrightBrowserDriver.ts b/test/automation/src/playwrightBrowserDriver.ts index 7a9d74b8a81..d626a3630f7 100644 --- a/test/automation/src/playwrightBrowserDriver.ts +++ b/test/automation/src/playwrightBrowserDriver.ts @@ -38,7 +38,7 @@ export class PlaywrightDriver implements IDriver { constructor( private readonly application: playwright.Browser | playwright.ElectronApplication, private readonly context: playwright.BrowserContext, - readonly page: playwright.Page, // TODO@bpasero make private again + private readonly page: playwright.Page, private readonly serverPid: number | undefined, private readonly options: LaunchOptions ) { @@ -77,6 +77,16 @@ export class PlaywrightDriver implements IDriver { } } + async takeScreenshot(name: string): Promise { + try { + const persistPath = join(this.options.logsPath, `playwright-screenshot-${PlaywrightDriver.traceCounter++}-${name.replace(/\s+/g, '-')}.png`); + + await measureAndLog(this.page.screenshot({ path: persistPath, type: 'png' }), 'takeScreenshot', this.options.logger); + } catch (error) { + // Ignore + } + } + async exitApplication() { // Stop tracing diff --git a/test/automation/src/playwrightElectronDriver.ts b/test/automation/src/playwrightElectronDriver.ts index de6859240e0..bda892b5914 100644 --- a/test/automation/src/playwrightElectronDriver.ts +++ b/test/automation/src/playwrightElectronDriver.ts @@ -48,12 +48,28 @@ async function launchElectron(configuration: IElectronConfiguration, options: La } } - window.on('pageerror', async (error) => logger.log(`Playwright ERROR: page error: ${error}`)); - window.on('crash', () => logger.log('Playwright ERROR: page crash')); - window.on('close', () => logger.log('Playwright: page close')); + if (options.verbose) { + electron.on('window', () => logger.log(`Playwright (Electron): electron.on('window')`)); + electron.on('close', () => logger.log(`Playwright (Electron): electron.on('close')`)); + + context.on('page', () => logger.log(`Playwright (Electron): context.on('page')`)); + context.on('requestfailed', (e) => logger.log(`Playwright (Electron): context.on('requestfailed') [${e.failure()?.errorText} for ${e.url()}]`)); + + window.on('console', (e) => logger.log(`Playwright (Electron): window.on('console') [${e.text()}]`)); + window.on('dialog', () => logger.log(`Playwright (Electron): window.on('dialog')`)); + window.on('domcontentloaded', () => logger.log(`Playwright (Electron): window.on('domcontentloaded')`)); + window.on('load', () => logger.log(`Playwright (Electron): window.on('load')`)); + window.on('popup', () => logger.log(`Playwright (Electron): window.on('popup')`)); + window.on('framenavigated', () => logger.log(`Playwright (Electron): window.on('framenavigated')`)); + window.on('requestfailed', (e) => logger.log(`Playwright (Electron): window.on('requestfailed') [${e.failure()?.errorText} for ${e.url()}]`)); + } + + window.on('pageerror', async (error) => logger.log(`Playwright (Electron) ERROR: page error: ${error}`)); + window.on('crash', () => logger.log('Playwright (Electron) ERROR: page crash')); + window.on('close', () => logger.log('Playwright (Electron): page close')); window.on('response', async (response) => { if (response.status() >= 400) { - logger.log(`Playwright ERROR: HTTP status ${response.status()} for ${response.url()}`); + logger.log(`Playwright (Electron) ERROR: HTTP status ${response.status()} for ${response.url()}`); } });