From 27e38a8e2714b061c1e1647c553ab017f41b99a4 Mon Sep 17 00:00:00 2001 From: Benjamin Pasero Date: Thu, 9 Dec 2021 09:51:55 +0100 Subject: [PATCH] smoke - always create screenshot on failure on desktop --- .../darwin/product-build-darwin.yml | 4 ++-- .../linux/product-build-linux.yml | 4 ++-- .../win32/product-build-win32.yml | 4 ++-- src/vs/platform/driver/electron-main/driver.ts | 17 +++++++++++++++-- test/automation/src/application.ts | 16 +--------------- test/smoke/README.md | 1 - .../src/areas/workbench/data-loss.test.ts | 4 ++-- test/smoke/src/main.ts | 8 -------- test/smoke/src/utils.ts | 18 ++++++++++++------ 9 files changed, 36 insertions(+), 40 deletions(-) diff --git a/build/azure-pipelines/darwin/product-build-darwin.yml b/build/azure-pipelines/darwin/product-build-darwin.yml index 5ca9eeea492..960282ec8fc 100644 --- a/build/azure-pipelines/darwin/product-build-darwin.yml +++ b/build/azure-pipelines/darwin/product-build-darwin.yml @@ -223,7 +223,7 @@ steps: set -e APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH) APP_NAME="`ls $APP_ROOT | head -n 1`" - yarn smoketest-no-compile --build "$APP_ROOT/$APP_NAME" --screenshots $(Build.SourcesDirectory)/.build/logs/smoke-tests + yarn smoketest-no-compile --build "$APP_ROOT/$APP_NAME" timeoutInMinutes: 10 displayName: Run smoke tests (Electron) condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) @@ -233,7 +233,7 @@ steps: APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH) APP_NAME="`ls $APP_ROOT | head -n 1`" VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin" \ - yarn smoketest-no-compile --build "$APP_ROOT/$APP_NAME" --remote --screenshots $(Build.SourcesDirectory)/.build/logs/smoke-tests-remote + yarn smoketest-no-compile --build "$APP_ROOT/$APP_NAME" --remote timeoutInMinutes: 10 displayName: Run smoke tests (Remote) condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) diff --git a/build/azure-pipelines/linux/product-build-linux.yml b/build/azure-pipelines/linux/product-build-linux.yml index 2a612813309..14fa3121127 100644 --- a/build/azure-pipelines/linux/product-build-linux.yml +++ b/build/azure-pipelines/linux/product-build-linux.yml @@ -214,7 +214,7 @@ steps: - script: | set -e APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH) - yarn smoketest-no-compile --build "$APP_PATH" --electronArgs="--disable-dev-shm-usage --use-gl=swiftshader" --screenshots $(Build.SourcesDirectory)/.build/logs/smoke-tests + yarn smoketest-no-compile --build "$APP_PATH" --electronArgs="--disable-dev-shm-usage --use-gl=swiftshader" timeoutInMinutes: 10 displayName: Run smoke tests (Electron) condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) @@ -223,7 +223,7 @@ steps: set -e APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH) VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \ - yarn smoketest-no-compile --build "$APP_PATH" --remote --electronArgs="--disable-dev-shm-usage --use-gl=swiftshader" --screenshots $(Build.SourcesDirectory)/.build/logs/smoke-tests-remote + yarn smoketest-no-compile --build "$APP_PATH" --remote --electronArgs="--disable-dev-shm-usage --use-gl=swiftshader" timeoutInMinutes: 10 displayName: Run smoke tests (Remote) condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false')) diff --git a/build/azure-pipelines/win32/product-build-win32.yml b/build/azure-pipelines/win32/product-build-win32.yml index c9b28addafd..a8c937858da 100644 --- a/build/azure-pipelines/win32/product-build-win32.yml +++ b/build/azure-pipelines/win32/product-build-win32.yml @@ -210,7 +210,7 @@ steps: . build/azure-pipelines/win32/exec.ps1 $ErrorActionPreference = "Stop" $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)" - exec { yarn smoketest-no-compile --build "$AppRoot" --screenshots $(Build.SourcesDirectory)\.build\logs\smoke-tests } + exec { yarn smoketest-no-compile --build "$AppRoot" } displayName: Run smoke tests (Electron) timeoutInMinutes: 10 condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64')) @@ -220,7 +220,7 @@ steps: $ErrorActionPreference = "Stop" $AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)" $env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)" - exec { yarn smoketest-no-compile --build "$AppRoot" --remote --screenshots $(Build.SourcesDirectory)\.build\logs\smoke-tests-remote } + exec { yarn smoketest-no-compile --build "$AppRoot" --remote } displayName: Run smoke tests (Remote) timeoutInMinutes: 10 condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64')) diff --git a/src/vs/platform/driver/electron-main/driver.ts b/src/vs/platform/driver/electron-main/driver.ts index 17d85fa4592..2c3ecb669ab 100644 --- a/src/vs/platform/driver/electron-main/driver.ts +++ b/src/vs/platform/driver/electron-main/driver.ts @@ -20,6 +20,10 @@ import { IInstantiationService } from 'vs/platform/instantiation/common/instanti import { USLayoutResolvedKeybinding } from 'vs/platform/keybinding/common/usLayoutResolvedKeybinding'; import { ILifecycleMainService } from 'vs/platform/lifecycle/electron-main/lifecycleMainService'; import { IWindowsMainService } from 'vs/platform/windows/electron-main/windows'; +import { IFileService } from 'vs/platform/files/common/files'; +import { URI } from 'vs/base/common/uri'; +import { join } from 'vs/base/common/path'; +import { VSBuffer } from 'vs/base/common/buffer'; function isSilentKeyCode(keyCode: KeyCode) { return keyCode < KeyCode.Digit0; @@ -37,7 +41,9 @@ export class Driver implements IDriver, IWindowDriverRegistry { private windowServer: IPCServer, private options: IDriverOptions, @IWindowsMainService private readonly windowsMainService: IWindowsMainService, - @ILifecycleMainService private readonly lifecycleMainService: ILifecycleMainService + @ILifecycleMainService private readonly lifecycleMainService: ILifecycleMainService, + @IFileService private readonly fileService: IFileService, + @IEnvironmentMainService private readonly environmentMainService: IEnvironmentMainService ) { } async registerWindowDriver(windowId: number): Promise { @@ -74,7 +80,14 @@ export class Driver implements IDriver, IWindowDriverRegistry { } async stopTracing(windowId: number, name: string, persist: boolean): Promise { - // ignore - tracing is not implemented yet + if (!persist) { + return; + } + + const raw = await this.capturePage(windowId); + const buffer = Buffer.from(raw, 'base64'); + + await this.fileService.writeFile(URI.file(join(this.environmentMainService.logsPath, `${name}.png`)), VSBuffer.wrap(buffer)); } async reloadWindow(windowId: number): Promise { diff --git a/test/automation/src/application.ts b/test/automation/src/application.ts index 5799a119498..39024dd49e9 100644 --- a/test/automation/src/application.ts +++ b/test/automation/src/application.ts @@ -3,11 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import * as fs from 'fs'; -import * as path from 'path'; import { Workbench } from './workbench'; import { Code, launch, LaunchOptions } from './code'; -import { Logger, measureAndLog } from './logger'; +import { Logger } from './logger'; export const enum Quality { Dev, @@ -19,7 +17,6 @@ export interface ApplicationOptions extends LaunchOptions { quality: Quality; workspacePath: string; waitTime: number; - screenshotsPath: string | null; } export class Application { @@ -92,17 +89,6 @@ export class Application { } } - async captureScreenshot(name: string): Promise { - if (this.options.screenshotsPath) { - const raw = await measureAndLog(this.code.capturePage(), 'capturePage', this.logger); - const buffer = Buffer.from(raw, 'base64'); - const screenshotPath = path.join(this.options.screenshotsPath, `${name}.png`); - this.logger.log('Screenshot recorded:', screenshotPath); - - fs.writeFileSync(screenshotPath, buffer); - } - } - async startTracing(name: string): Promise { await this._code?.startTracing(name); } diff --git a/test/smoke/README.md b/test/smoke/README.md index 84c2030a336..9d95d597675 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -61,7 +61,6 @@ xattr -d com.apple.quarantine - `--verbose` logs all the low level driver calls made to Code; - `-f PATTERN` (alias `-g PATTERN`) filters the tests to be run. You can also use pretty much any mocha argument; -- `--screenshots SCREENSHOT_DIR` captures screenshots when tests fail. - `--headless` will run playwright in headless mode when `--web` is used. **Note**: you can enable verbose logging of playwright library by setting a `DEBUG` environment variable before running the tests (https://playwright.dev/docs/debug#verbose-api-logs) diff --git a/test/smoke/src/areas/workbench/data-loss.test.ts b/test/smoke/src/areas/workbench/data-loss.test.ts index 0b9a48c5b9a..cda909d1f3f 100644 --- a/test/smoke/src/areas/workbench/data-loss.test.ts +++ b/test/smoke/src/areas/workbench/data-loss.test.ts @@ -12,7 +12,7 @@ export function setup(stableCodePath: string | undefined, isRemote: boolean, log let app: Application | undefined = undefined; // Shared before/after handling - installDiagnosticsHandler(logger); + installDiagnosticsHandler(logger, () => app); installAppAfterHandler(() => app); it('verifies opened editors are restored', async function () { @@ -99,7 +99,7 @@ export function setup(stableCodePath: string | undefined, isRemote: boolean, log let stableApp: Application | undefined = undefined; // Shared before/after handling - installDiagnosticsHandler(logger); + installDiagnosticsHandler(logger, () => insidersApp ?? stableApp); installAppAfterHandler(() => insidersApp ?? stableApp, async () => stableApp?.stop()); it('verifies opened editors are restored', async function () { diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts index 5c06fc05ee9..4dc35e3a1d0 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/main.ts @@ -39,7 +39,6 @@ const opts = minimist(args, { 'stable-build', 'wait-time', 'test-repo', - 'screenshots', 'electronArgs' ], boolean: [ @@ -56,7 +55,6 @@ const opts = minimist(args, { remote?: boolean, headless?: boolean, web?: boolean, - screenshots?: string, build?: string, 'stable-build'?: string, browser?: string, @@ -105,11 +103,6 @@ const workspacePath = path.join(testDataPath, 'vscode-smoketest-express'); const extensionsPath = path.join(testDataPath, 'extensions-dir'); mkdirp.sync(extensionsPath); -const screenshotsPath = opts.screenshots ? path.resolve(opts.screenshots) : null; -if (screenshotsPath) { - mkdirp.sync(screenshotsPath); -} - function fail(errorMessage): void { logger.log(errorMessage); process.exit(1); @@ -332,7 +325,6 @@ before(async function () { waitTime: parseInt(opts['wait-time'] || '0') || 20, logger, verbose: opts.verbose, - screenshotsPath, remote: opts.remote, web: opts.web, headless: opts.headless, diff --git a/test/smoke/src/utils.ts b/test/smoke/src/utils.ts index 4a9721ffda9..4d3fefc49a6 100644 --- a/test/smoke/src/utils.ts +++ b/test/smoke/src/utils.ts @@ -24,7 +24,7 @@ export function installAllHandlers(logger: Logger, optionsTransform?: (opts: App installAppAfterHandler(); } -export function installDiagnosticsHandler(logger: Logger) { +export function installDiagnosticsHandler(logger: Logger, appFn?: () => Application | undefined) { // Before each suite before(async function () { @@ -38,17 +38,22 @@ export function installDiagnosticsHandler(logger: Logger) { beforeEach(async function () { const testTitle = this.currentTest?.title; logger.log(''); - logger.log(`>>> Test start: '${testTitle}' <<<`); + logger.log(`>>> Test start: '${testTitle ?? 'unknown'}' <<<`); logger.log(''); - await this.app?.startTracing(testTitle); + const app: Application = appFn?.() ?? this.app; + await app?.startTracing(testTitle ?? 'unknown'); }); // After each test afterEach(async function () { - const failed = this.currentTest?.state === 'failed'; + const currentTest = this.currentTest; + if (!currentTest) { + return; + } - const testTitle = this.currentTest?.title; + const failed = currentTest.state === 'failed'; + const testTitle = currentTest.title; logger.log(''); if (failed) { logger.log(`>>> !!! FAILURE !!! Test end: '${testTitle}' !!! FAILURE !!! <<<`); @@ -57,7 +62,8 @@ export function installDiagnosticsHandler(logger: Logger) { } logger.log(''); - await this.app?.stopTracing(this.currentTest?.title, failed); + const app: Application = appFn?.() ?? this.app; + await app?.stopTracing(testTitle.replace(/[^a-z0-9\-]/ig, '_'), failed); }); }