Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-05-08 09:08:48 +01:00
Code Issues Packages Projects Releases Wiki Activity

smoke - always create screenshot on failure on desktop

Browse Source
This commit is contained in:
Benjamin Pasero
2021-12-09 09:51:55 +01:00
parent ae0ee547ba
commit 27e38a8e27
9 changed files with 36 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build/azure-pipelines/darwin/product-build-darwin.yml
+2 -2
View File
@@ -223,7 +223,7 @@ steps:
set -e
APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
APP_NAME="`ls $APP_ROOT | head -n 1`"
yarn smoketest-no-compile --build "$APP_ROOT/$APP_NAME" --screenshots $(Build.SourcesDirectory)/.build/logs/smoke-tests
yarn smoketest-no-compile --build "$APP_ROOT/$APP_NAME"
timeoutInMinutes: 10
displayName: Run smoke tests (Electron)
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
@@ -233,7 +233,7 @@ steps:
APP_ROOT=$(agent.builddirectory)/VSCode-darwin-$(VSCODE_ARCH)
APP_NAME="`ls $APP_ROOT | head -n 1`"
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-darwin" \
yarn smoketest-no-compile --build "$APP_ROOT/$APP_NAME" --remote --screenshots $(Build.SourcesDirectory)/.build/logs/smoke-tests-remote
yarn smoketest-no-compile --build "$APP_ROOT/$APP_NAME" --remote
timeoutInMinutes: 10
displayName: Run smoke tests (Remote)
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
build/azure-pipelines/linux/product-build-linux.yml
+2 -2
View File
@@ -214,7 +214,7 @@ steps:
- script: |
set -e
APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
yarn smoketest-no-compile --build "$APP_PATH" --electronArgs="--disable-dev-shm-usage --use-gl=swiftshader" --screenshots $(Build.SourcesDirectory)/.build/logs/smoke-tests
yarn smoketest-no-compile --build "$APP_PATH" --electronArgs="--disable-dev-shm-usage --use-gl=swiftshader"
timeoutInMinutes: 10
displayName: Run smoke tests (Electron)
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
@@ -223,7 +223,7 @@ steps:
set -e
APP_PATH=$(agent.builddirectory)/VSCode-linux-$(VSCODE_ARCH)
VSCODE_REMOTE_SERVER_PATH="$(agent.builddirectory)/vscode-reh-linux-$(VSCODE_ARCH)" \
yarn smoketest-no-compile --build "$APP_PATH" --remote --electronArgs="--disable-dev-shm-usage --use-gl=swiftshader" --screenshots $(Build.SourcesDirectory)/.build/logs/smoke-tests-remote
yarn smoketest-no-compile --build "$APP_PATH" --remote --electronArgs="--disable-dev-shm-usage --use-gl=swiftshader"
timeoutInMinutes: 10
displayName: Run smoke tests (Remote)
condition: and(succeeded(), eq(variables['VSCODE_ARCH'], 'x64'), eq(variables['VSCODE_STEP_ON_IT'], 'false'))
build/azure-pipelines/win32/product-build-win32.yml
+2 -2
View File
@@ -210,7 +210,7 @@ steps:
. build/azure-pipelines/win32/exec.ps1
$ErrorActionPreference = "Stop"
$AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
exec { yarn smoketest-no-compile --build "$AppRoot" --screenshots $(Build.SourcesDirectory)\.build\logs\smoke-tests }
exec { yarn smoketest-no-compile --build "$AppRoot" }
displayName: Run smoke tests (Electron)
timeoutInMinutes: 10
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
@@ -220,7 +220,7 @@ steps:
$ErrorActionPreference = "Stop"
$AppRoot = "$(agent.builddirectory)\VSCode-win32-$(VSCODE_ARCH)"
$env:VSCODE_REMOTE_SERVER_PATH = "$(agent.builddirectory)\vscode-reh-win32-$(VSCODE_ARCH)"
exec { yarn smoketest-no-compile --build "$AppRoot" --remote --screenshots $(Build.SourcesDirectory)\.build\logs\smoke-tests-remote }
exec { yarn smoketest-no-compile --build "$AppRoot" --remote }
displayName: Run smoke tests (Remote)
timeoutInMinutes: 10
condition: and(succeeded(), eq(variables['VSCODE_STEP_ON_IT'], 'false'), ne(variables['VSCODE_ARCH'], 'arm64'))
src/vs/platform/driver/electron-main/driver.ts
+15 -2
View File
@@ -20,6 +20,10 @@ import { IInstantiationService } from 'vs/platform/instantiation/common/instanti
import { USLayoutResolvedKeybinding } from 'vs/platform/keybinding/common/usLayoutResolvedKeybinding';
import { ILifecycleMainService } from 'vs/platform/lifecycle/electron-main/lifecycleMainService';
import { IWindowsMainService } from 'vs/platform/windows/electron-main/windows';
import { IFileService } from 'vs/platform/files/common/files';
import { URI } from 'vs/base/common/uri';
import { join } from 'vs/base/common/path';
import { VSBuffer } from 'vs/base/common/buffer';
function isSilentKeyCode(keyCode: KeyCode) {
return keyCode < KeyCode.Digit0;
@@ -37,7 +41,9 @@ export class Driver implements IDriver, IWindowDriverRegistry {
private windowServer: IPCServer,
private options: IDriverOptions,
@IWindowsMainService private readonly windowsMainService: IWindowsMainService,
@ILifecycleMainService private readonly lifecycleMainService: ILifecycleMainService
@ILifecycleMainService private readonly lifecycleMainService: ILifecycleMainService,
@IFileService private readonly fileService: IFileService,
@IEnvironmentMainService private readonly environmentMainService: IEnvironmentMainService
) { }
async registerWindowDriver(windowId: number): Promise<IDriverOptions> {
@@ -74,7 +80,14 @@ export class Driver implements IDriver, IWindowDriverRegistry {
}
async stopTracing(windowId: number, name: string, persist: boolean): Promise<void> {
// ignore - tracing is not implemented yet
if (!persist) {
return;
}
const raw = await this.capturePage(windowId);
const buffer = Buffer.from(raw, 'base64');
await this.fileService.writeFile(URI.file(join(this.environmentMainService.logsPath, `${name}.png`)), VSBuffer.wrap(buffer));
}
async reloadWindow(windowId: number): Promise<void> {
test/automation/src/application.ts
+1 -15
View File
@@ -3,11 +3,9 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as fs from 'fs';
import * as path from 'path';
import { Workbench } from './workbench';
import { Code, launch, LaunchOptions } from './code';
import { Logger, measureAndLog } from './logger';
import { Logger } from './logger';
export const enum Quality {
Dev,
@@ -19,7 +17,6 @@ export interface ApplicationOptions extends LaunchOptions {
quality: Quality;
workspacePath: string;
waitTime: number;
screenshotsPath: string | null;
}
export class Application {
@@ -92,17 +89,6 @@ export class Application {
}
}
async captureScreenshot(name: string): Promise<void> {
if (this.options.screenshotsPath) {
const raw = await measureAndLog(this.code.capturePage(), 'capturePage', this.logger);
const buffer = Buffer.from(raw, 'base64');
const screenshotPath = path.join(this.options.screenshotsPath, `${name}.png`);
this.logger.log('Screenshot recorded:', screenshotPath);
fs.writeFileSync(screenshotPath, buffer);
}
}
async startTracing(name: string): Promise<void> {
await this._code?.startTracing(name);
}
test/smoke/README.md
-1
View File
@@ -61,7 +61,6 @@ xattr -d com.apple.quarantine <path to server with web folder zip>
- `--verbose` logs all the low level driver calls made to Code;
- `-f PATTERN` (alias `-g PATTERN`) filters the tests to be run. You can also use pretty much any mocha argument;
- `--screenshots SCREENSHOT_DIR` captures screenshots when tests fail.
- `--headless` will run playwright in headless mode when `--web` is used.
**Note**: you can enable verbose logging of playwright library by setting a `DEBUG` environment variable before running the tests (https://playwright.dev/docs/debug#verbose-api-logs)
test/smoke/src/areas/workbench/data-loss.test.ts
+2 -2
View File
@@ -12,7 +12,7 @@ export function setup(stableCodePath: string | undefined, isRemote: boolean, log
let app: Application | undefined = undefined;
// Shared before/after handling
installDiagnosticsHandler(logger);
installDiagnosticsHandler(logger, () => app);
installAppAfterHandler(() => app);
it('verifies opened editors are restored', async function () {
@@ -99,7 +99,7 @@ export function setup(stableCodePath: string | undefined, isRemote: boolean, log
let stableApp: Application | undefined = undefined;
// Shared before/after handling
installDiagnosticsHandler(logger);
installDiagnosticsHandler(logger, () => insidersApp ?? stableApp);
installAppAfterHandler(() => insidersApp ?? stableApp, async () => stableApp?.stop());
it('verifies opened editors are restored', async function () {
test/smoke/src/main.ts
-8
View File
@@ -39,7 +39,6 @@ const opts = minimist(args, {
'stable-build',
'wait-time',
'test-repo',
'screenshots',
'electronArgs'
],
boolean: [
@@ -56,7 +55,6 @@ const opts = minimist(args, {
remote?: boolean,
headless?: boolean,
web?: boolean,
screenshots?: string,
build?: string,
'stable-build'?: string,
browser?: string,
@@ -105,11 +103,6 @@ const workspacePath = path.join(testDataPath, 'vscode-smoketest-express');
const extensionsPath = path.join(testDataPath, 'extensions-dir');
mkdirp.sync(extensionsPath);
const screenshotsPath = opts.screenshots ? path.resolve(opts.screenshots) : null;
if (screenshotsPath) {
mkdirp.sync(screenshotsPath);
}
function fail(errorMessage): void {
logger.log(errorMessage);
process.exit(1);
@@ -332,7 +325,6 @@ before(async function () {
waitTime: parseInt(opts['wait-time'] || '0') || 20,
logger,
verbose: opts.verbose,
screenshotsPath,
remote: opts.remote,
web: opts.web,
headless: opts.headless,
test/smoke/src/utils.ts
+12 -6
View File
@@ -24,7 +24,7 @@ export function installAllHandlers(logger: Logger, optionsTransform?: (opts: App
installAppAfterHandler();
}
export function installDiagnosticsHandler(logger: Logger) {
export function installDiagnosticsHandler(logger: Logger, appFn?: () => Application | undefined) {
// Before each suite
before(async function () {
@@ -38,17 +38,22 @@ export function installDiagnosticsHandler(logger: Logger) {
beforeEach(async function () {
const testTitle = this.currentTest?.title;
logger.log('');
logger.log(`>>> Test start: '${testTitle}' <<<`);
logger.log(`>>> Test start: '${testTitle ?? 'unknown'}' <<<`);
logger.log('');
await this.app?.startTracing(testTitle);
const app: Application = appFn?.() ?? this.app;
await app?.startTracing(testTitle ?? 'unknown');
});
// After each test
afterEach(async function () {
const failed = this.currentTest?.state === 'failed';
const currentTest = this.currentTest;
if (!currentTest) {
return;
}
const testTitle = this.currentTest?.title;
const failed = currentTest.state === 'failed';
const testTitle = currentTest.title;
logger.log('');
if (failed) {
logger.log(`>>> !!! FAILURE !!! Test end: '${testTitle}' !!! FAILURE !!! <<<`);
@@ -57,7 +62,8 @@ export function installDiagnosticsHandler(logger: Logger) {
}
logger.log('');
await this.app?.stopTracing(this.currentTest?.title, failed);
const app: Application = appFn?.() ?? this.app;
await app?.stopTracing(testTitle.replace(/[^a-z0-9\-]/ig, '_'), failed);
});
}