From d9db5d10ec9925c18f3ac118d9fa902fd918316d Mon Sep 17 00:00:00 2001 From: Benjamin Pasero Date: Sat, 9 Apr 2022 13:38:39 +0200 Subject: [PATCH] smoke - try to fix terminal tests and partition logs path per suite --- test/automation/src/code.ts | 2 +- test/automation/src/terminal.ts | 4 +-- .../smoke/src/areas/terminal/terminal.test.ts | 2 +- .../src/areas/workbench/data-loss.test.ts | 36 +++++++++++++------ test/smoke/src/main.ts | 16 ++++----- test/smoke/src/utils.ts | 9 ++++- 6 files changed, 45 insertions(+), 24 deletions(-) diff --git a/test/automation/src/code.ts b/test/automation/src/code.ts index 3bba5773f97..4ffdd0ae203 100644 --- a/test/automation/src/code.ts +++ b/test/automation/src/code.ts @@ -22,7 +22,7 @@ export interface LaunchOptions { userDataDir: string; readonly extensionsPath: string; readonly logger: Logger; - readonly logsPath: string; + logsPath: string; readonly verbose?: boolean; readonly extraArgs?: string[]; readonly remote?: boolean; diff --git a/test/automation/src/terminal.ts b/test/automation/src/terminal.ts index ec10f841279..61e64f664b6 100644 --- a/test/automation/src/terminal.ts +++ b/test/automation/src/terminal.ts @@ -73,8 +73,8 @@ export class Terminal { await this.code.dispatchKeybinding('enter'); await this.quickinput.waitForQuickInputClosed(); } - if (commandId === TerminalCommandId.Show) { - return await this._waitForTerminal(undefined); + if (commandId === TerminalCommandId.Show || commandId === TerminalCommandId.CreateNewEditor || commandId === TerminalCommandId.CreateNew || commandId === TerminalCommandId.NewWithProfile) { + return await this._waitForTerminal(commandId === TerminalCommandId.CreateNewEditor ? 'editor' : 'panel'); } } diff --git a/test/smoke/src/areas/terminal/terminal.test.ts b/test/smoke/src/areas/terminal/terminal.test.ts index dd98172bd18..7821552bc12 100644 --- a/test/smoke/src/areas/terminal/terminal.test.ts +++ b/test/smoke/src/areas/terminal/terminal.test.ts @@ -13,7 +13,7 @@ import { setup as setupTerminalTabsTests } from './terminal-tabs.test'; import { setup as setupTerminalSplitCwdTests } from './terminal-splitCwd.test'; export function setup(logger: Logger) { - describe.skip('Terminal', function () { // TODO@Tyriar (https://github.com/microsoft/vscode/issues/147063) + describe('Terminal', function () { // Retry tests 3 times to minimize build failures due to any flakiness this.retries(3); diff --git a/test/smoke/src/areas/workbench/data-loss.test.ts b/test/smoke/src/areas/workbench/data-loss.test.ts index aa6118a16ef..f116ec611f3 100644 --- a/test/smoke/src/areas/workbench/data-loss.test.ts +++ b/test/smoke/src/areas/workbench/data-loss.test.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { join } from 'path'; +import { dirname, join } from 'path'; import { Application, ApplicationOptions, Logger, Quality } from '../../../../automation'; import { createApp, timeout, installDiagnosticsHandler, installAppAfterHandler, getRandomUserDataDir } from '../../utils'; @@ -17,7 +17,10 @@ export function setup(ensureStableCode: () => string | undefined, logger: Logger installAppAfterHandler(() => app); it('verifies opened editors are restored', async function () { - app = createApp(this.defaultOptions); + app = createApp({ + ...this.defaultOptions, + logsPath: join(dirname(this.defaultOptions.logsPath), `test_verifies_opened_editors_are_restored`) + }); await app.start(); // Open 3 editors @@ -39,7 +42,10 @@ export function setup(ensureStableCode: () => string | undefined, logger: Logger }); it('verifies editors can save and restore', async function () { - app = createApp(this.defaultOptions); + app = createApp({ + ...this.defaultOptions, + logsPath: join(dirname(this.defaultOptions.logsPath), `test_verifies_editors_can_save_and_restore`) + }); await app.start(); const textToType = 'Hello, Code'; @@ -64,19 +70,22 @@ export function setup(ensureStableCode: () => string | undefined, logger: Logger }); it('verifies that "hot exit" works for dirty files (without delay)', function () { - return testHotExit.call(this, undefined); + return testHotExit.call(this, 'test_verifies_that_hot_exit_works_for_dirty_files_without_delay', undefined); }); it('verifies that "hot exit" works for dirty files (with delay)', function () { - return testHotExit.call(this, 2000); + return testHotExit.call(this, 'test_verifies_that_hot_exit_works_for_dirty_files_with_delay', 2000); }); it('verifies that auto save triggers on shutdown', function () { - return testHotExit.call(this, undefined, true); + return testHotExit.call(this, 'test_verifies_that_auto_save_triggers_on_shutdown', undefined, true); }); - async function testHotExit(restartDelay: number | undefined, autoSave: boolean | undefined) { - app = createApp(this.defaultOptions); + async function testHotExit(title: string, restartDelay: number | undefined, autoSave: boolean | undefined) { + app = createApp({ + ...this.defaultOptions, + logsPath: join(dirname(this.defaultOptions.logsPath), title) + }); await app.start(); if (autoSave) { @@ -143,11 +152,13 @@ export function setup(ensureStableCode: () => string | undefined, logger: Logger } const userDataDir = getRandomUserDataDir(this.defaultOptions); + const logsPath = join(dirname(this.defaultOptions.logsPath), `test_verifies_opened_editors_are_restored_from_stable`); const stableOptions: ApplicationOptions = Object.assign({}, this.defaultOptions); stableOptions.codePath = stableCodePath; stableOptions.userDataDir = userDataDir; stableOptions.quality = Quality.Stable; + stableOptions.logsPath = logsPath; stableApp = new Application(stableOptions); await stableApp.start(); @@ -164,6 +175,7 @@ export function setup(ensureStableCode: () => string | undefined, logger: Logger const insiderOptions: ApplicationOptions = Object.assign({}, this.defaultOptions); insiderOptions.userDataDir = userDataDir; + insiderOptions.logsPath = logsPath; insidersApp = new Application(insiderOptions); await insidersApp.start(); @@ -178,14 +190,14 @@ export function setup(ensureStableCode: () => string | undefined, logger: Logger }); it('verifies that "hot exit" works for dirty files (without delay)', async function () { - return testHotExit.call(this, undefined); + return testHotExit.call(this, `test_verifies_that_hot_exit_works_for_dirty_files_without_delay_from_stable`, undefined); }); it('verifies that "hot exit" works for dirty files (with delay)', async function () { - return testHotExit.call(this, 2000); + return testHotExit.call(this, `test_verifies_that_hot_exit_works_for_dirty_files_with_delay_from_stable`, 2000); }); - async function testHotExit(restartDelay: number | undefined) { + async function testHotExit(title: string, restartDelay: number | undefined) { const stableCodePath = ensureStableCode(); if (!stableCodePath) { this.skip(); @@ -197,6 +209,7 @@ export function setup(ensureStableCode: () => string | undefined, logger: Logger stableOptions.codePath = stableCodePath; stableOptions.userDataDir = userDataDir; stableOptions.quality = Quality.Stable; + stableOptions.logsPath = join(dirname(this.defaultOptions.logsPath), title); stableApp = new Application(stableOptions); await stableApp.start(); @@ -225,6 +238,7 @@ export function setup(ensureStableCode: () => string | undefined, logger: Logger const insiderOptions: ApplicationOptions = Object.assign({}, this.defaultOptions); insiderOptions.userDataDir = userDataDir; + insiderOptions.logsPath = join(dirname(this.defaultOptions.logsPath), title); insidersApp = new Application(insiderOptions); await insidersApp.start(); diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts index 5d9027234ee..3e4bcd8ba11 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/main.ts @@ -64,7 +64,7 @@ const opts = minimist(args, { electronArgs?: string; }; -const logsPath = (() => { +const logsRootPath = (() => { const logsParentPath = path.join(rootPath, '.build', 'logs'); let logsName: string; @@ -89,12 +89,12 @@ function createLogger(): Logger { loggers.push(new ConsoleLogger()); } - // Prepare logs path - fs.rmSync(logsPath, { recursive: true, force: true, maxRetries: 3 }); - mkdirp.sync(logsPath); + // Prepare logs rot path + fs.rmSync(logsRootPath, { recursive: true, force: true, maxRetries: 3 }); + mkdirp.sync(logsRootPath); // Always log to log file - loggers.push(new FileLogger(path.join(logsPath, 'smoke-test-runner.log'))); + loggers.push(new FileLogger(path.join(logsRootPath, 'smoke-test-runner.log'))); return new MultiLogger(loggers); } @@ -316,7 +316,7 @@ async function setup(): Promise { logger.log('Smoketest setup done!\n'); } -// Before main suite (before all tests) +// Before all tests run setup before(async function () { this.timeout(5 * 60 * 1000); // increase since we download VSCode @@ -328,7 +328,7 @@ before(async function () { extensionsPath, waitTime: parseInt(opts['wait-time'] || '0') || 20, logger, - logsPath, + logsPath: path.join(logsRootPath, 'suite_unknown'), verbose: opts.verbose, remote: opts.remote, web: opts.web, @@ -336,7 +336,7 @@ before(async function () { tracing: opts.tracing, headless: opts.headless, browser: opts.browser, - extraArgs: (opts.electronArgs || '').split(' ').map(a => a.trim()).filter(a => !!a) + extraArgs: (opts.electronArgs || '').split(' ').map(arg => arg.trim()).filter(arg => !!arg) }; await setup(); diff --git a/test/smoke/src/utils.ts b/test/smoke/src/utils.ts index 7f16a1ed38c..17271b50a08 100644 --- a/test/smoke/src/utils.ts +++ b/test/smoke/src/utils.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { Suite, Context } from 'mocha'; +import { dirname, join } from 'path'; import { Application, ApplicationOptions, Logger } from '../../automation'; export function describeRepeat(n: number, description: string, callback: (this: Suite) => void): void { @@ -88,7 +89,13 @@ export function installDiagnosticsHandler(logger: Logger, appFn?: () => Applicat function installAppBeforeHandler(optionsTransform?: (opts: ApplicationOptions) => ApplicationOptions) { before(async function () { - this.app = createApp(this.defaultOptions, optionsTransform); + const suiteName = this.currentTest?.titlePath()[1] ?? 'unknown'; + + this.app = createApp({ + ...this.defaultOptions, + // Set a suite specific logs path + logsPath: join(dirname(this.defaultOptions.logsPath), `suite_${suiteName.replace(/[^a-z0-9\-]/ig, '_')}`) + }, optionsTransform); await this.app.start(); }); }