diff --git a/test/automation/package.json b/test/automation/package.json index 297dce969b7..744778d5785 100644 --- a/test/automation/package.json +++ b/test/automation/package.json @@ -22,18 +22,18 @@ "devDependencies": { "@types/mkdirp": "0.5.1", "@types/ncp": "2.0.1", - "@types/node": "8.0.33", - "@types/puppeteer": "^1.19.0", + "@types/debug": "4.1.5", + "@types/node": "^12.11.7", "@types/tmp": "0.1.0", "concurrently": "^3.5.1", "cpx": "^1.5.0", - "typescript": "2.9.2", + "typescript": "3.7.5", "watch": "^1.0.2" }, "dependencies": { "mkdirp": "^0.5.1", "ncp": "^2.0.0", - "puppeteer": "^1.19.0", + "playwright": "0.10.0", "tmp": "0.1.0", "vscode-uri": "^2.0.3" } diff --git a/test/automation/src/application.ts b/test/automation/src/application.ts index 00005eebbf7..e95b5f561c2 100644 --- a/test/automation/src/application.ts +++ b/test/automation/src/application.ts @@ -126,6 +126,7 @@ export class Application { extraArgs, remote: this.options.remote, web: this.options.web, + browser: this.options.browser, headless: this.options.headless }); diff --git a/test/automation/src/code.ts b/test/automation/src/code.ts index 9900d92a880..dfe20727b8d 100644 --- a/test/automation/src/code.ts +++ b/test/automation/src/code.ts @@ -10,7 +10,7 @@ import * as fs from 'fs'; import * as mkdirp from 'mkdirp'; import { tmpName } from 'tmp'; import { IDriver, connect as connectElectronDriver, IDisposable, IElement, Thenable } from './driver'; -import { connect as connectPuppeteerDriver, launch } from './puppeteerDriver'; +import { connect as connectPlaywrightDriver, launch } from './playwrightDriver'; import { Logger } from './logger'; import { ncp } from 'ncp'; import { URI } from 'vscode-uri'; @@ -101,6 +101,8 @@ export interface SpawnOptions { remote?: boolean; /** Run in the web */ web?: boolean; + /** A specific browser to use (requires web: true) */ + browser?: 'chromium' | 'webkit' | 'firefox'; /** Run in headless mode (only applies when web is true) */ headless?: boolean; } @@ -120,68 +122,69 @@ export async function spawn(options: SpawnOptions): Promise { const outPath = codePath ? getBuildOutPath(codePath) : getDevOutPath(); const handle = await createDriverHandle(); - const args = [ - options.workspacePath, - '--skip-getting-started', - '--skip-release-notes', - '--sticky-quickopen', - '--disable-telemetry', - '--disable-updates', - '--disable-crash-reporter', - `--extensions-dir=${options.extensionsPath}`, - `--user-data-dir=${options.userDataDir}`, - '--driver', handle - ]; - - const env = process.env; - - if (options.remote) { - // Replace workspace path with URI - args[0] = `--${options.workspacePath.endsWith('.code-workspace') ? 'file' : 'folder'}-uri=vscode-remote://test+test/${URI.file(options.workspacePath).path}`; - - if (codePath) { - // running against a build: copy the test resolver extension - const testResolverExtPath = path.join(options.extensionsPath, 'vscode-test-resolver'); - if (!fs.existsSync(testResolverExtPath)) { - const orig = path.join(repoPath, 'extensions', 'vscode-test-resolver'); - await new Promise((c, e) => ncp(orig, testResolverExtPath, err => err ? e(err) : c())); - } - } - args.push('--enable-proposed-api=vscode.vscode-test-resolver'); - const remoteDataDir = `${options.userDataDir}-server`; - mkdirp.sync(remoteDataDir); - env['TESTRESOLVER_DATA_FOLDER'] = remoteDataDir; - } - - if (!codePath) { - args.unshift(repoPath); - } - - if (options.verbose) { - args.push('--driver-verbose'); - } - - if (options.log) { - args.push('--log', options.log); - } - - if (options.extraArgs) { - args.push(...options.extraArgs); - } - let child: cp.ChildProcess | undefined; let connectDriver: typeof connectElectronDriver; if (options.web) { - await launch(args); - connectDriver = connectPuppeteerDriver.bind(connectPuppeteerDriver, !!options.headless); + await launch(options.userDataDir, options.workspacePath, options.codePath); + connectDriver = connectPlaywrightDriver.bind(connectPlaywrightDriver, !!options.headless, options.browser); } else { + const env = process.env; + + const args = [ + options.workspacePath, + '--skip-getting-started', + '--skip-release-notes', + '--sticky-quickopen', + '--disable-telemetry', + '--disable-updates', + '--disable-crash-reporter', + `--extensions-dir=${options.extensionsPath}`, + `--user-data-dir=${options.userDataDir}`, + '--driver', handle + ]; + + if (options.remote) { + // Replace workspace path with URI + args[0] = `--${options.workspacePath.endsWith('.code-workspace') ? 'file' : 'folder'}-uri=vscode-remote://test+test/${URI.file(options.workspacePath).path}`; + + if (codePath) { + // running against a build: copy the test resolver extension + const testResolverExtPath = path.join(options.extensionsPath, 'vscode-test-resolver'); + if (!fs.existsSync(testResolverExtPath)) { + const orig = path.join(repoPath, 'extensions', 'vscode-test-resolver'); + await new Promise((c, e) => ncp(orig, testResolverExtPath, err => err ? e(err) : c())); + } + } + args.push('--enable-proposed-api=vscode.vscode-test-resolver'); + const remoteDataDir = `${options.userDataDir}-server`; + mkdirp.sync(remoteDataDir); + env['TESTRESOLVER_DATA_FOLDER'] = remoteDataDir; + } + + if (!codePath) { + args.unshift(repoPath); + } + + if (options.verbose) { + args.push('--driver-verbose'); + } + + if (options.log) { + args.push('--log', options.log); + } + + if (options.extraArgs) { + args.push(...options.extraArgs); + } + const spawnOptions: cp.SpawnOptions = { env }; child = cp.spawn(electronPath, args, spawnOptions); instances.add(child); child.once('exit', () => instances.delete(child!)); connectDriver = connectElectronDriver; } + return connect(connectDriver, child, outPath, handle, options.logger); } diff --git a/test/automation/src/puppeteerDriver.ts b/test/automation/src/playwrightDriver.ts similarity index 74% rename from test/automation/src/puppeteerDriver.ts rename to test/automation/src/playwrightDriver.ts index 73a118b6043..d1832f0c2e5 100644 --- a/test/automation/src/puppeteerDriver.ts +++ b/test/automation/src/playwrightDriver.ts @@ -3,17 +3,18 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import * as puppeteer from 'puppeteer'; +import * as playwright from 'playwright'; import { ChildProcess, spawn } from 'child_process'; import { join } from 'path'; import { mkdir } from 'fs'; import { promisify } from 'util'; import { IDriver, IDisposable } from './driver'; +import { URI } from 'vscode-uri'; const width = 1200; const height = 800; -const vscodeToPuppeteerKey: { [key: string]: string } = { +const vscodeToPlaywrightKey: { [key: string]: string } = { cmd: 'Meta', ctrl: 'Control', shift: 'Shift', @@ -26,7 +27,7 @@ const vscodeToPuppeteerKey: { [key: string]: string } = { home: 'Home' }; -function buildDriver(browser: puppeteer.Browser, page: puppeteer.Page): IDriver { +function buildDriver(browser: playwright.Browser, page: playwright.Page): IDriver { const driver: IDriver = { _serviceBrand: undefined, getWindowIds: () => { @@ -45,8 +46,8 @@ function buildDriver(browser: puppeteer.Browser, page: puppeteer.Page): IDriver const keys = chord.split('+'); const keysDown: string[] = []; for (let i = 0; i < keys.length; i++) { - if (keys[i] in vscodeToPuppeteerKey) { - keys[i] = vscodeToPuppeteerKey[keys[i]]; + if (keys[i] in vscodeToPlaywrightKey) { + keys[i] = vscodeToPlaywrightKey[keys[i]]; } await page.keyboard.down(keys[i]); keysDown.push(keys[i]); @@ -68,7 +69,7 @@ function buildDriver(browser: puppeteer.Browser, page: puppeteer.Page): IDriver await driver.click(windowId, selector, 0, 0); await timeout(100); }, - setValue: async (windowId, selector, text) => page.evaluate(`window.driver.setValue('${selector}', '${text}')`), + setValue: async (windowId, selector, text) => page.evaluate(`window.driver.setValue('${selector}', '${text}')`).then(undefined), getTitle: (windowId) => page.evaluate(`window.driver.getTitle()`), isActiveElement: (windowId, selector) => page.evaluate(`window.driver.isActiveElement('${selector}')`), getElements: (windowId, selector, recursive) => page.evaluate(`window.driver.getElements('${selector}', ${recursive})`), @@ -86,31 +87,32 @@ function timeout(ms: number): Promise { // function runInDriver(call: string, args: (string | boolean)[]): Promise {} -let args: string[] | undefined; let server: ChildProcess | undefined; let endpoint: string | undefined; +let workspacePath: string | undefined; -export async function launch(_args: string[]): Promise { - args = _args; - const agentFolder = args.filter(e => e.includes('--user-data-dir='))[0].replace('--user-data-dir=', ''); +export async function launch(userDataDir: string, _workspacePath: string, codeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH): Promise { + workspacePath = _workspacePath; + const agentFolder = userDataDir; await promisify(mkdir)(agentFolder); const env = { VSCODE_AGENT_FOLDER: agentFolder, + VSCODE_REMOTE_SERVER_PATH: codeServerPath, ...process.env }; let serverLocation: string | undefined; - if (process.env.VSCODE_REMOTE_SERVER_PATH) { - serverLocation = join(process.env.VSCODE_REMOTE_SERVER_PATH, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`); + if (codeServerPath) { + serverLocation = join(codeServerPath, `server.${process.platform === 'win32' ? 'cmd' : 'sh'}`); } else { - serverLocation = join(args[0], `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`); + serverLocation = join(__dirname, '..', '..', '..', `resources/server/web.${process.platform === 'win32' ? 'bat' : 'sh'}`); } server = spawn( serverLocation, ['--browser', 'none', '--driver', 'web'], { env } ); - server.stderr.on('data', e => console.log('Server stderr: ' + e)); - server.stdout.on('data', e => console.log('Server stdout: ' + e)); + server.stderr?.on('data', e => console.log('Server stderr: ' + e)); + server.stdout?.on('data', e => console.log('Server stdout: ' + e)); process.on('exit', teardown); process.on('SIGINT', teardown); process.on('SIGTERM', teardown); @@ -126,7 +128,7 @@ function teardown(): void { function waitForEndpoint(): Promise { return new Promise(r => { - server!.stdout.on('data', (d: Buffer) => { + server!.stdout?.on('data', (d: Buffer) => { const matches = d.toString('ascii').match(/Web UI available at (.+)/); if (matches !== null) { r(matches[1]); @@ -135,20 +137,18 @@ function waitForEndpoint(): Promise { }); } -export function connect(headless: boolean, outPath: string, handle: string): Promise<{ client: IDisposable, driver: IDriver }> { +export function connect(headless: boolean, engine: 'chromium' | 'webkit' | 'firefox' = 'chromium'): Promise<{ client: IDisposable, driver: IDriver }> { return new Promise(async (c) => { - const browser = await puppeteer.launch({ + const browser = await playwright[engine].launch({ // Run in Edge dev on macOS // executablePath: '/Applications/Microsoft\ Edge\ Dev.app/Contents/MacOS/Microsoft\ Edge\ Dev', - headless, - slowMo: 80, - args: [`--window-size=${width},${height}`] + headless }); - const page = (await browser.pages())[0]; + const page = (await browser.defaultContext().pages())[0]; await page.setViewport({ width, height }); - await page.goto(`${endpoint}&folder=vscode-remote://localhost:9888${args![1]}`); + await page.goto(`${endpoint}&folder=vscode-remote://localhost:9888${URI.file(workspacePath!).path}`); const result = { - client: { dispose: () => teardown }, + client: { dispose: () => teardown() }, driver: buildDriver(browser, page) }; c(result); diff --git a/test/automation/tsconfig.json b/test/automation/tsconfig.json index eecaf21a699..24deb99ddf8 100644 --- a/test/automation/tsconfig.json +++ b/test/automation/tsconfig.json @@ -16,6 +16,6 @@ "exclude": [ "node_modules", "out", - "tools", + "tools" ] } diff --git a/test/automation/yarn.lock b/test/automation/yarn.lock index 94a1350861f..65be684bc57 100644 --- a/test/automation/yarn.lock +++ b/test/automation/yarn.lock @@ -2,6 +2,11 @@ # yarn lockfile v1 +"@types/debug@4.1.5": + version "4.1.5" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.5.tgz#b14efa8852b7768d898906613c23f688713e02cd" + integrity sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ== + "@types/mkdirp@0.5.1": version "0.5.1" resolved "https://registry.yarnpkg.com/@types/mkdirp/-/mkdirp-0.5.1.tgz#ea887cd024f691c1ca67cce20b7606b053e43b0f" @@ -21,17 +26,10 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-12.7.1.tgz#3b5c3a26393c19b400844ac422bd0f631a94d69d" integrity sha512-aK9jxMypeSrhiYofWWBf/T7O+KwaiAHzM4sveCdWPn71lzUSMimRnKzhXDKfKwV1kWoBo2P1aGgaIYGLf9/ljw== -"@types/node@8.0.33": - version "8.0.33" - resolved "https://registry.yarnpkg.com/@types/node/-/node-8.0.33.tgz#1126e94374014e54478092830704f6ea89df04cd" - integrity sha512-vmCdO8Bm1ExT+FWfC9sd9r4jwqM7o97gGy2WBshkkXbf/2nLAJQUrZfIhw27yVOtLUev6kSZc4cav/46KbDd8A== - -"@types/puppeteer@^1.19.0": - version "1.19.1" - resolved "https://registry.yarnpkg.com/@types/puppeteer/-/puppeteer-1.19.1.tgz#942ca62288953a0f5fbbc25c103b5f2ba28b60ab" - integrity sha512-ReWZvoEfMiJIA3AG+eM+nCx5GKrU2ANVYY5TC0nbpeiTCtnJbcqnmBbR8TkXMBTvLBYcuTOAELbTcuX73siDNQ== - dependencies: - "@types/node" "*" +"@types/node@^12.11.7": + version "12.12.26" + resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.26.tgz#213e153babac0ed169d44a6d919501e68f59dea9" + integrity sha512-UmUm94/QZvU5xLcUlNR8hA7Ac+fGpO1EG/a8bcWVz0P0LqtxFmun9Y2bbtuckwGboWJIT70DoWq1r3hb56n3DA== "@types/tmp@0.1.0": version "0.1.0" @@ -729,10 +727,10 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.4.tgz#44119abaf4bc64692a16ace34700fed9c03e2546" integrity sha512-pzXIvANXEFrc5oFFXRMkbLPQ2rXRoDERwDLyrcUxGhaZhgP54BBSl9Oheh7Vv0T090cszWBxPjkQQ5Sq1PbBRQ== -https-proxy-agent@^2.2.1: - version "2.2.3" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.3.tgz#fb6cd98ed5b9c35056b5a73cd01a8a721d7193d1" - integrity sha512-Ytgnz23gm2DVftnzqRRz2dOXZbGd2uiajSw/95bPp6v53zPRspQjLm/AfBgqbJ2qfeRXWIOMVLpp86+/5yX39Q== +https-proxy-agent@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-3.0.1.tgz#b8c286433e87602311b01c8ea34413d856a4af81" + integrity sha512-+ML2Rbh6DAuee7d07tYGEKOEi2voWPUGan+ExdPbPW6Z3svq+JCqr0v8WmKPOkz1vOVykPCBSuobe7G8GJUtVg== dependencies: agent-base "^4.3.0" debug "^3.1.0" @@ -938,6 +936,11 @@ isobject@^3.0.0, isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8= +jpeg-js@^0.3.6: + version "0.3.6" + resolved "https://registry.yarnpkg.com/jpeg-js/-/jpeg-js-0.3.6.tgz#c40382aac9506e7d1f2d856eb02f6c7b2a98b37c" + integrity sha512-MUj2XlMB8kpe+8DJUGH/3UJm4XpI8XEgZQ+CiHDeyrGoKPdW/8FJv6ku+3UiYm5Fz3CWaL+iXmD8Q4Ap6aC1Jw== + json-parse-better-errors@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz#bb867cfb3450e69107c131d1c514bab3dc8bcaa9" @@ -1320,6 +1323,35 @@ pify@^3.0.0: resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176" integrity sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY= +playwright-core@=0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-0.10.0.tgz#86699c9cc3e613d733e6635a54aceea1993013d5" + integrity sha512-yernA6yrrBhmb8M5eO6GZsJOrBKWOZszlu65Luz8LP7ryaDExN1sE9XjQBNbiwJ5Gfs8cehtAO7GfTDJt+Z2cQ== + dependencies: + debug "^4.1.0" + extract-zip "^1.6.6" + https-proxy-agent "^3.0.0" + jpeg-js "^0.3.6" + mime "^2.0.3" + pngjs "^3.4.0" + progress "^2.0.3" + proxy-from-env "^1.0.0" + rimraf "^2.6.1" + uuid "^3.4.0" + ws "^6.1.0" + +playwright@0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-0.10.0.tgz#d37f7e42e0e868dcc4ec35cb0a8dbc6248457642" + integrity sha512-f3VRME/PIO5NbcWnlCDfXwPC0DAZJ7ETkcAdE+sensLCOkfDtLh97E71ZuxNCaPYsUA6FIPi5syD8pHJW/4hQQ== + dependencies: + playwright-core "=0.10.0" + +pngjs@^3.4.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-3.4.0.tgz#99ca7d725965fb655814eaf65f38f12bbdbf555f" + integrity sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w== + posix-character-classes@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" @@ -1335,7 +1367,7 @@ process-nextick-args@~2.0.0: resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== -progress@^2.0.1: +progress@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8" integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA== @@ -1345,20 +1377,6 @@ proxy-from-env@^1.0.0: resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee" integrity sha1-M8UDmPcOp+uW0h97gXYwpVeRx+4= -puppeteer@^1.19.0: - version "1.19.0" - resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-1.19.0.tgz#e3b7b448c2c97933517078d7a2c53687361bebea" - integrity sha512-2S6E6ygpoqcECaagDbBopoSOPDv0pAZvTbnBgUY+6hq0/XDFDOLEMNlHF/SKJlzcaZ9ckiKjKDuueWI3FN/WXw== - dependencies: - debug "^4.1.0" - extract-zip "^1.6.6" - https-proxy-agent "^2.2.1" - mime "^2.0.3" - progress "^2.0.1" - proxy-from-env "^1.0.0" - rimraf "^2.6.1" - ws "^6.1.0" - randomatic@^3.0.0: version "3.1.1" resolved "https://registry.yarnpkg.com/randomatic/-/randomatic-3.1.1.tgz#b776efc59375984e36c537b2f51a1f0aff0da1ed" @@ -1751,10 +1769,10 @@ typedarray@^0.0.6: resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= -typescript@2.9.2: - version "2.9.2" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.9.2.tgz#1cbf61d05d6b96269244eb6a3bce4bd914e0f00c" - integrity sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w== +typescript@3.7.5: + version "3.7.5" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.5.tgz#0692e21f65fd4108b9330238aac11dd2e177a1ae" + integrity sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw== union-value@^1.0.0: version "1.0.1" @@ -1789,6 +1807,11 @@ util-deprecate@~1.0.1: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= +uuid@^3.4.0: + version "3.4.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" + integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== + validate-npm-package-license@^3.0.1: version "3.0.4" resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz#fc91f6b9c7ba15c857f4cb2c5defeec39d4f410a" diff --git a/test/smoke/README.md b/test/smoke/README.md index 7fc0ced719f..1060250f7fd 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -7,15 +7,22 @@ Make sure you are on **Node v10.x**. ```bash # Install Dependencies and Compile yarn --cwd test/smoke +yarn --cwd test/automation -# Dev +# Dev (Electron) yarn smoketest -# Build -yarn smoketest --build PATH_TO_NEW_BUILD_PARENT_FOLDER --stable-build PATH_TO_LAST_STABLE_BUILD_PARENT_FOLDER +# Dev (Web) +yarn smoketest --web --browser -# Remote -yarn smoketest --build PATH_TO_NEW_BUILD_PARENT_FOLDER --remote +# Build (Electron) +yarn smoketest --build --stable-build + +# Build (Web - read instructions below) +yarn smoketest --build --web --browser + +# Remote (Electron) +yarn smoketest --build --remote ``` ### Run for a release @@ -27,18 +34,33 @@ git checkout release/1.22 yarn --cwd test/smoke ``` +#### Electron + In addition to the new build to be released you will need the previous stable build so that the smoketest can test the data migration. The recommended way to make these builds available for the smoketest is by downloading their archive version (\*.zip) and extracting them into two folders. Pass the folder paths to the smoketest as follows: ```bash -yarn smoketest --build PATH_TO_NEW_RELEASE_PARENT_FOLDER --stable-build PATH_TO_LAST_STABLE_RELEASE_PARENT_FOLDER +yarn smoketest --build --stable-build ``` +#### Web + +**macOS**: if you have downloaded the server with web bits, make sure to run the following command before unzipping it to avoid security issues on startup: + +```bash +xattr -d com.apple.quarantine +``` + +There is no support for testing an old version to a new one yet, so simply configure the `--build` command line argument to point to +the web server folder which includes the web client bits (e.g. `vscode-server-darwin-web` for macOS). + +**Note**: make sure to point to the server that includes the client bits! + ### Debug - `--verbose` logs all the low level driver calls made to Code; -- `-f PATTERN` filters the tests to be run. You can also use pretty much any mocha argument; +- `-f PATTERN` (alias `-g PATTERN`) filters the tests to be run. You can also use pretty much any mocha argument; - `--screenshots SCREENSHOT_DIR` captures screenshots when tests fail. ### Develop diff --git a/test/smoke/package.json b/test/smoke/package.json index 2ae2926ada1..28acde77a9f 100644 --- a/test/smoke/package.json +++ b/test/smoke/package.json @@ -27,7 +27,7 @@ "rimraf": "^2.6.1", "strip-json-comments": "^2.0.1", "tmp": "0.0.33", - "typescript": "2.9.2", + "typescript": "3.7.5", "watch": "^1.0.2" } } diff --git a/test/smoke/src/areas/workbench/data-migration.test.ts b/test/smoke/src/areas/workbench/data-migration.test.ts index 0a403bef78d..659fd62952a 100644 --- a/test/smoke/src/areas/workbench/data-migration.test.ts +++ b/test/smoke/src/areas/workbench/data-migration.test.ts @@ -8,7 +8,6 @@ import { join } from 'path'; export function setup(stableCodePath: string, testDataPath: string) { - describe('Data Migration: This test MUST run before releasing by providing the --stable-build command line argument', () => { it(`verifies opened editors are restored`, async function () { if (!stableCodePath) { diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts index 7b747a65f66..f792cd1ffa8 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/main.ts @@ -47,6 +47,7 @@ process.once('exit', () => rimraf.sync(testDataPath)); const [, , ...args] = process.argv; const opts = minimist(args, { string: [ + 'browser', 'build', 'stable-build', 'wait-time', @@ -58,7 +59,8 @@ const opts = minimist(args, { 'verbose', 'remote', 'web', - 'headless' + 'headless', + 'ci' ], default: { verbose: false @@ -82,42 +84,46 @@ function fail(errorMessage): void { const repoPath = path.join(__dirname, '..', '..', '..'); -function getDevElectronPath(): string { - const buildPath = path.join(repoPath, '.build'); - const product = require(path.join(repoPath, 'product.json')); - - switch (process.platform) { - case 'darwin': - return path.join(buildPath, 'electron', `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron'); - case 'linux': - return path.join(buildPath, 'electron', `${product.applicationName}`); - case 'win32': - return path.join(buildPath, 'electron', `${product.nameShort}.exe`); - default: - throw new Error('Unsupported platform.'); - } -} - -function getBuildElectronPath(root: string): string { - switch (process.platform) { - case 'darwin': - return path.join(root, 'Contents', 'MacOS', 'Electron'); - case 'linux': { - const product = require(path.join(root, 'resources', 'app', 'product.json')); - return path.join(root, product.applicationName); - } - case 'win32': { - const product = require(path.join(root, 'resources', 'app', 'product.json')); - return path.join(root, `${product.nameShort}.exe`); - } - default: - throw new Error('Unsupported platform.'); - } -} - let quality: Quality; +// +// #### Electron Smoke Tests #### +// if (!opts.web) { + + function getDevElectronPath(): string { + const buildPath = path.join(repoPath, '.build'); + const product = require(path.join(repoPath, 'product.json')); + + switch (process.platform) { + case 'darwin': + return path.join(buildPath, 'electron', `${product.nameLong}.app`, 'Contents', 'MacOS', 'Electron'); + case 'linux': + return path.join(buildPath, 'electron', `${product.applicationName}`); + case 'win32': + return path.join(buildPath, 'electron', `${product.nameShort}.exe`); + default: + throw new Error('Unsupported platform.'); + } + } + + function getBuildElectronPath(root: string): string { + switch (process.platform) { + case 'darwin': + return path.join(root, 'Contents', 'MacOS', 'Electron'); + case 'linux': { + const product = require(path.join(root, 'resources', 'app', 'product.json')); + return path.join(root, product.applicationName); + } + case 'win32': { + const product = require(path.join(root, 'resources', 'app', 'product.json')); + return path.join(root, `${product.nameShort}.exe`); + } + default: + throw new Error('Unsupported platform.'); + } + } + let testCodePath = opts.build; let stableCodePath = opts['stable-build']; let electronPath: string; @@ -152,8 +158,13 @@ if (!opts.web) { } else { quality = Quality.Stable; } -} else { - let testCodeServerPath = process.env.VSCODE_REMOTE_SERVER_PATH; +} + +// +// #### Web Smoke Tests #### +// +else { + const testCodeServerPath = opts.build || process.env.VSCODE_REMOTE_SERVER_PATH; if (typeof testCodeServerPath === 'string' && !fs.existsSync(testCodeServerPath)) { fail(`Can't find Code server at ${testCodeServerPath}.`); @@ -236,13 +247,13 @@ function createOptions(): ApplicationOptions { screenshotsPath, remote: opts.remote, web: opts.web, + browser: opts.browser, headless: opts.headless }; } before(async function () { - // allow two minutes for setup - this.timeout(2 * 60 * 1000); + this.timeout(2 * 60 * 1000); // allow two minutes for setup await setup(); this.defaultOptions = createOptions(); }); @@ -259,11 +270,7 @@ after(async function () { await new Promise((c, e) => rimraf(testDataPath, { maxBusyTries: 10 }, err => err ? e(err) : c())); }); -if (!opts.web) { - setupDataMigrationTests(opts['stable-build'], testDataPath); -} - -describe('Running Code', () => { +describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => { before(async function () { const app = new Application(this.defaultOptions); await app!.start(opts.web ? false : undefined); @@ -295,19 +302,25 @@ describe('Running Code', () => { }); } - if (!opts.web) { setupDataLossTests(); } - setupDataExplorerTests(); - if (!opts.web) { setupDataPreferencesTests(); } - setupDataSearchTests(); - setupDataCSSTests(); - setupDataEditorTests(); - setupDataStatusbarTests(!!opts.web); - setupDataExtensionTests(); - setupTerminalTests(); - if (!opts.web) { setupDataMultirootTests(); } - setupDataLocalizationTests(); -}); + // CI only tests (must be reliable) + if (opts.ci) { + // TODO@Ben figure out tests that can run continously and reliably + } -if (!opts.web) { - setupLaunchTests(); -} + // Non-CI execution (all tests) + else { + if (!opts.web) { setupDataMigrationTests(opts['stable-build'], testDataPath); } + if (!opts.web) { setupDataLossTests(); } + setupDataExplorerTests(); + if (!opts.web) { setupDataPreferencesTests(); } + setupDataSearchTests(); + setupDataCSSTests(); + setupDataEditorTests(); + setupDataStatusbarTests(!!opts.web); + setupDataExtensionTests(); + setupTerminalTests(); + if (!opts.web) { setupDataMultirootTests(); } + setupDataLocalizationTests(); + if (!opts.web) { setupLaunchTests(); } + } +}); diff --git a/test/smoke/test/index.js b/test/smoke/test/index.js index 71023e7e91f..5e33b701fa3 100644 --- a/test/smoke/test/index.js +++ b/test/smoke/test/index.js @@ -11,16 +11,14 @@ const suite = 'Smoke Tests'; const [, , ...args] = process.argv; const opts = minimist(args, { - string: [ - 'f' - ] + string: ['f', 'g'] }); const options = { - useColors: true, + color: true, timeout: 60000, slow: 30000, - grep: opts['f'] + grep: opts['f'] || opts['g'] }; if (process.env.BUILD_ARTIFACTSTAGINGDIRECTORY) { diff --git a/test/smoke/yarn.lock b/test/smoke/yarn.lock index 82626a55c7a..8649c9859c9 100644 --- a/test/smoke/yarn.lock +++ b/test/smoke/yarn.lock @@ -2122,10 +2122,10 @@ tree-kill@^1.1.0: resolved "https://registry.yarnpkg.com/tree-kill/-/tree-kill-1.2.0.tgz#5846786237b4239014f05db156b643212d4c6f36" integrity sha512-DlX6dR0lOIRDFxI0mjL9IYg6OTncLm/Zt+JiBhE5OlFcAR8yc9S7FFXU9so0oda47frdM/JFsk7UjNt9vscKcg== -typescript@2.9.2: - version "2.9.2" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.9.2.tgz#1cbf61d05d6b96269244eb6a3bce4bd914e0f00c" - integrity sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w== +typescript@3.7.5: + version "3.7.5" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-3.7.5.tgz#0692e21f65fd4108b9330238aac11dd2e177a1ae" + integrity sha512-/P5lkRXkWHNAbcJIiHPfRoKqyd7bsyCma1hZNUGfn20qm64T6ZBlrzprymeu918H+mB/0rIg2gGK/BXkhhYgBw== union-value@^1.0.0: version "1.0.1"