diff --git a/test/smoke/.vscode/launch.json b/test/smoke/.vscode/launch.json index 2de33bbb20b..25b4e7e4c0e 100644 --- a/test/smoke/.vscode/launch.json +++ b/test/smoke/.vscode/launch.json @@ -15,7 +15,7 @@ "--timeout", "999999", "--colors", - "${workspaceRoot}/out/main.js" + "${workspaceRoot}/out/tests.js" ], "outFiles": [ "${workspaceRoot}/out/**/*.js" ], "internalConsoleOptions": "openOnSessionStart", diff --git a/test/smoke/CONTRIBUTING.md b/test/smoke/CONTRIBUTING.md index c15016dc1ad..8aaef8a7875 100644 --- a/test/smoke/CONTRIBUTING.md +++ b/test/smoke/CONTRIBUTING.md @@ -1,5 +1,7 @@ # Architecture -* `main.ts` contains the main smoke test suite. It includes all tests separated into mocha `describe()` groups that represent each of the areas of [Smoke Test document](https://github.com/Microsoft/vscode/wiki/Smoke-Test). +* `main.js` is used to prepare all smoke test dependencies (fetching key bindings and 'Express' repository, running `npm install` there). +* `mocha-runner.js` launches Mocha programmatically. It is spawned in Node environment from main.js to ensure that it is possible to listen on `stderr`s (primary `process.stderr` is not readable otherwise). This is accomplished because WebDriverIO command deprecation warnings need to be redirected to a separate log. Those warnings are coming from WebDriverIO because ChromeDriver has not migrated from JsonWire to W3C WebDriver protocol. +* `tests.ts` contains the main smoke test suite. It includes all tests separated into mocha `describe()` groups that represent each of the areas of [Smoke Test document](https://github.com/Microsoft/vscode/wiki/Smoke-Test). * `./areas/` folder contains a `.ts` file per each area of the document. E.g. `'Search'` area goes under `'search.ts'`. Every area file contains a list of methods with the name that represents the action that can be performed in the corresponding test. This reduces the amount of test suite code and means that if the UI changes, the fix need only be applied in one place. The name of the method reflects the action the tester would do if he would perform the test manually. See [Selenium Page Objects Wiki](https://github.com/SeleniumHQ/selenium/wiki/PageObjects) and [Selenium Bot Style Tests Wiki](https://github.com/SeleniumHQ/selenium/wiki/Bot-Style-Tests) for a good explanation of the implementation. Every smoke test area contains methods that are used in a bot-style approach in `main.ts`. * `./spectron/` wraps the Spectron, with WebDriverIO API wrapped in `client.ts` and instance of Spectron Application is wrapped in `application.ts`. diff --git a/test/smoke/README.md b/test/smoke/README.md index 318ddb07517..501ec51f087 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -1,9 +1,9 @@ # VS Code Smoke Testing This repository contains the smoke test automation code with Spectron for Visual Studio Code. -The following command is used to run the tests: `.\scripts\run.ps1 -latest "path\to\Code.exe"` on Windows (from PowerShell) and `./scripts/run.sh path/to/binary` on Unix system. +The following command is used to run the tests: `npm test -- --latest "path/to/binary"`. -If you want to include 'Data Migration' area tests use `.\scripts\run.ps1 -latest "path\to\Code.exe" -stable "path\to\CurrentStable.exe"` and `./scripts/run.sh path/to/binary path/to/currentStable` respectively. +If you want to include 'Data Migration' area tests use `npm test -- --latest path/to/binary --stable path/to/currentStable` respectively. # Contributing diff --git a/test/smoke/package.json b/test/smoke/package.json index a5c3a78273b..4f637a5adf7 100644 --- a/test/smoke/package.json +++ b/test/smoke/package.json @@ -1,9 +1,10 @@ { "name": "code-oss-dev-smoke-test", "version": "0.1.0", - "main": "./out/main.js", + "main": "./src/main.js", "scripts": { - "test": "mocha -u tdd --timeout 360000 --retries 2 --slow 50000 --colors ./out/main.js 2> test_data/errors.log" + "pretest": "tsc", + "test": "node src/main.js" }, "devDependencies": { "@types/mocha": "^2.2.41", @@ -14,6 +15,8 @@ "mocha": "^3.2.0", "spectron": "^3.6.4", "typescript": "^2.2.2", - "rimraf": "^2.6.1" + "rimraf": "^2.6.1", + "commander": "^2.9.0", + "simple-git": "^1.73.0" } } diff --git a/test/smoke/scripts/run.ps1 b/test/smoke/scripts/run.ps1 deleted file mode 100644 index 5883ce0dbe8..00000000000 --- a/test/smoke/scripts/run.ps1 +++ /dev/null @@ -1,44 +0,0 @@ -Param( - [Parameter(Position=0,mandatory=$true)] - [string]$arch -) - - -# Setup sample repository for the smoke test -Set-Location .. -if (-Not (Test-Path vscode-smoketest-express)) { - git clone https://github.com/Microsoft/vscode-smoketest-express.git - Set-Location ./vscode-smoketest-express -} else { - Set-Location ./vscode-smoketest-express - git fetch origin master - git reset --hard FETCH_HEAD - git clean -fd -} -npm install - -Write-Output "My path: " + $(pwd) - -# Setup the test directory for running -Set-Location ..\smoke -if (-Not (Test-Path node_modules)) { - npm install -} - -# Configure environment variables -$env:VSCODE_LATEST_PATH = "$(pwd)\..\VSCode-win32-$arch\Code - Insiders.exe" -# $env:VSCODE_STABLE_PATH = $stable -$env:SMOKETEST_REPO = "..\vscode-smoketest-express" - -if ($env:VSCODE_LATEST_PATH.Contains('Insiders')) { - $env:VSCODE_EDITION = 'insiders' -} - -# Retrieve key bindings config file for Windows -$testDirectory = (Resolve-Path .\).Path -$client = New-Object System.Net.WebClient -$client.DownloadFile("https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings/doc.keybindings.win.json","$testDirectory\test_data\keybindings.win32.json") - -# Compile and launch the smoke test -tsc -npm test \ No newline at end of file diff --git a/test/smoke/scripts/run.sh b/test/smoke/scripts/run.sh deleted file mode 100644 index fc103a55539..00000000000 --- a/test/smoke/scripts/run.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -if [[ "$#" -ne 1 ]]; then - echo "Usage: ./scripts/run.sh path/to/binary" - echo "To perform data migration tests, use: ./scripts/run.sh path/to/latest_binary path/to/stable_binary" - exit 1 -fi - -# Cloning sample repository for the smoke test -cd .. -if ! [ -d vscode-smoketest-express ]; then - git clone https://github.com/Microsoft/vscode-smoketest-express.git - cd vscode-smoketest-express -else - cd vscode-smoketest-express - git fetch origin master - git reset --hard FETCH_HEAD - git clean -fd -fi -npm install - -# Install Node modules for Spectron -cd ../vscode-smoketest -test -d node_modules || npm install - -# Configuration -export VSCODE_LATEST_PATH="$1" -export VSCODE_STABLE_PATH="$2" -export SMOKETEST_REPO="../vscode-smoketest-express" -mkdir -p test_data - -if [[ $1 == *"Insiders"* || $1 == *"insiders"* ]]; then - export VSCODE_EDITION="insiders" -fi - -if [[ "$OSTYPE" == "darwin"* ]]; then - curl "https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings/doc.keybindings.osx.json" -o "test_data/keybindings.darwin.json" # Download OS X keybindings -else - wget https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings/doc.keybindings.linux.json -O test_data/keybindings.linux.json # Download Linux keybindings -fi - -# Compile and launch the smoke test -tsc -exec npm test diff --git a/test/smoke/src/areas/common.ts b/test/smoke/src/areas/common.ts index c955c206eea..33127905ec6 100644 --- a/test/smoke/src/areas/common.ts +++ b/test/smoke/src/areas/common.ts @@ -19,7 +19,7 @@ export class CommonActions { public async getWindowTitle(): Promise { return this.spectron.client.getTitle(); } - + public enter(): Promise { return this.spectron.client.keys(['Enter', 'NULL']); } @@ -34,7 +34,7 @@ export class CommonActions { await this.spectron.wait(); return this.saveOpenedFile(); } - + public async newUntitledFile(): Promise { await this.spectron.command('workbench.action.files.newUntitledFile'); return this.spectron.wait(); @@ -50,7 +50,7 @@ export class CommonActions { if (el.status === 0) { return el; } - + return undefined; } @@ -118,7 +118,7 @@ export class CommonActions { selector += ' explorer-item'; } selector += '"]'; - + await this.spectron.waitFor(this.spectron.client.doubleClick, selector); return this.spectron.wait(); } @@ -132,7 +132,7 @@ export class CommonActions { } else if (extension === 'md') { return 'md-ext-file-icon markdown-lang-file-icon'; } - + throw new Error('No class defined for this file extension'); } @@ -142,7 +142,7 @@ export class CommonActions { if (Array.isArray(span)) { return span[0]; } - + return span; } catch (e) { return undefined; diff --git a/test/smoke/src/areas/css.ts b/test/smoke/src/areas/css.ts index 3388ab4e465..d2cbd62b0a8 100644 --- a/test/smoke/src/areas/css.ts +++ b/test/smoke/src/areas/css.ts @@ -11,7 +11,7 @@ export enum CSSProblem { }; export class CSS { - + constructor(private spectron: SpectronApplication) { // noop } @@ -56,7 +56,7 @@ export class CSS { if (el.status === 0) { return el; } - + return undefined; } } \ No newline at end of file diff --git a/test/smoke/src/areas/data-loss.ts b/test/smoke/src/areas/data-loss.ts index dc1ecf93730..5988ce6f7f8 100644 --- a/test/smoke/src/areas/data-loss.ts +++ b/test/smoke/src/areas/data-loss.ts @@ -20,7 +20,7 @@ export class DataLoss { if (el.status === 0) { return el; } - + return undefined; } } \ No newline at end of file diff --git a/test/smoke/src/areas/extensions.ts b/test/smoke/src/areas/extensions.ts index 4edec9d06ef..bc4e2743e44 100644 --- a/test/smoke/src/areas/extensions.ts +++ b/test/smoke/src/areas/extensions.ts @@ -7,7 +7,7 @@ import { SpectronApplication } from '../spectron/application'; import { CommonActions } from "./common"; export class Extensions { - + private readonly extensionsViewletSelector = 'div[id="workbench.view.extensions"]'; constructor(private spectron: SpectronApplication, private common: CommonActions) { diff --git a/test/smoke/src/areas/first-experience.ts b/test/smoke/src/areas/first-experience.ts index e9141bda899..2d6e9c30eaa 100644 --- a/test/smoke/src/areas/first-experience.ts +++ b/test/smoke/src/areas/first-experience.ts @@ -15,7 +15,7 @@ export class FirstExperience { if (el.status === 0) { return el; } - + return undefined; } } \ No newline at end of file diff --git a/test/smoke/src/areas/integrated-terminal.ts b/test/smoke/src/areas/integrated-terminal.ts index b066db8adf7..c41f2946d4d 100644 --- a/test/smoke/src/areas/integrated-terminal.ts +++ b/test/smoke/src/areas/integrated-terminal.ts @@ -25,6 +25,7 @@ export class IntegratedTerminal { public async getCommandOutput(command: string): Promise { const selector = 'div[id="workbench.panel.terminal"] .xterm-rows'; + // Default Powershell terminal adds 3 header rows at the top, whereas bash does not. let readRow = process.platform === 'win32' ? 5 : 2; let output: string = await this.spectron.client.getText(`${selector}>:nth-child(${readRow})`); diff --git a/test/smoke/src/areas/javascript-debug.ts b/test/smoke/src/areas/javascript-debug.ts index 948594945d7..ff39a985d66 100644 --- a/test/smoke/src/areas/javascript-debug.ts +++ b/test/smoke/src/areas/javascript-debug.ts @@ -36,7 +36,7 @@ export class JavaScriptDebug { if (el.status === 0) { return el; } - + return undefined; } } \ No newline at end of file diff --git a/test/smoke/src/areas/statusbar.ts b/test/smoke/src/areas/statusbar.ts index 93b73495183..8e330c8e121 100644 --- a/test/smoke/src/areas/statusbar.ts +++ b/test/smoke/src/areas/statusbar.ts @@ -49,7 +49,7 @@ export class StatusBar { if (el.status === 0) { return el; } - + return undefined; } @@ -58,7 +58,7 @@ export class StatusBar { if (el.status === 0) { return el; } - + return undefined; } @@ -71,7 +71,7 @@ export class StatusBar { if (el.status === 0) { return el; } - + return undefined; } diff --git a/test/smoke/src/areas/tasks.ts b/test/smoke/src/areas/tasks.ts index e46c2961df5..d83cc173efa 100644 --- a/test/smoke/src/areas/tasks.ts +++ b/test/smoke/src/areas/tasks.ts @@ -25,6 +25,7 @@ export class Tasks { public async firstOutputLineEndsWith(fileName: string): Promise { const firstLine = await this.spectron.waitFor(this.spectron.client.getText, `${this.outputViewSelector}>:nth-child(2)`); + return firstLine.endsWith(fileName); } @@ -40,7 +41,7 @@ export class Tasks { return this.spectron.client.getValue(`${this.workbenchPanelSelector} .select-box`); } - public getProblemsViewFirstElementName(): Promise { + public getProblemsViewFirstElementName(): Promise { return this.spectron.waitFor(this.spectron.client.getText, `${this.problemsViewSelector} .label-name`); } diff --git a/test/smoke/src/main.js b/test/smoke/src/main.js new file mode 100644 index 00000000000..8df41b5c73c --- /dev/null +++ b/test/smoke/src/main.js @@ -0,0 +1,163 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +var fs = require('fs'); +var https = require('https'); +var program = require('commander'); +var git = require('simple-git')(); +var child_process = require('child_process'); +var path = require('path'); + +var tempFolder = `test_data`; +var testRepoUrl = 'https://github.com/Microsoft/vscode-smoketest-express'; +var testRepoLocalDir = path.join(process.cwd(), `${tempFolder}/vscode-smoketest-express`); +var keybindingsUrl = 'https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings'; + +program + .option('-l, --latest ', 'path to the latest VS Code to test') + .option('-s, --stable [file path]', 'path to the stable VS Code to be used in data migration tests'); + +program.on('--help', () => { + console.log(' Examples:'); + console.log(''); + console.log(' $ npm test -- --latest path/to/binary'); + console.log(' $ npm test -- -l path/to/binary'); + console.log(''); + console.log(' $ npm test -- --latest path/to/latest/binary --stable path/to/stable/binary'); + console.log(' $ npm test -- -l path/to/latest/binary -s path/to/stable/binary'); + console.log(''); +}); +program.parse(process.argv); + +if (!program.latest) { + console.error('You must specify the binary to run the smoke test against'); + process.exit(1); +} +if (!binaryExists(program.latest) || (program.stable && !binaryExists(program.stable))) { + console.error('The file path to electron binary does not exist or permissions do not allow to execute it. Please check the path provided.'); + process.exit(1); +} + +// Setting up environment variables +process.env['VSCODE_LATEST_PATH'] = program.latest; +if (program.stable) process.env['VSCODE_STABLE_PATH'] = program.stable; +process.env['SMOKETEST_REPO'] = testRepoLocalDir; +if (program.stable && program.stable.toLowerCase().startsWith('insiders')) process.env['VSCODE_EDITION'] = 'insiders'; + +// Setting up 'vscode-smoketest-express' project +var os = process.platform; +if (os === 'darwin') os = 'osx'; +else if (os === 'win32') os = 'win'; +var promises = []; + +try { + // promises.push(execute('npm install'), process.cwd()); + promises.push(getKeybindings(`${keybindingsUrl}/doc.keybindings.${os}.json`, `${tempFolder}/keybindings.json`)); + promises.push(cleanOrClone(testRepoUrl, testRepoLocalDir)); + + Promise.all(promises).then(() => { execute('npm install', testRepoLocalDir).then(() => runTests()); }); +} catch (e) { + throw new Error('Error caught running the smoke test: ' + e); +} + + +function runTests() { + console.log('Running tests...') + const spawn = require('child_process').spawn; + var proc = spawn(process.execPath, [ + 'src/mocha-runner.js' + ]); + proc.stdout.on('data', data => { + console.log(data.toString()); + }); + proc.stderr.on('data', data => { + var date = new Date().toLocaleString(); + fs.appendFile(`${tempFolder}/errors.log`, `${date}: ${data.toString()}`, (err) => { + if (err) throw new Error(`Could not write stderr to errors.log with the following error: ${err}`); + }); + }); +} + +function cleanOrClone(repo, dir) { + console.log('Cleaning or cloning test project repository...'); + return new Promise((res, rej) => { + if (!folderExists(dir)) { + git.clone(repo, dir, () => { + console.log('Test repository successfully cloned.'); + res(); + }); + } else { + git.cwd(dir); + git.fetch((err) => { + if (err) rej(err); + resetAndClean(); + }); + } + + var resetAndClean = () => { + git.reset(['FETCH_HEAD', '--hard'], (err) => { + if (err) rej(err); + + git.clean('f', ['-d'], (err) => { + if (err) rej(err); + console.log('Test project was successfully reset to initial state.'); + res(); + }); + }); + } + }); +} + +function execute(cmd, dir) { + return new Promise((res, rej) => { + console.log(`Running ${cmd}...`); + var output = child_process.exec(cmd, { cwd: dir, stdio: [0, 1, 2] }, (error, stdout, stderr) => { + if (error) rej(error); + if (stderr) console.error(stderr); + console.log(stdout); + res(); + }); + }); +} + +function getKeybindings(url, location) { + console.log(`Fetching keybindings from ${url}...`); + return new Promise((resolve, reject) => { + https.get(url, (res) => { + if (res.statusCode != 200) { + reject(`Failed to obtain key bindings with response code: ${res.statusCode}`); + } + + var buffer = []; + res.on('data', (chunk) => buffer.push(chunk)); + res.on('end', () => { + fs.writeFile(location, Buffer.concat(buffer), 'utf8', () => { + console.log('Keybindings were successfully fetched.'); + resolve(); + }); + }); + }).on('error', (e) => { + reject(`Failed to obtain key bindings with an error: ${e}`); + }); + }); +} + +function folderExists(folder) { + try { + fs.accessSync(folder, 'rw'); + return true; + } catch (e) { + return false; + } +} + +function binaryExists(filePath) { + try { + fs.accessSync(filePath, 'x'); + return true; + } catch (e) { + return false; + } +} \ No newline at end of file diff --git a/test/smoke/src/mocha-runner.js b/test/smoke/src/mocha-runner.js new file mode 100644 index 00000000000..2a08adca647 --- /dev/null +++ b/test/smoke/src/mocha-runner.js @@ -0,0 +1,21 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +var Mocha = require('mocha'); +var path = require('path'); + +var mocha = new Mocha({ + timeout: 360000, + retries: 2, + slow: 50000, + useColors: true +}); + +mocha.addFile(path.join(process.cwd(), 'out/tests.js')); +mocha.run((failures) => { + process.on('exit', () => { + process.exit(failures); + }); +}); \ No newline at end of file diff --git a/test/smoke/src/spectron/application.ts b/test/smoke/src/spectron/application.ts index 5e45474768c..d1027a1ef70 100644 --- a/test/smoke/src/spectron/application.ts +++ b/test/smoke/src/spectron/application.ts @@ -7,6 +7,7 @@ import { Application } from 'spectron'; import { SpectronClient } from './client'; import { Screenshot } from "../helpers/screenshot"; var fs = require('fs'); +var path = require('path'); /** * Wraps Spectron's Application instance with its used methods. @@ -16,7 +17,7 @@ export class SpectronApplication { private spectron: Application; private readonly pollTrials = 5; - private readonly pollTimeout = 3; // in secs + private readonly pollTimeout = 3; // in secs private keybindings: any[]; private screenshot: Screenshot; @@ -73,13 +74,15 @@ export class SpectronApplication { } private retrieveKeybindings() { - const os = process.platform; - fs.readFile(`test_data/keybindings.${os}.json`, (err, data) => { + fs.readFile(path.join(process.cwd(), `test_data/keybindings.json`), 'utf8', (err, data) => { if (err) { throw err; } - - this.keybindings = JSON.parse(data); + try { + this.keybindings = JSON.parse(data); + } catch (e) { + throw new Error(`Error parsing keybindings JSON: ${e}`); + } }); } diff --git a/test/smoke/src/spectron/client.ts b/test/smoke/src/spectron/client.ts index 9376bd3e100..6a9003199dc 100644 --- a/test/smoke/src/spectron/client.ts +++ b/test/smoke/src/spectron/client.ts @@ -7,7 +7,7 @@ import { Application } from 'spectron'; import { Screenshot } from '../helpers/screenshot'; /** - * Abstracts the Spectron's WebdriverIO managed client property on the created Application instances. + * Abstracts the Spectron's WebdriverIO managed client property on the created Application instances. */ export class SpectronClient { diff --git a/test/smoke/src/main.ts b/test/smoke/src/tests.ts similarity index 93% rename from test/smoke/src/main.ts rename to test/smoke/src/tests.ts index 9da55ce546a..35d6d1f79ac 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/tests.ts @@ -6,7 +6,7 @@ import * as assert from 'assert'; import { SpectronApplication } from "./spectron/application"; import { CommonActions } from './areas/common'; -import { FirstExperience } from './areas/first-experience'; +// import { FirstExperience } from './areas/first-experience'; import { ConfigurationView, ActivityBarPosition } from './areas/configuration-views'; import { Search } from './areas/search'; import { CSS, CSSProblem } from './areas/css'; @@ -23,14 +23,13 @@ import { Localization, ViewletType } from "./areas/localization"; describe('Smoke Test Suite', function () { const latestPath = process.env.VSCODE_LATEST_PATH; const stablePath = process.env.VSCODE_STABLE_PATH; - const insiders = process.env.VSCODE_EDITION; + // const insiders = process.env.VSCODE_EDITION; const workspacePath = process.env.SMOKETEST_REPO; const tempUserDir = 'test_data/temp_user_dir'; const tempExtensionsDir = 'test_data/temp_extensions_dir'; let app: SpectronApplication; let common: CommonActions; - this.retries(2); if (stablePath) { context('Data Migration', function () { @@ -176,37 +175,38 @@ describe('Smoke Test Suite', function () { }); }); - context('First User Experience', function () { - let experience: FirstExperience; + // Do not run until experiments are finished over the first-time startup behaviour. + // context('First User Experience', function () { + // let experience: FirstExperience; - beforeEach(async function () { - app = new SpectronApplication(latestPath, this.currentTest.fullTitle(), (this.currentTest as any).currentRetry(), undefined, [`--user-data-dir=${tempUserDir}`, `--excludeSwitches=load-component-extension`]); - common = new CommonActions(app); - experience = new FirstExperience(app); + // beforeEach(async function () { + // app = new SpectronApplication(latestPath, this.currentTest.fullTitle(), (this.currentTest as any).currentRetry(), undefined, [`--user-data-dir=${tempUserDir}`, `--excludeSwitches=load-component-extension`]); + // common = new CommonActions(app); + // experience = new FirstExperience(app); + + // await common.removeDirectory(tempUserDir); + // return await app.start(); + // }); + // afterEach(async function () { + // return await app.stop(); + // }); - await common.removeDirectory(tempUserDir); - return await app.start(); - }); - afterEach(async function () { - return await app.stop(); - }); + // it(`verifies if title is set correctly on the clean user-directory startup`, async function () { + // const title = await common.getWindowTitle(); + + // let expectedTitle = 'Welcome'; + // if (process.platform !== 'darwin') { + // expectedTitle += ' — Visual Studio Code'; + // if (insiders) expectedTitle += ' - Insiders'; + // } - it(`verifies if title is set correctly on the clean user-directory startup`, async function () { - const title = await common.getWindowTitle(); + // assert.equal(title, expectedTitle); + // }); - let expectedTitle = 'Welcome'; - if (process.platform !== 'darwin') { - expectedTitle += ' — Visual Studio Code'; - if (insiders) expectedTitle += ' - Insiders'; - } - - assert.equal(title, expectedTitle); - }); - - it(`verifies if 'Welcome page' tab is presented on the clean user-directory startup`, async function () { - assert.ok(await experience.getWelcomeTab()); - }); - }); + // it(`verifies if 'Welcome page' tab is presented on the clean user-directory startup`, async function () { + // assert.ok(await experience.getWelcomeTab()); + // }); + // }); context('Explorer', function () { beforeEach(async function () { @@ -557,7 +557,6 @@ describe('Smoke Test Suite', function () { await common.type(command); await common.enter(); await app.wait(); - // Default Powershell terminal adds 3 header rows at the top, whereas bash does not. let output = await terminal.getCommandOutput(command); assert.equal(output, 'test'); }); @@ -660,7 +659,7 @@ describe('Smoke Test Suite', function () { const res = await tasks.getOutputResult(); assert.equal(res, '✖ 6 problems (6 errors, 0 warnings)'); }); - + it(`is able to select 'Git' output`, async function () { await tasks.build(); await app.wait(); @@ -674,7 +673,7 @@ describe('Smoke Test Suite', function () { assert.ok(await tasks.firstOutputLineEndsWith('index.js')); }); - it(`verifies build errors are reflected in 'Problems View'`, async function () { + it(`verifies build errors are reflected in 'Problems View'`, async function () { await tasks.build(); await app.wait(); await tasks.openProblemsView(); @@ -717,9 +716,9 @@ describe('Smoke Test Suite', function () { await extensions.installFirstResult(); await app.wait(); await extensions.getFirstReloadText(); - + await app.stop(); - await app.wait(); // wait until all resources are released (e.g. locked local storage) + await app.wait(); // wait until all resources are released (e.g. locked local storage) await app.start(); await extensions.selectMinimalIconsTheme(); const x = await extensions.verifyFolderIconAppearance(); @@ -739,13 +738,14 @@ describe('Smoke Test Suite', function () { common.removeDirectory(tempUserDir); await app.start(); - - let expectedTitle = 'Willkommen — vscode-smoketest-express'; - if (process.platform !== 'darwin') { - expectedTitle += ' — Visual Studio Code'; - if (insiders) expectedTitle += ' - Insiders'; - } - assert.equal(await common.getWindowTitle(), expectedTitle); + + // Do not run until experiments are finished over the first-time startup behaviour. + // let expectedTitle = 'Willkommen — vscode-smoketest-express'; + // if (process.platform !== 'darwin') { + // expectedTitle += ' — Visual Studio Code'; + // if (insiders) expectedTitle += ' - Insiders'; + // } + // assert.equal(await common.getWindowTitle(), expectedTitle); let text = await locale.getOpenEditorsText(); assert.equal(text.toLowerCase(), 'geöffnete editoren'); @@ -764,8 +764,8 @@ describe('Smoke Test Suite', function () { await locale.openViewlet(ViewletType.EXTENSIONS); text = await locale.getExtensionsSearchPlaceholder(); - assert.equal(text.toLowerCase(), 'nach extensions in marketplace suchen'); + assert.equal(text.toLowerCase(), 'nach erweiterungen im marketplace suchen'); }); }); - + }); \ No newline at end of file diff --git a/test/smoke/tsconfig.json b/test/smoke/tsconfig.json index 1b0b03c2d67..733c1107e83 100644 --- a/test/smoke/tsconfig.json +++ b/test/smoke/tsconfig.json @@ -18,4 +18,4 @@ "exclude": [ "node_modules" ] -} \ No newline at end of file +} diff --git a/vscode-smoketest-express b/vscode-smoketest-express deleted file mode 160000 index 636dd7a2ff7..00000000000 --- a/vscode-smoketest-express +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 636dd7a2ff71d266c0e015411b47cf5c3a25be5a