Updated the way to run the smoke test.

test/smoke/.vscode/launch.json

@@ -15,7 +15,7 @@
                 "--timeout",
                 "999999",
                 "--colors",
-                "${workspaceRoot}/out/main.js"
+                "${workspaceRoot}/out/tests.js"
             ],
 			"outFiles": [ "${workspaceRoot}/out/**/*.js" ],
             "internalConsoleOptions": "openOnSessionStart",

test/smoke/CONTRIBUTING.md

@@ -1,5 +1,7 @@
 # Architecture
-* `main.ts` contains the main smoke test suite. It includes all tests separated into mocha `describe()` groups that represent each of the areas of [Smoke Test document](https://github.com/Microsoft/vscode/wiki/Smoke-Test).
+* `main.js` is used to prepare all smoke test dependencies (fetching key bindings and 'Express' repository, running `npm install` there).
+* `mocha-runner.js` launches Mocha programmatically. It is spawned in Node environment from main.js to ensure that it is possible to listen on `stderr`s (primary `process.stderr` is not readable otherwise). This is accomplished because WebDriverIO command deprecation warnings need to be redirected to a separate log. Those warnings are coming from WebDriverIO because ChromeDriver has not migrated from JsonWire to W3C WebDriver protocol.
+* `tests.ts` contains the main smoke test suite. It includes all tests separated into mocha `describe()` groups that represent each of the areas of [Smoke Test document](https://github.com/Microsoft/vscode/wiki/Smoke-Test).
 
 * `./areas/` folder contains a `.ts` file per each area of the document. E.g. `'Search'` area goes under `'search.ts'`. Every area file contains a list of methods with the name that represents the action that can be performed in the corresponding test. This reduces the amount of test suite code and means that if the UI changes, the fix need only be applied in one place. The name of the method reflects the action the tester would do if he would perform the test manually. See [Selenium Page Objects Wiki](https://github.com/SeleniumHQ/selenium/wiki/PageObjects) and [Selenium Bot Style Tests Wiki](https://github.com/SeleniumHQ/selenium/wiki/Bot-Style-Tests) for a good explanation of the implementation. Every smoke test area contains methods that are used in a bot-style approach in `main.ts`.
 * `./spectron/` wraps the Spectron, with WebDriverIO API wrapped in `client.ts` and instance of Spectron Application is wrapped in `application.ts`.

test/smoke/README.md

@@ -1,9 +1,9 @@
 # VS Code Smoke Testing
 This repository contains the smoke test automation code with Spectron for Visual Studio Code.
 
-The following command is used to run the tests: `.\scripts\run.ps1 -latest "path\to\Code.exe"` on Windows (from PowerShell) and `./scripts/run.sh path/to/binary` on Unix system.
+The following command is used to run the tests: `npm test -- --latest "path/to/binary"`.
 
-If you want to include 'Data Migration' area tests use  `.\scripts\run.ps1 -latest "path\to\Code.exe" -stable "path\to\CurrentStable.exe"` and `./scripts/run.sh path/to/binary path/to/currentStable` respectively.
+If you want to include 'Data Migration' area tests use  `npm test -- --latest path/to/binary --stable path/to/currentStable` respectively.
 
 # Contributing
 

test/smoke/package.json

@@ -1,9 +1,10 @@
 {
   "name": "code-oss-dev-smoke-test",
   "version": "0.1.0",
-  "main": "./out/main.js",
+  "main": "./src/main.js",
   "scripts": {
-    "test": "mocha -u tdd --timeout 360000 --retries 2 --slow 50000 --colors ./out/main.js 2> test_data/errors.log"
+    "pretest": "tsc",
+    "test": "node src/main.js"
   },
   "devDependencies": {
     "@types/mocha": "^2.2.41",
@@ -14,6 +15,8 @@
     "mocha": "^3.2.0",
     "spectron": "^3.6.4",
     "typescript": "^2.2.2",
-    "rimraf": "^2.6.1"
+    "rimraf": "^2.6.1",
+    "commander": "^2.9.0",
+    "simple-git": "^1.73.0"
   }
 }

test/smoke/scripts/run.ps1

@@ -1,44 +0,0 @@
-Param(
-  [Parameter(Position=0,mandatory=$true)]
-  [string]$arch
-)
-
-
-# Setup sample repository for the smoke test
-Set-Location ..
-if (-Not (Test-Path vscode-smoketest-express)) {
-  git clone https://github.com/Microsoft/vscode-smoketest-express.git
-  Set-Location ./vscode-smoketest-express
-} else {
-  Set-Location ./vscode-smoketest-express
-  git fetch origin master
-  git reset --hard FETCH_HEAD
-  git clean -fd
-}
-npm install
-
-Write-Output "My path: " + $(pwd)
-
-# Setup the test directory for running
-Set-Location ..\smoke
-if (-Not (Test-Path node_modules)) {
-  npm install
-}
-
-# Configure environment variables
-$env:VSCODE_LATEST_PATH = "$(pwd)\..\VSCode-win32-$arch\Code - Insiders.exe"
-# $env:VSCODE_STABLE_PATH = $stable
-$env:SMOKETEST_REPO = "..\vscode-smoketest-express"
-
-if ($env:VSCODE_LATEST_PATH.Contains('Insiders')) {
-  $env:VSCODE_EDITION = 'insiders'
-}
-
-# Retrieve key bindings config file for Windows
-$testDirectory = (Resolve-Path .\).Path
-$client = New-Object System.Net.WebClient
-$client.DownloadFile("https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings/doc.keybindings.win.json","$testDirectory\test_data\keybindings.win32.json")
-
-# Compile and launch the smoke test
-tsc
-npm test

test/smoke/scripts/run.sh

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-if [[ "$#" -ne 1 ]]; then
-	echo "Usage: ./scripts/run.sh path/to/binary"
-	echo "To perform data migration tests, use: ./scripts/run.sh path/to/latest_binary path/to/stable_binary"
-	exit 1
-fi
-
-# Cloning sample repository for the smoke test
-cd ..
-if ! [ -d vscode-smoketest-express ]; then
-	git clone https://github.com/Microsoft/vscode-smoketest-express.git
-	cd vscode-smoketest-express
-else
-	cd vscode-smoketest-express
-	git fetch origin master
-	git reset --hard FETCH_HEAD
-	git clean -fd
-fi
-npm install
-
-# Install Node modules for Spectron
-cd ../vscode-smoketest
-test -d node_modules || npm install
-
-# Configuration
-export VSCODE_LATEST_PATH="$1"
-export VSCODE_STABLE_PATH="$2"
-export SMOKETEST_REPO="../vscode-smoketest-express"
-mkdir -p test_data
-
-if [[ $1 == *"Insiders"* || $1 == *"insiders"* ]]; then
-	export VSCODE_EDITION="insiders"
-fi
-
-if [[ "$OSTYPE" == "darwin"* ]]; then
-	curl "https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings/doc.keybindings.osx.json" -o "test_data/keybindings.darwin.json" # Download OS X keybindings
-else
-	wget https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings/doc.keybindings.linux.json -O test_data/keybindings.linux.json # Download Linux keybindings
-fi
-
-# Compile and launch the smoke test
-tsc
-exec npm test

test/smoke/src/areas/common.ts

@@ -19,7 +19,7 @@ export class CommonActions {
 	public async getWindowTitle(): Promise<any> {
 		return this.spectron.client.getTitle();
 	}
-
+	
 	public enter(): Promise<any> {
 		return this.spectron.client.keys(['Enter', 'NULL']);
 	}
@@ -34,7 +34,7 @@ export class CommonActions {
 		await this.spectron.wait();
 		return this.saveOpenedFile();
 	}
-
+	
 	public async newUntitledFile(): Promise<any> {
 		await this.spectron.command('workbench.action.files.newUntitledFile');
 		return this.spectron.wait();
@@ -50,7 +50,7 @@ export class CommonActions {
 		if (el.status === 0) {
 			return el;
 		}
-
+		
 		return undefined;
 	}
 
@@ -118,7 +118,7 @@ export class CommonActions {
 			selector += ' explorer-item';
 		}
 		selector += '"]';
-
+		
 		await this.spectron.waitFor(this.spectron.client.doubleClick, selector);
 		return this.spectron.wait();
 	}
@@ -132,7 +132,7 @@ export class CommonActions {
 		} else if (extension === 'md') {
 			return 'md-ext-file-icon markdown-lang-file-icon';
 		}
-
+		
 		throw new Error('No class defined for this file extension');
 	}
 
@@ -142,7 +142,7 @@ export class CommonActions {
 			if (Array.isArray(span)) {
 				return span[0];
 			}
-
+			
 			return span;
 		} catch (e) {
 			return undefined;

test/smoke/src/areas/css.ts

@@ -11,7 +11,7 @@ export enum CSSProblem {
 };
 
 export class CSS {
-
+	
 	constructor(private spectron: SpectronApplication) {
 		// noop
 	}
@@ -56,7 +56,7 @@ export class CSS {
 		if (el.status === 0) {
 			return el;
 		}
-
+		
 		return undefined;
 	}
 }

test/smoke/src/areas/data-loss.ts

@@ -20,7 +20,7 @@ export class DataLoss {
 		if (el.status === 0) {
 			return el;
 		}
-
+		
 		return undefined;
 	}
 }

test/smoke/src/areas/extensions.ts

@@ -7,7 +7,7 @@ import { SpectronApplication } from '../spectron/application';
 import { CommonActions } from "./common";
 
 export class Extensions {
-
+	
 	private readonly extensionsViewletSelector = 'div[id="workbench.view.extensions"]';
 
 	constructor(private spectron: SpectronApplication, private common: CommonActions) {

test/smoke/src/areas/first-experience.ts

@@ -15,7 +15,7 @@ export class FirstExperience {
 		if (el.status === 0) {
 			return el;
 		}
-
+		
 		return undefined;
 	}
 }

test/smoke/src/areas/integrated-terminal.ts

@@ -25,6 +25,7 @@ export class IntegratedTerminal {
 
 	public async getCommandOutput(command: string): Promise<string> {
 		const selector = 'div[id="workbench.panel.terminal"] .xterm-rows';
+		// Default Powershell terminal adds 3 header rows at the top, whereas bash does not.
 		let readRow = process.platform === 'win32' ? 5 : 2;
 		let output: string = await this.spectron.client.getText(`${selector}>:nth-child(${readRow})`);
 

test/smoke/src/areas/javascript-debug.ts

@@ -36,7 +36,7 @@ export class JavaScriptDebug {
 		if (el.status === 0) {
 			return el;
 		}
-
+		
 		return undefined;
 	}
 }

test/smoke/src/areas/statusbar.ts

@@ -49,7 +49,7 @@ export class StatusBar {
 		if (el.status === 0) {
 			return el;
 		}
-
+		
 		return undefined;
 	}
 
@@ -58,7 +58,7 @@ export class StatusBar {
 		if (el.status === 0) {
 			return el;
 		}
-
+		
 		return undefined;
 	}
 
@@ -71,7 +71,7 @@ export class StatusBar {
 		if (el.status === 0) {
 			return el;
 		}
-
+		
 		return undefined;
 	}
 

test/smoke/src/areas/tasks.ts

@@ -25,6 +25,7 @@ export class Tasks {
 
 	public async firstOutputLineEndsWith(fileName: string): Promise<boolean> {
 		const firstLine = await this.spectron.waitFor(this.spectron.client.getText, `${this.outputViewSelector}>:nth-child(2)`);
+		
 		return firstLine.endsWith(fileName);
 	}
 
@@ -40,7 +41,7 @@ export class Tasks {
 		return this.spectron.client.getValue(`${this.workbenchPanelSelector} .select-box`);
 	}
 
-	public getProblemsViewFirstElementName(): Promise<any> {
+	public getProblemsViewFirstElementName(): Promise<any> {        
 		return this.spectron.waitFor(this.spectron.client.getText, `${this.problemsViewSelector} .label-name`);
 	}
 

test/smoke/src/main.js

@@ -0,0 +1,163 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+var fs = require('fs');
+var https = require('https');
+var program = require('commander');
+var git = require('simple-git')();
+var child_process = require('child_process');
+var path = require('path');
+
+var tempFolder = `test_data`;
+var testRepoUrl = 'https://github.com/Microsoft/vscode-smoketest-express';
+var testRepoLocalDir = path.join(process.cwd(), `${tempFolder}/vscode-smoketest-express`);
+var keybindingsUrl = 'https://raw.githubusercontent.com/Microsoft/vscode-docs/master/scripts/keybindings';
+
+program
+	.option('-l, --latest <file path>', 'path to the latest VS Code to test')
+	.option('-s, --stable [file path]', 'path to the stable VS Code to be used in data migration tests');
+
+program.on('--help', () => {
+	console.log('  Examples:');
+	console.log('');
+	console.log('    $ npm test -- --latest path/to/binary');
+	console.log('    $ npm test -- -l path/to/binary');
+	console.log('');
+	console.log('    $ npm test -- --latest path/to/latest/binary --stable path/to/stable/binary');
+	console.log('    $ npm test -- -l path/to/latest/binary -s path/to/stable/binary');
+	console.log('');
+});
+program.parse(process.argv);
+
+if (!program.latest) {
+	console.error('You must specify the binary to run the smoke test against');
+	process.exit(1);
+}
+if (!binaryExists(program.latest) || (program.stable && !binaryExists(program.stable))) {
+	console.error('The file path to electron binary does not exist or permissions do not allow to execute it. Please check the path provided.');
+	process.exit(1);
+}
+
+// Setting up environment variables
+process.env['VSCODE_LATEST_PATH'] = program.latest;
+if (program.stable) process.env['VSCODE_STABLE_PATH'] = program.stable;
+process.env['SMOKETEST_REPO'] = testRepoLocalDir;
+if (program.stable && program.stable.toLowerCase().startsWith('insiders')) process.env['VSCODE_EDITION'] = 'insiders';
+
+// Setting up 'vscode-smoketest-express' project
+var os = process.platform;
+if (os === 'darwin') os = 'osx';
+else if (os === 'win32') os = 'win';
+var promises = [];
+
+try {
+	// promises.push(execute('npm install'), process.cwd());
+	promises.push(getKeybindings(`${keybindingsUrl}/doc.keybindings.${os}.json`, `${tempFolder}/keybindings.json`));
+	promises.push(cleanOrClone(testRepoUrl, testRepoLocalDir));
+
+	Promise.all(promises).then(() => { execute('npm install', testRepoLocalDir).then(() => runTests()); });
+} catch (e) {
+	throw new Error('Error caught running the smoke test: ' + e);
+}
+
+
+function runTests() {
+	console.log('Running tests...')
+	const spawn = require('child_process').spawn;
+	var proc = spawn(process.execPath, [
+		'src/mocha-runner.js'
+	]);
+	proc.stdout.on('data', data => {
+		console.log(data.toString());
+	});
+	proc.stderr.on('data', data => {
+		var date = new Date().toLocaleString();
+		fs.appendFile(`${tempFolder}/errors.log`, `${date}: ${data.toString()}`, (err) => {
+			if (err) throw new Error(`Could not write stderr to errors.log with the following error: ${err}`);
+		});
+	});
+}
+
+function cleanOrClone(repo, dir) {
+	console.log('Cleaning or cloning test project repository...');
+	return new Promise((res, rej) => {
+		if (!folderExists(dir)) {
+			git.clone(repo, dir, () => {
+				console.log('Test repository successfully cloned.');
+				res();
+			});
+		} else {
+			git.cwd(dir);
+			git.fetch((err) => {
+				if (err) rej(err);
+				resetAndClean();
+			});
+		}
+
+		var resetAndClean = () => {
+			git.reset(['FETCH_HEAD', '--hard'], (err) => {
+				if (err) rej(err);
+
+				git.clean('f', ['-d'], (err) => {
+					if (err) rej(err);
+					console.log('Test project was successfully reset to initial state.');
+					res();
+				});
+			});
+		}
+	});
+}
+
+function execute(cmd, dir) {
+	return new Promise((res, rej) => {
+		console.log(`Running ${cmd}...`);
+		var output = child_process.exec(cmd, { cwd: dir, stdio: [0, 1, 2] }, (error, stdout, stderr) => {
+			if (error) rej(error);
+			if (stderr) console.error(stderr);
+			console.log(stdout);
+			res();
+		});
+	});
+}
+
+function getKeybindings(url, location) {
+	console.log(`Fetching keybindings from ${url}...`);
+	return new Promise((resolve, reject) => {
+		https.get(url, (res) => {
+			if (res.statusCode != 200) {
+				reject(`Failed to obtain key bindings with response code: ${res.statusCode}`);
+			}
+
+			var buffer = [];
+			res.on('data', (chunk) => buffer.push(chunk));
+			res.on('end', () => {
+				fs.writeFile(location, Buffer.concat(buffer), 'utf8', () => {
+					console.log('Keybindings were successfully fetched.');
+					resolve();
+				});
+			});
+		}).on('error', (e) => {
+			reject(`Failed to obtain key bindings with an error: ${e}`);
+		});
+	});
+}
+
+function folderExists(folder) {
+	try {
+		fs.accessSync(folder, 'rw');
+		return true;
+	} catch (e) {
+		return false;
+	}
+}
+
+function binaryExists(filePath) {
+	try {
+		fs.accessSync(filePath, 'x');
+		return true;
+	} catch (e) {
+		return false;
+	}
+}

test/smoke/src/mocha-runner.js

@@ -0,0 +1,21 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+var Mocha = require('mocha');
+var path = require('path');
+
+var mocha = new Mocha({
+	timeout: 360000,
+	retries: 2,
+	slow: 50000,
+	useColors: true
+});
+
+mocha.addFile(path.join(process.cwd(), 'out/tests.js'));
+mocha.run((failures) => {
+	process.on('exit', () => {
+		process.exit(failures);
+	});
+});

test/smoke/src/spectron/application.ts

@@ -7,6 +7,7 @@ import { Application } from 'spectron';
 import { SpectronClient } from './client';
 import { Screenshot } from "../helpers/screenshot";
 var fs = require('fs');
+var path = require('path');
 
 /**
  * Wraps Spectron's Application instance with its used methods.
@@ -16,7 +17,7 @@ export class SpectronApplication {
 
 	private spectron: Application;
 	private readonly pollTrials = 5;
-	private readonly pollTimeout = 3; // in secs
+	private readonly pollTimeout = 3; // in secs 
 	private keybindings: any[];
 	private screenshot: Screenshot;
 
@@ -73,13 +74,15 @@ export class SpectronApplication {
 	}
 
 	private retrieveKeybindings() {
-		const os = process.platform;
-		fs.readFile(`test_data/keybindings.${os}.json`, (err, data) => {
+		fs.readFile(path.join(process.cwd(), `test_data/keybindings.json`), 'utf8', (err, data) => {
 			if (err) {
 				throw err;
 			}
-
-			this.keybindings = JSON.parse(data);
+			try {
+				this.keybindings = JSON.parse(data);
+			} catch (e) {
+				throw new Error(`Error parsing keybindings JSON: ${e}`);
+			}
 		});
 	}
 

test/smoke/src/spectron/client.ts

@@ -7,7 +7,7 @@ import { Application } from 'spectron';
 import { Screenshot } from '../helpers/screenshot';
 
 /**
- * Abstracts the Spectron's WebdriverIO managed client property on the created Application instances.
+ * Abstracts the Spectron's WebdriverIO managed client property on the created Application instances. 
  */
 export class SpectronClient {
 

test/smoke/src/tests.ts

@@ -6,7 +6,7 @@
 import * as assert from 'assert';
 import { SpectronApplication } from "./spectron/application";
 import { CommonActions } from './areas/common';
-import { FirstExperience } from './areas/first-experience';
+// import { FirstExperience } from './areas/first-experience';
 import { ConfigurationView, ActivityBarPosition } from './areas/configuration-views';
 import { Search } from './areas/search';
 import { CSS, CSSProblem } from './areas/css';
@@ -23,14 +23,13 @@ import { Localization, ViewletType } from "./areas/localization";
 describe('Smoke Test Suite', function () {
 	const latestPath = process.env.VSCODE_LATEST_PATH;
 	const stablePath = process.env.VSCODE_STABLE_PATH;
-	const insiders = process.env.VSCODE_EDITION;
+	// const insiders = process.env.VSCODE_EDITION;
 	const workspacePath = process.env.SMOKETEST_REPO;
 	const tempUserDir = 'test_data/temp_user_dir';
 	const tempExtensionsDir = 'test_data/temp_extensions_dir';
 
 	let app: SpectronApplication;
 	let common: CommonActions;
-	this.retries(2);
 
 	if (stablePath) {
 		context('Data Migration', function () {
@@ -176,37 +175,38 @@ describe('Smoke Test Suite', function () {
 		});
 	});
 
-	context('First User Experience', function () {
-		let experience: FirstExperience;
+	// Do not run until experiments are finished over the first-time startup behaviour.
+	// context('First User Experience', function () {
+	// 	let experience: FirstExperience;
 
-		beforeEach(async function () {
-			app = new SpectronApplication(latestPath, this.currentTest.fullTitle(), (this.currentTest as any).currentRetry(), undefined, [`--user-data-dir=${tempUserDir}`, `--excludeSwitches=load-component-extension`]);
-			common = new CommonActions(app);
-			experience = new FirstExperience(app);
+	// 	beforeEach(async function () {
+	// 		app = new SpectronApplication(latestPath, this.currentTest.fullTitle(), (this.currentTest as any).currentRetry(), undefined, [`--user-data-dir=${tempUserDir}`, `--excludeSwitches=load-component-extension`]);
+	// 		common = new CommonActions(app);
+	// 		experience = new FirstExperience(app);
+			
+	// 		await common.removeDirectory(tempUserDir);
+	// 		return await app.start();
+	// 	});
+	// 	afterEach(async function () {
+	// 		return await app.stop();
+	// 	});
 
-			await common.removeDirectory(tempUserDir);
-			return await app.start();
-		});
-		afterEach(async function () {
-			return await app.stop();
-		});
+	// 	it(`verifies if title is set correctly on the clean user-directory startup`, async function () {
+	// 		const title = await common.getWindowTitle();
+			
+	// 		let expectedTitle = 'Welcome';
+	// 		if (process.platform !== 'darwin') {
+	// 			expectedTitle += ' — Visual Studio Code';
+	// 			if (insiders) expectedTitle += ' - Insiders';
+	// 		}
 
-		it(`verifies if title is set correctly on the clean user-directory startup`, async function () {
-			const title = await common.getWindowTitle();
+	// 		assert.equal(title, expectedTitle);
+	// 	});
 
-			let expectedTitle = 'Welcome';
-			if (process.platform !== 'darwin') {
-				expectedTitle += ' — Visual Studio Code';
-				if (insiders) expectedTitle += ' - Insiders';
-			}
-
-			assert.equal(title, expectedTitle);
-		});
-
-		it(`verifies if 'Welcome page' tab is presented on the clean user-directory startup`, async function () {
-			assert.ok(await experience.getWelcomeTab());
-		});
-	});
+	// 	it(`verifies if 'Welcome page' tab is presented on the clean user-directory startup`, async function () {
+	// 		assert.ok(await experience.getWelcomeTab());
+	// 	});
+	// });
 
 	context('Explorer', function () {
 		beforeEach(async function () {
@@ -557,7 +557,6 @@ describe('Smoke Test Suite', function () {
 			await common.type(command);
 			await common.enter();
 			await app.wait();
-			// Default Powershell terminal adds 3 header rows at the top, whereas bash does not.
 			let output = await terminal.getCommandOutput(command);
 			assert.equal(output, 'test');
 		});
@@ -660,7 +659,7 @@ describe('Smoke Test Suite', function () {
 			const res = await tasks.getOutputResult();
 			assert.equal(res, '✖ 6 problems (6 errors, 0 warnings)');
 		});
-
+		
 		it(`is able to select 'Git' output`, async function () {
 			await tasks.build();
 			await app.wait();
@@ -674,7 +673,7 @@ describe('Smoke Test Suite', function () {
 			assert.ok(await tasks.firstOutputLineEndsWith('index.js'));
 		});
 
-		it(`verifies build errors are reflected in 'Problems View'`, async function () {
+		it(`verifies build errors are reflected in 'Problems View'`, async function () {		
 			await tasks.build();
 			await app.wait();
 			await tasks.openProblemsView();
@@ -717,9 +716,9 @@ describe('Smoke Test Suite', function () {
 			await extensions.installFirstResult();
 			await app.wait();
 			await extensions.getFirstReloadText();
-
+			
 			await app.stop();
-			await app.wait(); // wait until all resources are released (e.g. locked local storage)
+			await app.wait(); // wait until all resources are released (e.g. locked local storage) 
 			await app.start();
 			await extensions.selectMinimalIconsTheme();
 			const x = await extensions.verifyFolderIconAppearance();
@@ -739,13 +738,14 @@ describe('Smoke Test Suite', function () {
 			common.removeDirectory(tempUserDir);
 
 			await app.start();
-
-			let expectedTitle = 'Willkommen — vscode-smoketest-express';
-			if (process.platform !== 'darwin') {
-				expectedTitle += ' — Visual Studio Code';
-				if (insiders) expectedTitle += ' - Insiders';
-			}
-			assert.equal(await common.getWindowTitle(), expectedTitle);
+			
+			// Do not run until experiments are finished over the first-time startup behaviour.
+			// let expectedTitle = 'Willkommen — vscode-smoketest-express';
+			// if (process.platform !== 'darwin') {
+			// 	expectedTitle += ' — Visual Studio Code';
+			// 	if (insiders) expectedTitle += ' - Insiders';
+			// }
+			// assert.equal(await common.getWindowTitle(), expectedTitle);
 
 			let text = await locale.getOpenEditorsText();
 			assert.equal(text.toLowerCase(), 'geöffnete editoren');
@@ -764,8 +764,8 @@ describe('Smoke Test Suite', function () {
 
 			await locale.openViewlet(ViewletType.EXTENSIONS);
 			text = await locale.getExtensionsSearchPlaceholder();
-			assert.equal(text.toLowerCase(), 'nach extensions in marketplace suchen');
+			assert.equal(text.toLowerCase(), 'nach erweiterungen im marketplace suchen');
 		});
 	});
-
+	
 });

test/smoke/tsconfig.json

@@ -18,4 +18,4 @@
 	"exclude": [
 		"node_modules"
 	]
-}
+}