Enable CLI DevTunnel sanity tests (#305807)

build/azure-pipelines/common/sanity-tests.yml

@@ -117,23 +117,39 @@ jobs:
         workingDirectory: $(TEST_DIR)
         displayName: Compile Sanity Tests
 
+      - task: AzureKeyVault@2
+        displayName: "Azure Key Vault: Get Secrets"
+        inputs:
+          azureSubscription: vscode
+          KeyVaultName: vscode-build-secrets
+          SecretsFilter: "sanity-tests-account,sanity-tests-password"
+
       # Windows
       - ${{ if eq(parameters.os, 'windows') }}:
         - script: $(TEST_DIR)/scripts/run-win32.cmd -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }}
           workingDirectory: $(TEST_DIR)
           displayName: Run Sanity Tests
+          env:
+            GITHUB_ACCOUNT: $(sanity-tests-account)
+            GITHUB_PASSWORD: $(sanity-tests-password)
 
       # macOS
       - ${{ if eq(parameters.os, 'macOS') }}:
         - bash: $(TEST_DIR)/scripts/run-macOS.sh -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }}
           workingDirectory: $(TEST_DIR)
           displayName: Run Sanity Tests
+          env:
+            GITHUB_ACCOUNT: $(sanity-tests-account)
+            GITHUB_PASSWORD: $(sanity-tests-password)
 
       # Native Linux host
       - ${{ if and(eq(parameters.container, ''), eq(parameters.os, 'linux')) }}:
         - bash: $(TEST_DIR)/scripts/run-ubuntu.sh -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }}
           workingDirectory: $(TEST_DIR)
           displayName: Run Sanity Tests
+          env:
+            GITHUB_ACCOUNT: $(sanity-tests-account)
+            GITHUB_PASSWORD: $(sanity-tests-password)
 
       # Linux Docker container
       - ${{ if ne(parameters.container, '') }}:
@@ -164,6 +180,9 @@ jobs:
               ${{ parameters.args }}
           workingDirectory: $(TEST_DIR)
           displayName: Run Sanity Tests
+          env:
+            GITHUB_ACCOUNT: $(sanity-tests-account)
+            GITHUB_PASSWORD: $(sanity-tests-password)
 
         - bash: |
             mkdir -p "$(DOCKER_CACHE_DIR)"

test/sanity/scripts/run-docker.sh

@@ -43,6 +43,8 @@ docker run \
 	--rm \
 	--platform "linux/$ARCH" \
 	--volume "$ROOT_DIR:/root" \
+	${GITHUB_ACCOUNT:+--env GITHUB_ACCOUNT} \
+	${GITHUB_PASSWORD:+--env GITHUB_PASSWORD} \
 	--entrypoint sh \
 	"$CONTAINER" \
 	/root/containers/entrypoint.sh $ARGS

test/sanity/src/cli.test.ts

@@ -4,10 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import assert from 'assert';
-import { Browser, Page } from 'playwright';
 import { TestContext } from './context.js';
-import { GitHubAuth } from './githubAuth.js';
-import { UITest } from './uiTest.js';
 
 export function setup(context: TestContext) {
 	context.test('cli-alpine-arm64', ['alpine', 'arm64'], async () => {
@@ -78,77 +75,5 @@ export function setup(context: TestContext) {
 		const result = context.runNoErrors(entryPoint, '--version');
 		const version = result.stdout.trim().match(/\(commit ([a-f0-9]+)\)/)?.[1];
 		assert.strictEqual(version, context.options.commit, `Expected commit ${context.options.commit} but got ${version}`);
-
-		if (!context.capabilities.has('github-account')) {
-			return;
-		}
-
-		const cliDataDir = context.createTempDir();
-		const test = new UITest(context);
-		const auth = new GitHubAuth(context);
-		let browser: Browser | undefined;
-		let page: Page | undefined;
-
-		context.log('Logging out of Dev Tunnel to ensure fresh authentication');
-		context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout');
-
-		context.log('Starting Dev Tunnel to local server using CLI');
-		await context.runCliApp('CLI', entryPoint,
-			[
-				'--cli-data-dir', cliDataDir,
-				'tunnel',
-				'--accept-server-license-terms',
-				'--server-data-dir', context.createTempDir(),
-				'--extensions-dir', test.extensionsDir,
-				'--verbose'
-			],
-			async (line) => {
-				const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1];
-				if (deviceCode) {
-					context.log(`Device code detected: ${deviceCode}, starting device flow authentication`);
-					browser = await context.launchBrowser();
-					page = await context.getPage(browser.newPage());
-					await auth.runDeviceCodeFlow(page, deviceCode);
-					return;
-				}
-
-				const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1];
-				if (tunnelUrl) {
-					const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!;
-					const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir);
-					context.log(`CLI started successfully with tunnel URL: ${url}`);
-
-					if (!browser || !page) {
-						throw new Error('Browser instance is not available');
-					}
-
-					context.log(`Navigating to ${url}`);
-					await page.goto(url);
-
-					context.log('Waiting for the workbench to load');
-					await page.waitForSelector('.monaco-workbench');
-
-					context.log('Selecting GitHub Account');
-					await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click();
-
-					context.log('Clicking Allow on confirmation dialog');
-					const popup = page.waitForEvent('popup');
-					await page.getByRole('button', { name: 'Allow' }).click();
-
-					await auth.runAuthorizeFlow(await popup);
-
-					context.log('Waiting for connection to be established');
-					await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 });
-
-					await test.run(page);
-
-					context.log('Closing browser');
-					await browser.close();
-
-					test.validate();
-					return true;
-				}
-			}
-		);
 	}
 }

test/sanity/src/context.ts

@@ -40,6 +40,7 @@ export class TestContext {
 	private readonly wslTempDirs = new Set<string>();
 	private nextPort = 3010;
 	private currentTestName: string | undefined;
+	private screenshotCounter = 0;
 
 	public constructor(public readonly options: Readonly<{
 		quality: 'stable' | 'insider' | 'exploration';
@@ -92,6 +93,7 @@ export class TestContext {
 		const self = this;
 		return test(name, async function () {
 			self.currentTestName = name;
+			self.screenshotCounter = 0;
 			self.log(`Starting test: ${name}`);
 
 			const homeDir = os.homedir();
@@ -1133,7 +1135,7 @@ export class TestContext {
 			const screenshotDir = this.options.screenshotsDir ?? path.join(this.osTempDir, 'vscode-sanity-screenshots');
 			fs.mkdirSync(screenshotDir, { recursive: true });
 			const sanitizedName = this.currentTestName.replace(/[^a-zA-Z0-9_-]/g, '_');
-			const screenshotPath = path.join(screenshotDir, `${sanitizedName}.png`);
+			const screenshotPath = path.join(screenshotDir, `${sanitizedName}-${++this.screenshotCounter}.png`);
 			await page.screenshot({ path: screenshotPath, fullPage: true });
 			this.log(`Screenshot saved to: ${screenshotPath}`);
 		} catch (e) {

test/sanity/src/devTunnel.test.ts

@@ -0,0 +1,153 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Page } from 'playwright';
+import { TestContext } from './context.js';
+import { GitHubAuth } from './githubAuth.js';
+import { UITest } from './uiTest.js';
+
+export function setup(context: TestContext) {
+	/*
+	TODO: @dmitrivMS Reenable other platforms once throttling issues with GitHub account are resolved.
+
+	context.test('dev-tunnel-alpine-arm64', ['alpine', 'arm64', 'browser', 'github-account'], async () => {
+		const dir = await context.downloadAndUnpack('cli-alpine-arm64');
+		const entryPoint = context.getCliEntryPoint(dir);
+		await testCliApp(entryPoint);
+	});
+
+	context.test('dev-tunnel-alpine-x64', ['alpine', 'x64', 'browser', 'github-account'], async () => {
+		const dir = await context.downloadAndUnpack('cli-alpine-x64');
+		const entryPoint = context.getCliEntryPoint(dir);
+		await testCliApp(entryPoint);
+	});
+
+	context.test('dev-tunnel-linux-arm64', ['linux', 'arm64', 'browser', 'github-account'], async () => {
+		const dir = await context.downloadAndUnpack('cli-linux-arm64');
+		const entryPoint = context.getCliEntryPoint(dir);
+		await testCliApp(entryPoint);
+	});
+
+	context.test('dev-tunnel-linux-armhf', ['linux', 'arm32', 'browser', 'github-account'], async () => {
+		const dir = await context.downloadAndUnpack('cli-linux-armhf');
+		const entryPoint = context.getCliEntryPoint(dir);
+		await testCliApp(entryPoint);
+	});
+
+	context.test('dev-tunnel-linux-x64', ['linux', 'x64', 'browser', 'github-account'], async () => {
+		const dir = await context.downloadAndUnpack('cli-linux-x64');
+		const entryPoint = context.getCliEntryPoint(dir);
+		await testCliApp(entryPoint);
+	});
+
+	*/
+
+	context.test('dev-tunnel-darwin-arm64', ['darwin', 'arm64', 'browser', 'github-account'], async () => {
+		const dir = await context.downloadAndUnpack('cli-darwin-arm64');
+		context.validateAllCodesignSignatures(dir);
+		const entryPoint = context.getCliEntryPoint(dir);
+		await testCliApp(entryPoint);
+	});
+
+	context.test('dev-tunnel-darwin-x64', ['darwin', 'x64', 'browser', 'github-account'], async () => {
+		const dir = await context.downloadAndUnpack('cli-darwin-x64');
+		context.validateAllCodesignSignatures(dir);
+		const entryPoint = context.getCliEntryPoint(dir);
+		await testCliApp(entryPoint);
+	});
+
+	context.test('dev-tunnel-win32-arm64', ['windows', 'arm64', 'browser', 'github-account'], async () => {
+		const dir = await context.downloadAndUnpack('cli-win32-arm64');
+		context.validateAllAuthenticodeSignatures(dir);
+		context.validateAllVersionInfo(dir);
+		const entryPoint = context.getCliEntryPoint(dir);
+		await testCliApp(entryPoint);
+	});
+
+	context.test('dev-tunnel-win32-x64', ['windows', 'x64', 'browser', 'github-account'], async () => {
+		const dir = await context.downloadAndUnpack('cli-win32-x64');
+		context.validateAllAuthenticodeSignatures(dir);
+		context.validateAllVersionInfo(dir);
+		const entryPoint = context.getCliEntryPoint(dir);
+		await testCliApp(entryPoint);
+	});
+
+	async function testCliApp(entryPoint: string) {
+		if (context.options.downloadOnly) {
+			return;
+		}
+
+		const cliDataDir = context.createTempDir();
+		context.log('Logging out of Dev Tunnel to ensure fresh authentication');
+		context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout');
+
+		const test = new UITest(context);
+		const auth = new GitHubAuth(context);
+		const browser = await context.launchBrowser();
+		try {
+			const page = await context.getPage(browser.newPage());
+			context.log('Starting Dev Tunnel to local server using CLI');
+			await context.runCliApp('CLI', entryPoint,
+				[
+					'--cli-data-dir', cliDataDir,
+					'tunnel',
+					'--accept-server-license-terms',
+					'--server-data-dir', context.createTempDir(),
+					'--extensions-dir', test.extensionsDir,
+					'--verbose'
+				],
+				async (line) => {
+					const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1];
+					if (deviceCode) {
+						context.log(`Device code detected: ${deviceCode}, starting device flow authentication`);
+						await auth.runDeviceCodeFlow(page, deviceCode);
+						return;
+					}
+
+					const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1];
+					if (tunnelUrl) {
+						await connectToTunnel(tunnelUrl, page, test, auth);
+						await test.run(page);
+						test.validate();
+						return true;
+					}
+				}
+			);
+		} finally {
+			context.log('Closing browser');
+			await browser.close();
+		}
+	}
+
+	async function connectToTunnel(tunnelUrl: string, page: Page, test: UITest, auth: GitHubAuth) {
+		try {
+			const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!;
+			const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir);
+			context.log(`CLI started successfully with tunnel URL: ${url}`);
+
+			context.log(`Navigating to ${url}`);
+			await page.goto(url);
+
+			context.log('Waiting for the workbench to load');
+			await page.waitForSelector('.monaco-workbench');
+
+			context.log('Selecting GitHub Account');
+			await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click();
+
+			context.log('Clicking Allow on confirmation dialog');
+			const popup = page.waitForEvent('popup');
+			await page.getByRole('button', { name: 'Allow' }).click();
+
+			await auth.runAuthorizeFlow(await popup);
+
+			context.log('Waiting for connection to be established');
+			await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 });
+		} catch (error) {
+			context.log('Error during tunnel connection, capturing screenshot');
+			await context.captureScreenshot(page);
+			throw error;
+		}
+	}
+}

test/sanity/src/githubAuth.ts

@@ -16,7 +16,7 @@ export class GitHubAuth {
 	public constructor(private readonly context: TestContext) { }
 
 	/**
-	 * Runs GitHub device authentication flow in a browser.
+	 * Runs GitHub device authentication flow in a browser, signing in first.
 	 * @param page Page to use.
 	 * @param code Device authentication code to use.
 	 */
@@ -25,26 +25,32 @@ export class GitHubAuth {
 			this.context.error('GITHUB_ACCOUNT and GITHUB_PASSWORD environment variables must be set');
 		}
 
-		this.context.log(`Running GitHub device flow with code ${code}`);
+		try {
-		await page.goto('https://github.com/login/device');
+			this.context.log(`Running GitHub device flow with code ${code}`);
+			await page.goto('https://github.com/login/device');
 
-		this.context.log('Filling in GitHub credentials');
+			this.context.log('Signing in to GitHub');
-		await page.getByLabel('Username or email address').fill(this.username);
+			await page.getByLabel('Username or email address').fill(this.username);
-		await page.getByLabel('Password').fill(this.password);
+			await page.getByLabel('Password').fill(this.password);
-		await page.getByRole('button', { name: 'Sign in', exact: true }).click();
+			await page.getByRole('button', { name: 'Sign in', exact: true }).click();
 
-		this.context.log('Confirming device activation');
+			this.context.log('Confirming signed-in account');
-		await page.getByRole('button', { name: 'Continue' }).click();
+			await page.getByRole('button', { name: 'Continue' }).click();
 
-		this.context.log('Entering device code');
+			this.context.log('Entering device code');
-		const codeChars = code.replace(/-/g, '');
+			const codeChars = code.replace(/-/g, '');
-		for (let i = 0; i < codeChars.length; i++) {
+			for (let i = 0; i < codeChars.length; i++) {
-			await page.getByRole('textbox').nth(i).fill(codeChars[i]);
+				await page.getByRole('textbox').nth(i).fill(codeChars[i]);
+			}
+			await page.getByRole('button', { name: 'Continue' }).click();
+
+			this.context.log('Authorizing device');
+			await page.getByRole('button', { name: 'Authorize' }).click();
+		} catch (error) {
+			this.context.log('Error during device code flow, capturing screenshot');
+			await this.context.captureScreenshot(page);
+			throw error;
 		}
-		await page.getByRole('button', { name: 'Continue' }).click();
-
-		this.context.log('Authorizing device');
-		await page.getByRole('button', { name: 'Authorize' }).click();
 	}
 
 	/**
@@ -52,7 +58,13 @@ export class GitHubAuth {
 	 * @param page Page to use.
 	 */
 	public async runAuthorizeFlow(page: Page) {
-		this.context.log(`Authorizing app at ${page.url()}`);
+		try {
-		await page.getByRole('button', { name: 'Continue' }).click();
+			this.context.log(`Authorizing app at ${page.url()}`);
+			await page.getByRole('button', { name: 'Continue' }).click();
+		} catch (error) {
+			this.context.log('Error during authorization, capturing screenshot');
+			await this.context.captureScreenshot(page);
+			throw error;
+		}
 	}
 }

test/sanity/src/main.ts

@@ -11,11 +11,12 @@ import { setup as setupDesktopTests } from './desktop.test.js';
 import { setup as setupServerTests } from './server.test.js';
 import { setup as setupServerWebTests } from './serverWeb.test.js';
 import { setup as setupWSLTests } from './wsl.test.js';
+import { setup as setupDevTunnelTests } from './devTunnel.test.js';
 
 const options = minimist(process.argv.slice(2), {
 	string: ['commit', 'quality', 'screenshots-dir'],
 	boolean: ['cleanup', 'verbose', 'signing-check', 'headless', 'detection'],
-	alias: { commit: 'c', quality: 'q', verbose: 'v' },
+	alias: { commit: 'c', quality: 'q', verbose: 'v', 'screenshots-dir': 's' },
 	default: { cleanup: true, verbose: false, 'signing-check': true, headless: true, 'detection': true },
 });
 
@@ -52,3 +53,4 @@ setupDesktopTests(context);
 setupServerTests(context);
 setupServerWebTests(context);
 setupWSLTests(context);
+setupDevTunnelTests(context);

test/sanity/src/wsl.test.ts

@@ -157,11 +157,16 @@ export function setup(context: TestContext) {
 		try {
 			const window = await context.getPage(app.firstWindow());
 
-			context.log('Installing WSL extension');
+			try {
-			await window.getByRole('button', { name: 'Install and Reload' }).click();
+				context.log('Installing WSL extension');
+				await window.getByRole('button', { name: 'Install and Reload' }).click();
 
-			context.log('Waiting for WSL connection');
+				context.log('Waiting for WSL connection');
-			await window.getByText(/WSL/).waitFor();
+				await window.getByText(/WSL/).waitFor();
+			} catch (error) {
+				await context.captureScreenshot(window);
+				throw error;
+			}
 
 			await test.run(window);
 		} finally {