Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-04-02 00:09:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Enable CLI DevTunnel sanity tests (#305807)

Browse Source
This commit is contained in:
Dmitriy Vasyura
2026-03-30 20:13:23 -07:00
committed by GitHub
parent d13d7c7add
commit 3d91bf7907
8 changed files with 220 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
build/azure-pipelines/common/sanity-tests.yml
View File

@@ -117,23 +117,39 @@ jobs:
workingDirectory: $(TEST_DIR)
displayName: Compile Sanity Tests
- task: AzureKeyVault@2
displayName: "Azure Key Vault: Get Secrets"
inputs:
azureSubscription: vscode
KeyVaultName: vscode-build-secrets
SecretsFilter: "sanity-tests-account,sanity-tests-password"
# Windows
- ${{ if eq(parameters.os, 'windows') }}:
- script: $(TEST_DIR)/scripts/run-win32.cmd -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }}
workingDirectory: $(TEST_DIR)
displayName: Run Sanity Tests
env:
GITHUB_ACCOUNT: $(sanity-tests-account)
GITHUB_PASSWORD: $(sanity-tests-password)
# macOS
- ${{ if eq(parameters.os, 'macOS') }}:
- bash: $(TEST_DIR)/scripts/run-macOS.sh -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }}
workingDirectory: $(TEST_DIR)
displayName: Run Sanity Tests
env:
GITHUB_ACCOUNT: $(sanity-tests-account)
GITHUB_PASSWORD: $(sanity-tests-password)
# Native Linux host
- ${{ if and(eq(parameters.container, ''), eq(parameters.os, 'linux')) }}:
- bash: $(TEST_DIR)/scripts/run-ubuntu.sh -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }}
workingDirectory: $(TEST_DIR)
displayName: Run Sanity Tests
env:
GITHUB_ACCOUNT: $(sanity-tests-account)
GITHUB_PASSWORD: $(sanity-tests-password)
# Linux Docker container
- ${{ if ne(parameters.container, '') }}:
@@ -164,6 +180,9 @@ jobs:
${{ parameters.args }}
workingDirectory: $(TEST_DIR)
displayName: Run Sanity Tests
env:
GITHUB_ACCOUNT: $(sanity-tests-account)
GITHUB_PASSWORD: $(sanity-tests-password)
- bash: |
mkdir -p "$(DOCKER_CACHE_DIR)"

2
test/sanity/scripts/run-docker.sh
View File

@@ -43,6 +43,8 @@ docker run \
--rm \
--platform "linux/$ARCH" \
--volume "$ROOT_DIR:/root" \
${GITHUB_ACCOUNT:+--env GITHUB_ACCOUNT} \
${GITHUB_PASSWORD:+--env GITHUB_PASSWORD} \
--entrypoint sh \
"$CONTAINER" \
/root/containers/entrypoint.sh $ARGS

75
test/sanity/src/cli.test.ts
View File

@@ -4,10 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import assert from 'assert';
import { Browser, Page } from 'playwright';
import { TestContext } from './context.js';
import { GitHubAuth } from './githubAuth.js';
import { UITest } from './uiTest.js';
export function setup(context: TestContext) {
context.test('cli-alpine-arm64', ['alpine', 'arm64'], async () => {
@@ -78,77 +75,5 @@ export function setup(context: TestContext) {
const result = context.runNoErrors(entryPoint, '--version');
const version = result.stdout.trim().match(/\(commit ([a-f0-9]+)\)/)?.[1];
assert.strictEqual(version, context.options.commit, `Expected commit ${context.options.commit} but got ${version}`);
if (!context.capabilities.has('github-account')) {
return;
}
const cliDataDir = context.createTempDir();
const test = new UITest(context);
const auth = new GitHubAuth(context);
let browser: Browser | undefined;
let page: Page | undefined;
context.log('Logging out of Dev Tunnel to ensure fresh authentication');
context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout');
context.log('Starting Dev Tunnel to local server using CLI');
await context.runCliApp('CLI', entryPoint,
[
'--cli-data-dir', cliDataDir,
'tunnel',
'--accept-server-license-terms',
'--server-data-dir', context.createTempDir(),
'--extensions-dir', test.extensionsDir,
'--verbose'
],
async (line) => {
const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1];
if (deviceCode) {
context.log(`Device code detected: ${deviceCode}, starting device flow authentication`);
browser = await context.launchBrowser();
page = await context.getPage(browser.newPage());
await auth.runDeviceCodeFlow(page, deviceCode);
return;
}
const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1];
if (tunnelUrl) {
const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!;
const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir);
context.log(`CLI started successfully with tunnel URL: ${url}`);
if (!browser || !page) {
throw new Error('Browser instance is not available');
}
context.log(`Navigating to ${url}`);
await page.goto(url);
context.log('Waiting for the workbench to load');
await page.waitForSelector('.monaco-workbench');
context.log('Selecting GitHub Account');
await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click();
context.log('Clicking Allow on confirmation dialog');
const popup = page.waitForEvent('popup');
await page.getByRole('button', { name: 'Allow' }).click();
await auth.runAuthorizeFlow(await popup);
context.log('Waiting for connection to be established');
await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 });
await test.run(page);
context.log('Closing browser');
await browser.close();
test.validate();
return true;
}
}
);
}
}

4
test/sanity/src/context.ts
View File

@@ -40,6 +40,7 @@ export class TestContext {
private readonly wslTempDirs = new Set<string>();
private nextPort = 3010;
private currentTestName: string | undefined;
private screenshotCounter = 0;
public constructor(public readonly options: Readonly<{
quality: 'stable' | 'insider' | 'exploration';
@@ -92,6 +93,7 @@ export class TestContext {
const self = this;
return test(name, async function () {
self.currentTestName = name;
self.screenshotCounter = 0;
self.log(`Starting test: ${name}`);
const homeDir = os.homedir();
@@ -1133,7 +1135,7 @@ export class TestContext {
const screenshotDir = this.options.screenshotsDir ?? path.join(this.osTempDir, 'vscode-sanity-screenshots');
fs.mkdirSync(screenshotDir, { recursive: true });
const sanitizedName = this.currentTestName.replace(/[^a-zA-Z0-9_-]/g, '_');
const screenshotPath = path.join(screenshotDir, `${sanitizedName}.png`);
const screenshotPath = path.join(screenshotDir, `${sanitizedName}-${++this.screenshotCounter}.png`);
await page.screenshot({ path: screenshotPath, fullPage: true });
this.log(`Screenshot saved to: ${screenshotPath}`);
} catch (e) {

153
test/sanity/src/devTunnel.test.ts Normal file
View File

@@ -0,0 +1,153 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { Page } from 'playwright';
import { TestContext } from './context.js';
import { GitHubAuth } from './githubAuth.js';
import { UITest } from './uiTest.js';
export function setup(context: TestContext) {
/*
TODO: @dmitrivMS Reenable other platforms once throttling issues with GitHub account are resolved.
context.test('dev-tunnel-alpine-arm64', ['alpine', 'arm64', 'browser', 'github-account'], async () => {
const dir = await context.downloadAndUnpack('cli-alpine-arm64');
const entryPoint = context.getCliEntryPoint(dir);
await testCliApp(entryPoint);
});
context.test('dev-tunnel-alpine-x64', ['alpine', 'x64', 'browser', 'github-account'], async () => {
const dir = await context.downloadAndUnpack('cli-alpine-x64');
const entryPoint = context.getCliEntryPoint(dir);
await testCliApp(entryPoint);
});
context.test('dev-tunnel-linux-arm64', ['linux', 'arm64', 'browser', 'github-account'], async () => {
const dir = await context.downloadAndUnpack('cli-linux-arm64');
const entryPoint = context.getCliEntryPoint(dir);
await testCliApp(entryPoint);
});
context.test('dev-tunnel-linux-armhf', ['linux', 'arm32', 'browser', 'github-account'], async () => {
const dir = await context.downloadAndUnpack('cli-linux-armhf');
const entryPoint = context.getCliEntryPoint(dir);
await testCliApp(entryPoint);
});
context.test('dev-tunnel-linux-x64', ['linux', 'x64', 'browser', 'github-account'], async () => {
const dir = await context.downloadAndUnpack('cli-linux-x64');
const entryPoint = context.getCliEntryPoint(dir);
await testCliApp(entryPoint);
});
*/
context.test('dev-tunnel-darwin-arm64', ['darwin', 'arm64', 'browser', 'github-account'], async () => {
const dir = await context.downloadAndUnpack('cli-darwin-arm64');
context.validateAllCodesignSignatures(dir);
const entryPoint = context.getCliEntryPoint(dir);
await testCliApp(entryPoint);
});
context.test('dev-tunnel-darwin-x64', ['darwin', 'x64', 'browser', 'github-account'], async () => {
const dir = await context.downloadAndUnpack('cli-darwin-x64');
context.validateAllCodesignSignatures(dir);
const entryPoint = context.getCliEntryPoint(dir);
await testCliApp(entryPoint);
});
context.test('dev-tunnel-win32-arm64', ['windows', 'arm64', 'browser', 'github-account'], async () => {
const dir = await context.downloadAndUnpack('cli-win32-arm64');
context.validateAllAuthenticodeSignatures(dir);
context.validateAllVersionInfo(dir);
const entryPoint = context.getCliEntryPoint(dir);
await testCliApp(entryPoint);
});
context.test('dev-tunnel-win32-x64', ['windows', 'x64', 'browser', 'github-account'], async () => {
const dir = await context.downloadAndUnpack('cli-win32-x64');
context.validateAllAuthenticodeSignatures(dir);
context.validateAllVersionInfo(dir);
const entryPoint = context.getCliEntryPoint(dir);
await testCliApp(entryPoint);
});
async function testCliApp(entryPoint: string) {
if (context.options.downloadOnly) {
return;
}
const cliDataDir = context.createTempDir();
context.log('Logging out of Dev Tunnel to ensure fresh authentication');
context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout');
const test = new UITest(context);
const auth = new GitHubAuth(context);
const browser = await context.launchBrowser();
try {
const page = await context.getPage(browser.newPage());
context.log('Starting Dev Tunnel to local server using CLI');
await context.runCliApp('CLI', entryPoint,
[
'--cli-data-dir', cliDataDir,
'tunnel',
'--accept-server-license-terms',
'--server-data-dir', context.createTempDir(),
'--extensions-dir', test.extensionsDir,
'--verbose'
],
async (line) => {
const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1];
if (deviceCode) {
context.log(`Device code detected: ${deviceCode}, starting device flow authentication`);
await auth.runDeviceCodeFlow(page, deviceCode);
return;
}
const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1];
if (tunnelUrl) {
await connectToTunnel(tunnelUrl, page, test, auth);
await test.run(page);
test.validate();
return true;
}
}
);
} finally {
context.log('Closing browser');
await browser.close();
}
}
async function connectToTunnel(tunnelUrl: string, page: Page, test: UITest, auth: GitHubAuth) {
try {
const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!;
const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir);
context.log(`CLI started successfully with tunnel URL: ${url}`);
context.log(`Navigating to ${url}`);
await page.goto(url);
context.log('Waiting for the workbench to load');
await page.waitForSelector('.monaco-workbench');
context.log('Selecting GitHub Account');
await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click();
context.log('Clicking Allow on confirmation dialog');
const popup = page.waitForEvent('popup');
await page.getByRole('button', { name: 'Allow' }).click();
await auth.runAuthorizeFlow(await popup);
context.log('Waiting for connection to be established');
await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 });
} catch (error) {
context.log('Error during tunnel connection, capturing screenshot');
await context.captureScreenshot(page);
throw error;
}
}
}

50
test/sanity/src/githubAuth.ts
View File

@@ -16,7 +16,7 @@ export class GitHubAuth {
public constructor(private readonly context: TestContext) { }
/**
* Runs GitHub device authentication flow in a browser.
* Runs GitHub device authentication flow in a browser, signing in first.
* @param page Page to use.
* @param code Device authentication code to use.
*/
@@ -25,26 +25,32 @@ export class GitHubAuth {
this.context.error('GITHUB_ACCOUNT and GITHUB_PASSWORD environment variables must be set');
}
this.context.log(`Running GitHub device flow with code ${code}`);
await page.goto('https://github.com/login/device');
try {
this.context.log(`Running GitHub device flow with code ${code}`);
await page.goto('https://github.com/login/device');
this.context.log('Filling in GitHub credentials');
await page.getByLabel('Username or email address').fill(this.username);
await page.getByLabel('Password').fill(this.password);
await page.getByRole('button', { name: 'Sign in', exact: true }).click();
this.context.log('Signing in to GitHub');
await page.getByLabel('Username or email address').fill(this.username);
await page.getByLabel('Password').fill(this.password);
await page.getByRole('button', { name: 'Sign in', exact: true }).click();
this.context.log('Confirming device activation');
await page.getByRole('button', { name: 'Continue' }).click();
this.context.log('Confirming signed-in account');
await page.getByRole('button', { name: 'Continue' }).click();
this.context.log('Entering device code');
const codeChars = code.replace(/-/g, '');
for (let i = 0; i < codeChars.length; i++) {
await page.getByRole('textbox').nth(i).fill(codeChars[i]);
this.context.log('Entering device code');
const codeChars = code.replace(/-/g, '');
for (let i = 0; i < codeChars.length; i++) {
await page.getByRole('textbox').nth(i).fill(codeChars[i]);
}
await page.getByRole('button', { name: 'Continue' }).click();
this.context.log('Authorizing device');
await page.getByRole('button', { name: 'Authorize' }).click();
} catch (error) {
this.context.log('Error during device code flow, capturing screenshot');
await this.context.captureScreenshot(page);
throw error;
}
await page.getByRole('button', { name: 'Continue' }).click();
this.context.log('Authorizing device');
await page.getByRole('button', { name: 'Authorize' }).click();
}
/**
@@ -52,7 +58,13 @@ export class GitHubAuth {
* @param page Page to use.
*/
public async runAuthorizeFlow(page: Page) {
this.context.log(`Authorizing app at ${page.url()}`);
await page.getByRole('button', { name: 'Continue' }).click();
try {
this.context.log(`Authorizing app at ${page.url()}`);
await page.getByRole('button', { name: 'Continue' }).click();
} catch (error) {
this.context.log('Error during authorization, capturing screenshot');
await this.context.captureScreenshot(page);
throw error;
}
}
}

4
test/sanity/src/main.ts
View File

@@ -11,11 +11,12 @@ import { setup as setupDesktopTests } from './desktop.test.js';
import { setup as setupServerTests } from './server.test.js';
import { setup as setupServerWebTests } from './serverWeb.test.js';
import { setup as setupWSLTests } from './wsl.test.js';
import { setup as setupDevTunnelTests } from './devTunnel.test.js';
const options = minimist(process.argv.slice(2), {
string: ['commit', 'quality', 'screenshots-dir'],
boolean: ['cleanup', 'verbose', 'signing-check', 'headless', 'detection'],
alias: { commit: 'c', quality: 'q', verbose: 'v' },
alias: { commit: 'c', quality: 'q', verbose: 'v', 'screenshots-dir': 's' },
default: { cleanup: true, verbose: false, 'signing-check': true, headless: true, 'detection': true },
});
@@ -52,3 +53,4 @@ setupDesktopTests(context);
setupServerTests(context);
setupServerWebTests(context);
setupWSLTests(context);
setupDevTunnelTests(context);

13
test/sanity/src/wsl.test.ts
View File

@@ -157,11 +157,16 @@ export function setup(context: TestContext) {
try {
const window = await context.getPage(app.firstWindow());
context.log('Installing WSL extension');
await window.getByRole('button', { name: 'Install and Reload' }).click();
try {
context.log('Installing WSL extension');
await window.getByRole('button', { name: 'Install and Reload' }).click();
context.log('Waiting for WSL connection');
await window.getByText(/WSL/).waitFor();
context.log('Waiting for WSL connection');
await window.getByText(/WSL/).waitFor();
} catch (error) {
await context.captureScreenshot(window);
throw error;
}
await test.run(window);
} finally {