diff --git a/build/azure-pipelines/common/sanity-tests.yml b/build/azure-pipelines/common/sanity-tests.yml index fdf6b2cd3dd..7778d30a58c 100644 --- a/build/azure-pipelines/common/sanity-tests.yml +++ b/build/azure-pipelines/common/sanity-tests.yml @@ -117,23 +117,39 @@ jobs: workingDirectory: $(TEST_DIR) displayName: Compile Sanity Tests + - task: AzureKeyVault@2 + displayName: "Azure Key Vault: Get Secrets" + inputs: + azureSubscription: vscode + KeyVaultName: vscode-build-secrets + SecretsFilter: "sanity-tests-account,sanity-tests-password" + # Windows - ${{ if eq(parameters.os, 'windows') }}: - script: $(TEST_DIR)/scripts/run-win32.cmd -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }} workingDirectory: $(TEST_DIR) displayName: Run Sanity Tests + env: + GITHUB_ACCOUNT: $(sanity-tests-account) + GITHUB_PASSWORD: $(sanity-tests-password) # macOS - ${{ if eq(parameters.os, 'macOS') }}: - bash: $(TEST_DIR)/scripts/run-macOS.sh -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }} workingDirectory: $(TEST_DIR) displayName: Run Sanity Tests + env: + GITHUB_ACCOUNT: $(sanity-tests-account) + GITHUB_PASSWORD: $(sanity-tests-password) # Native Linux host - ${{ if and(eq(parameters.container, ''), eq(parameters.os, 'linux')) }}: - bash: $(TEST_DIR)/scripts/run-ubuntu.sh -c $(BUILD_COMMIT) -q $(BUILD_QUALITY) -t $(LOG_FILE) -s $(SCREENSHOTS_DIR) -v ${{ parameters.args }} workingDirectory: $(TEST_DIR) displayName: Run Sanity Tests + env: + GITHUB_ACCOUNT: $(sanity-tests-account) + GITHUB_PASSWORD: $(sanity-tests-password) # Linux Docker container - ${{ if ne(parameters.container, '') }}: @@ -164,6 +180,9 @@ jobs: ${{ parameters.args }} workingDirectory: $(TEST_DIR) displayName: Run Sanity Tests + env: + GITHUB_ACCOUNT: $(sanity-tests-account) + GITHUB_PASSWORD: $(sanity-tests-password) - bash: | mkdir -p "$(DOCKER_CACHE_DIR)" diff --git a/test/sanity/scripts/run-docker.sh b/test/sanity/scripts/run-docker.sh index 8b3da44b1f7..b91f78197d8 100755 --- a/test/sanity/scripts/run-docker.sh +++ b/test/sanity/scripts/run-docker.sh @@ -43,6 +43,8 @@ docker run \ --rm \ --platform "linux/$ARCH" \ --volume "$ROOT_DIR:/root" \ + ${GITHUB_ACCOUNT:+--env GITHUB_ACCOUNT} \ + ${GITHUB_PASSWORD:+--env GITHUB_PASSWORD} \ --entrypoint sh \ "$CONTAINER" \ /root/containers/entrypoint.sh $ARGS diff --git a/test/sanity/src/cli.test.ts b/test/sanity/src/cli.test.ts index 8732acde0b7..52e1672d54e 100644 --- a/test/sanity/src/cli.test.ts +++ b/test/sanity/src/cli.test.ts @@ -4,10 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import assert from 'assert'; -import { Browser, Page } from 'playwright'; import { TestContext } from './context.js'; -import { GitHubAuth } from './githubAuth.js'; -import { UITest } from './uiTest.js'; export function setup(context: TestContext) { context.test('cli-alpine-arm64', ['alpine', 'arm64'], async () => { @@ -78,77 +75,5 @@ export function setup(context: TestContext) { const result = context.runNoErrors(entryPoint, '--version'); const version = result.stdout.trim().match(/\(commit ([a-f0-9]+)\)/)?.[1]; assert.strictEqual(version, context.options.commit, `Expected commit ${context.options.commit} but got ${version}`); - - if (!context.capabilities.has('github-account')) { - return; - } - - const cliDataDir = context.createTempDir(); - const test = new UITest(context); - const auth = new GitHubAuth(context); - let browser: Browser | undefined; - let page: Page | undefined; - - context.log('Logging out of Dev Tunnel to ensure fresh authentication'); - context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout'); - - context.log('Starting Dev Tunnel to local server using CLI'); - await context.runCliApp('CLI', entryPoint, - [ - '--cli-data-dir', cliDataDir, - 'tunnel', - '--accept-server-license-terms', - '--server-data-dir', context.createTempDir(), - '--extensions-dir', test.extensionsDir, - '--verbose' - ], - async (line) => { - const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1]; - if (deviceCode) { - context.log(`Device code detected: ${deviceCode}, starting device flow authentication`); - browser = await context.launchBrowser(); - page = await context.getPage(browser.newPage()); - await auth.runDeviceCodeFlow(page, deviceCode); - return; - } - - const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1]; - if (tunnelUrl) { - const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!; - const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir); - context.log(`CLI started successfully with tunnel URL: ${url}`); - - if (!browser || !page) { - throw new Error('Browser instance is not available'); - } - - context.log(`Navigating to ${url}`); - await page.goto(url); - - context.log('Waiting for the workbench to load'); - await page.waitForSelector('.monaco-workbench'); - - context.log('Selecting GitHub Account'); - await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); - - context.log('Clicking Allow on confirmation dialog'); - const popup = page.waitForEvent('popup'); - await page.getByRole('button', { name: 'Allow' }).click(); - - await auth.runAuthorizeFlow(await popup); - - context.log('Waiting for connection to be established'); - await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); - - await test.run(page); - - context.log('Closing browser'); - await browser.close(); - - test.validate(); - return true; - } - } - ); } } diff --git a/test/sanity/src/context.ts b/test/sanity/src/context.ts index 02d94340375..f32c6931147 100644 --- a/test/sanity/src/context.ts +++ b/test/sanity/src/context.ts @@ -40,6 +40,7 @@ export class TestContext { private readonly wslTempDirs = new Set(); private nextPort = 3010; private currentTestName: string | undefined; + private screenshotCounter = 0; public constructor(public readonly options: Readonly<{ quality: 'stable' | 'insider' | 'exploration'; @@ -92,6 +93,7 @@ export class TestContext { const self = this; return test(name, async function () { self.currentTestName = name; + self.screenshotCounter = 0; self.log(`Starting test: ${name}`); const homeDir = os.homedir(); @@ -1133,7 +1135,7 @@ export class TestContext { const screenshotDir = this.options.screenshotsDir ?? path.join(this.osTempDir, 'vscode-sanity-screenshots'); fs.mkdirSync(screenshotDir, { recursive: true }); const sanitizedName = this.currentTestName.replace(/[^a-zA-Z0-9_-]/g, '_'); - const screenshotPath = path.join(screenshotDir, `${sanitizedName}.png`); + const screenshotPath = path.join(screenshotDir, `${sanitizedName}-${++this.screenshotCounter}.png`); await page.screenshot({ path: screenshotPath, fullPage: true }); this.log(`Screenshot saved to: ${screenshotPath}`); } catch (e) { diff --git a/test/sanity/src/devTunnel.test.ts b/test/sanity/src/devTunnel.test.ts new file mode 100644 index 00000000000..3c54432d668 --- /dev/null +++ b/test/sanity/src/devTunnel.test.ts @@ -0,0 +1,153 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Page } from 'playwright'; +import { TestContext } from './context.js'; +import { GitHubAuth } from './githubAuth.js'; +import { UITest } from './uiTest.js'; + +export function setup(context: TestContext) { + /* + TODO: @dmitrivMS Reenable other platforms once throttling issues with GitHub account are resolved. + + context.test('dev-tunnel-alpine-arm64', ['alpine', 'arm64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-alpine-arm64'); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-alpine-x64', ['alpine', 'x64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-alpine-x64'); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-linux-arm64', ['linux', 'arm64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-linux-arm64'); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-linux-armhf', ['linux', 'arm32', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-linux-armhf'); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-linux-x64', ['linux', 'x64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-linux-x64'); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + */ + + context.test('dev-tunnel-darwin-arm64', ['darwin', 'arm64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-darwin-arm64'); + context.validateAllCodesignSignatures(dir); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-darwin-x64', ['darwin', 'x64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-darwin-x64'); + context.validateAllCodesignSignatures(dir); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-win32-arm64', ['windows', 'arm64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-win32-arm64'); + context.validateAllAuthenticodeSignatures(dir); + context.validateAllVersionInfo(dir); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + context.test('dev-tunnel-win32-x64', ['windows', 'x64', 'browser', 'github-account'], async () => { + const dir = await context.downloadAndUnpack('cli-win32-x64'); + context.validateAllAuthenticodeSignatures(dir); + context.validateAllVersionInfo(dir); + const entryPoint = context.getCliEntryPoint(dir); + await testCliApp(entryPoint); + }); + + async function testCliApp(entryPoint: string) { + if (context.options.downloadOnly) { + return; + } + + const cliDataDir = context.createTempDir(); + context.log('Logging out of Dev Tunnel to ensure fresh authentication'); + context.run(entryPoint, '--cli-data-dir', cliDataDir, 'tunnel', 'user', 'logout'); + + const test = new UITest(context); + const auth = new GitHubAuth(context); + const browser = await context.launchBrowser(); + try { + const page = await context.getPage(browser.newPage()); + context.log('Starting Dev Tunnel to local server using CLI'); + await context.runCliApp('CLI', entryPoint, + [ + '--cli-data-dir', cliDataDir, + 'tunnel', + '--accept-server-license-terms', + '--server-data-dir', context.createTempDir(), + '--extensions-dir', test.extensionsDir, + '--verbose' + ], + async (line) => { + const deviceCode = /To grant access .* use code ([A-Z0-9-]+)/.exec(line)?.[1]; + if (deviceCode) { + context.log(`Device code detected: ${deviceCode}, starting device flow authentication`); + await auth.runDeviceCodeFlow(page, deviceCode); + return; + } + + const tunnelUrl = /Open this link in your browser (https?:\/\/[^\s]+)/.exec(line)?.[1]; + if (tunnelUrl) { + await connectToTunnel(tunnelUrl, page, test, auth); + await test.run(page); + test.validate(); + return true; + } + } + ); + } finally { + context.log('Closing browser'); + await browser.close(); + } + } + + async function connectToTunnel(tunnelUrl: string, page: Page, test: UITest, auth: GitHubAuth) { + try { + const tunnelId = new URL(tunnelUrl).pathname.split('/').pop()!; + const url = context.getTunnelUrl(tunnelUrl, test.workspaceDir); + context.log(`CLI started successfully with tunnel URL: ${url}`); + + context.log(`Navigating to ${url}`); + await page.goto(url); + + context.log('Waiting for the workbench to load'); + await page.waitForSelector('.monaco-workbench'); + + context.log('Selecting GitHub Account'); + await page.locator('span.monaco-highlighted-label', { hasText: 'GitHub' }).click(); + + context.log('Clicking Allow on confirmation dialog'); + const popup = page.waitForEvent('popup'); + await page.getByRole('button', { name: 'Allow' }).click(); + + await auth.runAuthorizeFlow(await popup); + + context.log('Waiting for connection to be established'); + await page.getByRole('button', { name: `remote ${tunnelId}` }).waitFor({ timeout: 5 * 60 * 1000 }); + } catch (error) { + context.log('Error during tunnel connection, capturing screenshot'); + await context.captureScreenshot(page); + throw error; + } + } +} diff --git a/test/sanity/src/githubAuth.ts b/test/sanity/src/githubAuth.ts index 0a1844f7e2b..420ca94f7e6 100644 --- a/test/sanity/src/githubAuth.ts +++ b/test/sanity/src/githubAuth.ts @@ -16,7 +16,7 @@ export class GitHubAuth { public constructor(private readonly context: TestContext) { } /** - * Runs GitHub device authentication flow in a browser. + * Runs GitHub device authentication flow in a browser, signing in first. * @param page Page to use. * @param code Device authentication code to use. */ @@ -25,26 +25,32 @@ export class GitHubAuth { this.context.error('GITHUB_ACCOUNT and GITHUB_PASSWORD environment variables must be set'); } - this.context.log(`Running GitHub device flow with code ${code}`); - await page.goto('https://github.com/login/device'); + try { + this.context.log(`Running GitHub device flow with code ${code}`); + await page.goto('https://github.com/login/device'); - this.context.log('Filling in GitHub credentials'); - await page.getByLabel('Username or email address').fill(this.username); - await page.getByLabel('Password').fill(this.password); - await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + this.context.log('Signing in to GitHub'); + await page.getByLabel('Username or email address').fill(this.username); + await page.getByLabel('Password').fill(this.password); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); - this.context.log('Confirming device activation'); - await page.getByRole('button', { name: 'Continue' }).click(); + this.context.log('Confirming signed-in account'); + await page.getByRole('button', { name: 'Continue' }).click(); - this.context.log('Entering device code'); - const codeChars = code.replace(/-/g, ''); - for (let i = 0; i < codeChars.length; i++) { - await page.getByRole('textbox').nth(i).fill(codeChars[i]); + this.context.log('Entering device code'); + const codeChars = code.replace(/-/g, ''); + for (let i = 0; i < codeChars.length; i++) { + await page.getByRole('textbox').nth(i).fill(codeChars[i]); + } + await page.getByRole('button', { name: 'Continue' }).click(); + + this.context.log('Authorizing device'); + await page.getByRole('button', { name: 'Authorize' }).click(); + } catch (error) { + this.context.log('Error during device code flow, capturing screenshot'); + await this.context.captureScreenshot(page); + throw error; } - await page.getByRole('button', { name: 'Continue' }).click(); - - this.context.log('Authorizing device'); - await page.getByRole('button', { name: 'Authorize' }).click(); } /** @@ -52,7 +58,13 @@ export class GitHubAuth { * @param page Page to use. */ public async runAuthorizeFlow(page: Page) { - this.context.log(`Authorizing app at ${page.url()}`); - await page.getByRole('button', { name: 'Continue' }).click(); + try { + this.context.log(`Authorizing app at ${page.url()}`); + await page.getByRole('button', { name: 'Continue' }).click(); + } catch (error) { + this.context.log('Error during authorization, capturing screenshot'); + await this.context.captureScreenshot(page); + throw error; + } } } diff --git a/test/sanity/src/main.ts b/test/sanity/src/main.ts index 840364e68a5..e1026eb0412 100644 --- a/test/sanity/src/main.ts +++ b/test/sanity/src/main.ts @@ -11,11 +11,12 @@ import { setup as setupDesktopTests } from './desktop.test.js'; import { setup as setupServerTests } from './server.test.js'; import { setup as setupServerWebTests } from './serverWeb.test.js'; import { setup as setupWSLTests } from './wsl.test.js'; +import { setup as setupDevTunnelTests } from './devTunnel.test.js'; const options = minimist(process.argv.slice(2), { string: ['commit', 'quality', 'screenshots-dir'], boolean: ['cleanup', 'verbose', 'signing-check', 'headless', 'detection'], - alias: { commit: 'c', quality: 'q', verbose: 'v' }, + alias: { commit: 'c', quality: 'q', verbose: 'v', 'screenshots-dir': 's' }, default: { cleanup: true, verbose: false, 'signing-check': true, headless: true, 'detection': true }, }); @@ -52,3 +53,4 @@ setupDesktopTests(context); setupServerTests(context); setupServerWebTests(context); setupWSLTests(context); +setupDevTunnelTests(context); diff --git a/test/sanity/src/wsl.test.ts b/test/sanity/src/wsl.test.ts index 77df84d7275..2108c0b08e6 100644 --- a/test/sanity/src/wsl.test.ts +++ b/test/sanity/src/wsl.test.ts @@ -157,11 +157,16 @@ export function setup(context: TestContext) { try { const window = await context.getPage(app.firstWindow()); - context.log('Installing WSL extension'); - await window.getByRole('button', { name: 'Install and Reload' }).click(); + try { + context.log('Installing WSL extension'); + await window.getByRole('button', { name: 'Install and Reload' }).click(); - context.log('Waiting for WSL connection'); - await window.getByText(/WSL/).waitFor(); + context.log('Waiting for WSL connection'); + await window.getByText(/WSL/).waitFor(); + } catch (error) { + await context.captureScreenshot(window); + throw error; + } await test.run(window); } finally {