diff --git a/eslint.config.js b/eslint.config.js index b8bcee337fce..37fb7fe63bf5 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -1935,6 +1935,13 @@ export default tseslint.config( '*' // node modules ] }, + { + 'target': 'test/sanity/**', + 'restrictions': [ + 'test/sanity/**', + '*' // node modules + ] + }, { 'target': 'test/automation/**', 'restrictions': [ diff --git a/package.json b/package.json index f7f3d89d1413..db996b54ddaf 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "mixin-telemetry-docs": "node build/npm/mixin-telemetry-docs.ts", "smoketest": "node build/lib/preLaunch.ts && cd test/smoke && npm run compile && node test/index.js", "smoketest-no-compile": "cd test/smoke && node test/index.js", + "sanity-test": "cd test/sanity && npm run compile && node out/index.js", "download-builtin-extensions": "node build/lib/builtInExtensions.ts", "download-builtin-extensions-cg": "node build/lib/builtInExtensionsCG.ts", "monaco-compile-check": "tsgo --project src/tsconfig.monaco.json --noEmit", diff --git a/test/README.md b/test/README.md index 1e4d114dce5b..727f5c2ccf2f 100644 --- a/test/README.md +++ b/test/README.md @@ -1,9 +1,10 @@ -# VSCode Tests +# VS Code Tests ## Contents -This folder contains the various test runners for VSCode. Please refer to the documentation within for how to run them: +This folder contains the various test runners for VS Code. Please refer to the documentation within for how to run them: * `unit`: our suite of unit tests ([README](unit/README.md)) * `integration`: our suite of API tests ([README](integration/browser/README.md)) * `smoke`: our suite of automated UI tests ([README](smoke/README.md)) +* `sanity`: release sanity tests ([README](sanity/README.md)) diff --git a/test/sanity/.gitignore b/test/sanity/.gitignore new file mode 100644 index 000000000000..e7d563c46ad5 --- /dev/null +++ b/test/sanity/.gitignore @@ -0,0 +1,5 @@ +.DS_Store +npm-debug.log +Thumbs.db +node_modules/ +out/ diff --git a/test/sanity/README.md b/test/sanity/README.md new file mode 100644 index 000000000000..f1f52b8b1a19 --- /dev/null +++ b/test/sanity/README.md @@ -0,0 +1,46 @@ +# VS Code Release Sanity Check Tests + +## Overview + +Automated end-to-end release sanity tests for published VS Code builds. +These tests verify critical functionality across different platforms and installation methods, +ensuring that published builds meet quality standards before reaching end users. + +See [Sanity Check wiki page](https://github.com/microsoft/vscode/wiki/Sanity-Check) for more details on sanity testing. + +## Usage + +Many tests will use the underlying platform to install and verify basic VS Code functionality. +Such tests will need to be run on the corresponding target OS/virtual machine and will fail if ran outside. +Use -g or -f command-line options to filter tests to match the host platform. + +### Command-Line Options + +| Option | Alias | Description | +|--------|-------|-------------| +| `--commit ` | `-c` | The commit SHA to test (required) | +| `--quality ` | `-q` | The quality to test (required, "stable", "insider" or "exploration") | +| `--no-cleanup` | | Do not cleanup downloaded files after each test | +| `--grep ` | `-g` | Only run tests matching the given regex pattern | +| `--fgrep ` | `-f` | Only run tests containing the given string | +| `--help` | `-h` | Show help message | + +### Example + +To run CLI tests for all platforms on given commit of Insiders build, from the root directory run: + +```bash +npm run sanity-test -- --commit 19228f26df517fecbfda96c20956f7c521e072be --quality insider -g "cli*" +``` + +## References + +The following public documentation pages provide details on end-user VS Code setup scenarios. + +- [Setup Overview](https://code.visualstudio.com/docs/setup/setup-overview) +- [Linux Setup](https://code.visualstudio.com/docs/setup/linux) +- [macOS Setup](https://code.visualstudio.com/docs/setup/mac) +- [Windows Setup](https://code.visualstudio.com/docs/setup/windows) +- [Portable Mode](https://code.visualstudio.com/docs/editor/portable) +- [VS Code Server](https://code.visualstudio.com/docs/remote/vscode-server) +- [Developing in WSL](https://code.visualstudio.com/docs/remote/wsl) diff --git a/test/sanity/package-lock.json b/test/sanity/package-lock.json new file mode 100644 index 000000000000..13c973d80618 --- /dev/null +++ b/test/sanity/package-lock.json @@ -0,0 +1,172 @@ +{ + "name": "code-oss-dev-sanity-test", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "code-oss-dev-sanity-test", + "version": "0.1.0", + "license": "MIT", + "dependencies": { + "node-fetch": "^3.3.2", + "playwright": "^1.57.0" + }, + "devDependencies": { + "@types/node": "22.x" + } + }, + "node_modules/@types/node": { + "version": "22.19.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.1.tgz", + "integrity": "sha512-LCCV0HdSZZZb34qifBsyWlUmok6W7ouER+oQIGBScS8EsZsQbrtFTUrDX4hOl+CS6p7cnNC4td+qrSVGSCTUfQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/playwright": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.57.0.tgz", + "integrity": "sha512-ilYQj1s8sr2ppEJ2YVadYBN0Mb3mdo9J0wQ+UuDhzYqURwSoW4n1Xs5vs7ORwgDGmyEh33tRMeS8KhdkMoLXQw==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.57.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.57.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.57.0.tgz", + "integrity": "sha512-agTcKlMw/mjBWOnD6kFZttAAGHgi/Nw0CZ2o6JqWSbMlI219lAFLZZCyqByTsvVAJq5XA5H8cA6PrvBRpBWEuQ==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + } + } +} diff --git a/test/sanity/package.json b/test/sanity/package.json new file mode 100644 index 000000000000..005e66b248ad --- /dev/null +++ b/test/sanity/package.json @@ -0,0 +1,17 @@ +{ + "name": "code-oss-dev-sanity-test", + "version": "0.1.0", + "license": "MIT", + "main": "./out/index.js", + "scripts": { + "compile": "node ../../node_modules/typescript/bin/tsc", + "start": "node ./out/index.js" + }, + "dependencies": { + "node-fetch": "^3.3.2", + "playwright": "^1.57.0" + }, + "devDependencies": { + "@types/node": "22.x" + } +} diff --git a/test/sanity/src/cli.test.ts b/test/sanity/src/cli.test.ts new file mode 100644 index 000000000000..1b574c75b3da --- /dev/null +++ b/test/sanity/src/cli.test.ts @@ -0,0 +1,138 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { spawn } from 'child_process'; +import { TestContext } from './context'; + +export function setup(context: TestContext) { + describe('CLI', () => { + if (context.platform === 'linux-arm64') { + it('cli-alpine-arm64', async () => { + const dir = await context.downloadAndUnpack('cli-alpine-arm64'); + const entryPoint = context.getEntryPoint('cli', dir); + await testCliApp(entryPoint); + }); + } + + if (context.platform === 'linux-x64') { + it('cli-alpine-x64', async () => { + const dir = await context.downloadAndUnpack('cli-alpine-x64'); + const entryPoint = context.getEntryPoint('cli', dir); + await testCliApp(entryPoint); + }); + } + + if (context.platform === 'darwin-arm64') { + it('cli-darwin-arm64', async () => { + const dir = await context.downloadAndUnpack('cli-darwin-arm64'); + const entryPoint = context.getEntryPoint('cli', dir); + await testCliApp(entryPoint); + }); + } + + if (context.platform === 'darwin-x64') { + it('cli-darwin-x64', async () => { + const dir = await context.downloadAndUnpack('cli-darwin-x64'); + const entryPoint = context.getEntryPoint('cli', dir); + await testCliApp(entryPoint); + }); + } + + if (context.platform === 'linux-arm64') { + it('cli-linux-arm64', async () => { + const dir = await context.downloadAndUnpack('cli-linux-arm64'); + const entryPoint = context.getEntryPoint('cli', dir); + await testCliApp(entryPoint); + }); + } + + if (context.platform === 'linux-arm') { + it('cli-linux-armhf', async () => { + const dir = await context.downloadAndUnpack('cli-linux-armhf'); + const entryPoint = context.getEntryPoint('cli', dir); + await testCliApp(entryPoint); + }); + } + + if (context.platform === 'linux-x64') { + it('cli-linux-x64', async () => { + const dir = await context.downloadAndUnpack('cli-linux-x64'); + const entryPoint = context.getEntryPoint('cli', dir); + await testCliApp(entryPoint); + }); + } + + if (context.platform === 'win32-arm64') { + it('cli-win32-arm64', async () => { + const dir = await context.downloadAndUnpack('cli-win32-arm64'); + context.validateAllSignatures(dir); + const entryPoint = context.getEntryPoint('cli', dir); + await testCliApp(entryPoint); + }); + } + + if (context.platform === 'win32-x64') { + it('cli-win32-x64', async () => { + const dir = await context.downloadAndUnpack('cli-win32-x64'); + context.validateAllSignatures(dir); + const entryPoint = context.getEntryPoint('cli', dir); + await testCliApp(entryPoint); + }); + } + + async function testCliApp(entryPoint: string) { + const result = context.runNoErrors(entryPoint, '--version'); + const version = result.stdout.trim(); + assert.ok(version.includes(`(commit ${context.commit})`)); + + const workspaceDir = context.createTempDir(); + process.chdir(workspaceDir); + context.log(`Changed current directory to: ${workspaceDir}`); + + const cliDataDir = context.createTempDir(); + const userDataDir = context.createTempDir(); + const serverDataDir = context.createTempDir(); + const extensionsDir = context.createTempDir(); + const args = [ + '--cli-data-dir', cliDataDir, + '--user-data-dir', userDataDir, + 'tunnel', + '--accept-server-license-terms', + '--server-data-dir', serverDataDir, + '--extensions-dir', extensionsDir, + ]; + + context.log(`Running CLI ${entryPoint} with args: ${args.join(' ')}`); + const cli = spawn(entryPoint, args); + + cli.stderr.on('data', (data) => { + context.error(`[CLI Error] ${data.toString().trim()}`); + }); + + let tunnelUrl: string | undefined = undefined; + cli.stdout.on('data', (data) => { + const text = data.toString(); + text.trim().split('\n').forEach((line: string) => { + context.log(`[CLI Output] ${line}`); + }); + + const match = /Open this link in your browser (https:\/\/.+)/.exec(text); + if (match !== null) { + tunnelUrl = context.getTunnelUrl(match[1]); + context.log(`Tunnel URL: ${tunnelUrl}`); + cli.kill(); + } + }); + + await new Promise((resolve, reject) => { + cli.on('error', reject); + cli.on('exit', () => resolve()); + }); + + assert.ok(tunnelUrl, 'Expected to receive a tunnel URL from the CLI'); + } + }); +} diff --git a/test/sanity/src/context.ts b/test/sanity/src/context.ts new file mode 100644 index 000000000000..6228aefb0cb4 --- /dev/null +++ b/test/sanity/src/context.ts @@ -0,0 +1,439 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { spawnSync, SpawnSyncReturns } from 'child_process'; +import { createHash } from 'crypto'; +import fs from 'fs'; +import fetch, { Response } from 'node-fetch'; +import os from 'os'; +import path from 'path'; + +/** + * Response from https://update.code.visualstudio.com/api/versions/commit:// + */ +interface ITargetMetadata { + url: string; + name: string; + version: string; + productVersion: string; + hash: string; + timestamp: number; + sha256hash: string; + supportsFastUpdate: boolean; +} + +/** + * Provides context and utilities for VS Code sanity tests. + */ +export class TestContext { + private static readonly authenticodeInclude = /^.+\.(exe|dll|sys|cab|cat|msi|jar|ocx|ps1|psm1|psd1|ps1xml|pssc1)$/i; + + private readonly tempDirs = new Set(); + private readonly logFile: string; + + public constructor( + public readonly quality: 'stable' | 'insider' | 'exploration', + public readonly commit: string, + public readonly verbose: boolean, + ) { + const osTempDir = fs.realpathSync(os.tmpdir()); + const logDir = fs.mkdtempSync(path.join(osTempDir, 'vscode-sanity-log')); + this.logFile = path.join(logDir, 'sanity.log'); + console.log(`Log file: ${this.logFile}`); + } + + /** + * Returns the current platform in the format -. + */ + public get platform(): string { + return `${os.platform()}-${os.arch()}`; + } + + /** + * Logs a message with a timestamp. + */ + public log(message: string) { + const line = `[${new Date().toISOString()}] ${message}\n`; + fs.appendFileSync(this.logFile, line); + if (this.verbose) { + console.log(line.trimEnd()); + } + } + + /** + * Logs an error message and throws an Error. + */ + public error(message: string): never { + const line = `[${new Date().toISOString()}] ERROR: ${message}\n`; + fs.appendFileSync(this.logFile, line); + console.error(line.trimEnd()); + throw new Error(message); + } + + /** + * Creates a new temporary directory and returns its path. + */ + public createTempDir(): string { + const osTempDir = fs.realpathSync(os.tmpdir()); + const tempDir = fs.mkdtempSync(path.join(osTempDir, 'vscode-sanity')); + this.log(`Created temp directory: ${tempDir}`); + this.tempDirs.add(tempDir); + return tempDir; + } + + /** + * Ensures that the directory for the specified file path exists. + */ + public ensureDirExists(filePath: string) { + const dir = path.dirname(filePath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + } + + /** + * Cleans up all temporary directories created during the test run. + */ + public cleanup() { + for (const dir of this.tempDirs) { + this.log(`Deleting temp directory: ${dir}`); + try { + fs.rmSync(dir, { recursive: true, force: true }); + this.log(`Deleted temp directory: ${dir}`); + } catch (error) { + this.log(`Failed to delete temp directory: ${dir}: ${error}`); + } + } + this.tempDirs.clear(); + } + + /** + * Fetches a URL and ensures there are no errors. + * @param url The URL to fetch. + * @returns The fetch Response object. + */ + public async fetchNoErrors(url: string): Promise { + const response = await fetch(url); + if (!response.ok) { + this.error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); + } + + if (response.body === null) { + this.error(`Response body is null for ${url}`); + } + + return response as Response & { body: NodeJS.ReadableStream }; + } + + /** + * Fetches metadata for a specific VS Code release target. + * @param target The target platform (e.g., 'cli-linux-x64'). + * @returns The target metadata. + */ + public async fetchMetadata(target: string): Promise { + const url = `https://update.code.visualstudio.com/api/versions/commit:${this.commit}/${target}/${this.quality}`; + + this.log(`Fetching metadata for ${target} from ${url}`); + const response = await this.fetchNoErrors(url); + + const result = await response.json() as ITargetMetadata; + if (result.url === undefined || result.sha256hash === undefined) { + this.error(`Invalid metadata response for ${target}: ${JSON.stringify(result)}`); + } + + this.log(`Fetched metadata for ${target}: ${JSON.stringify(result)}`); + return result; + } + + /** + * Downloads installer for specified VS Code release target. + * @param target The target platform (e.g., 'cli-linux-x64'). + * @returns The path to the downloaded file. + */ + public async downloadTarget(target: string): Promise { + const { url, sha256hash } = await this.fetchMetadata(target); + const filePath = path.join(this.createTempDir(), path.basename(url)); + + this.log(`Downloading ${url} to ${filePath}`); + const { body } = await this.fetchNoErrors(url); + + const stream = fs.createWriteStream(filePath); + await new Promise((resolve, reject) => { + body.on('error', reject); + stream.on('error', reject); + stream.on('finish', resolve); + body.pipe(stream); + }); + + this.log(`Downloaded ${url} to ${filePath}`); + this.validateSha256Hash(filePath, sha256hash); + + if (TestContext.authenticodeInclude.test(filePath) && os.platform() === 'win32') { + this.validateSignature(filePath); + } + + return filePath; + } + + /** + * Validates the SHA-256 hash of a file. + * @param filePath The path to the file to validate. + * @param expectedHash The expected SHA-256 hash in hexadecimal format. + */ + public validateSha256Hash(filePath: string, expectedHash: string) { + this.log(`Validating SHA256 hash for ${filePath}`); + + const buffer = fs.readFileSync(filePath); + const hash = createHash('sha256').update(buffer).digest('hex'); + + if (hash !== expectedHash) { + this.error(`Hash mismatch for ${filePath}: expected ${expectedHash}, got ${hash}`); + } + } + + /** + * Validates the Authenticode signature of a Windows executable. + * @param filePath The path to the file to validate. + */ + public validateSignature(filePath: string) { + this.log(`Validating Authenticode signature for ${filePath}`); + + const result = this.run('powershell', '-Command', `Get-AuthenticodeSignature "${filePath}" | Select-Object -ExpandProperty Status`); + if (result.error !== undefined) { + this.error(`Failed to run Get-AuthenticodeSignature: ${result.error.message}`); + } + + const status = result.stdout.trim(); + if (status !== 'Valid') { + this.error(`Authenticode signature is not valid for ${filePath}: ${status}`); + } + } + + /** + * Validates signatures for all executable files in the specified directory. + * @param dir The directory to scan for executable files. + */ + public validateAllSignatures(dir: string) { + const files = fs.readdirSync(dir, { withFileTypes: true }); + for (const file of files) { + const filePath = path.join(dir, file.name); + if (file.isDirectory()) { + this.validateAllSignatures(filePath); + } else if (TestContext.authenticodeInclude.test(file.name)) { + this.validateSignature(filePath); + } + } + } + + /** + * Downloads and unpacks the specified VS Code release target. + * @param target The target platform (e.g., 'cli-linux-x64'). + * @returns The path to the unpacked directory. + */ + public async downloadAndUnpack(target: string): Promise { + const filePath = await this.downloadTarget(target); + return this.unpackArchive(filePath); + } + + /** + * Unpacks a .zip or .tar.gz archive to a temporary directory. + * @param archivePath The path to the archive file. + * @returns The path to the temporary directory where the archive was unpacked. + */ + public unpackArchive(archivePath: string): string { + const dir = this.createTempDir(); + + this.log(`Unpacking ${archivePath} to ${dir}`); + this.runNoErrors('tar', '-xzf', archivePath, '-C', dir); + this.log(`Unpacked ${archivePath} to ${dir}`); + + return dir; + } + + /** + * Runs a command synchronously. + * @param command The command to run. + * @param args Optional arguments for the command. + * @returns The result of the spawnSync call. + */ + public run(command: string, ...args: string[]): SpawnSyncReturns { + this.log(`Running command: ${command} ${args.join(' ')}`); + return spawnSync(command, args, { encoding: 'utf-8' }) as SpawnSyncReturns; + } + + /** + * Runs a command synchronously and ensures it succeeds. + * @param command The command to run. + * @param args Optional arguments for the command. + * @returns The result of the spawnSync call. + */ + public runNoErrors(command: string, ...args: string[]): SpawnSyncReturns { + const result = this.run(command, ...args); + if (result.error !== undefined) { + this.error(`Failed to run command: ${result.error.message}`); + } + + if (result.status !== 0) { + this.error(`Command exited with code ${result.status}: ${result.stderr}`); + } + + return result; + } + + /** + * Installs a Microsoft Installer package silently. + * @param installerPath The path to the installer executable. + * @returns The path to the installed VS Code executable. + */ + public installWindowsApp(type: 'user' | 'system', installerPath: string): string { + this.log(`Installing ${installerPath} in silent mode`); + this.runNoErrors(installerPath, '/silent', '/mergetasks=!runcode'); + this.log(`Installed ${installerPath} successfully`); + + const varName = type === 'system' ? 'PROGRAMFILES' : 'LOCALAPPDATA'; + const parentDir = process.env[varName]; + if (parentDir === undefined) { + this.error(`Environment variable ${varName} is not defined`); + } + + let entryPoint: string; + switch (this.quality) { + case 'stable': + entryPoint = path.join(parentDir, 'Microsoft VS Code', 'Code.exe'); + break; + case 'insider': + entryPoint = path.join(parentDir, 'Microsoft VS Code Insiders', 'Code - Insiders.exe'); + break; + case 'exploration': + entryPoint = path.join(parentDir, 'Microsoft VS Code Exploration', 'Code - Exploration.exe'); + break; + } + + if (!fs.existsSync(entryPoint)) { + this.error(`Desktop entry point does not exist: ${entryPoint}`); + } + + this.log(`Installed VS Code executable at: ${entryPoint}`); + return entryPoint; + } + + /** + * Prepares a macOS .app bundle for execution by removing the quarantine attribute. + * @param bundleDir The directory containing the .app bundle. + * @returns The path to the VS Code executable. + */ + public installMacApp(bundleDir: string): string { + let appName: string; + switch (this.quality) { + case 'stable': + appName = 'Visual Studio Code.app'; + break; + case 'insider': + appName = 'Visual Studio Code - Insiders.app'; + break; + case 'exploration': + appName = 'Visual Studio Code - Exploration.app'; + break; + } + + const appPath = path.join(bundleDir, appName); + + this.log(`Removing quarantine attribute from ${appPath}`); + this.runNoErrors('xattr', '-rd', 'com.apple.quarantine', appPath); + this.log(`Removed quarantine attribute successfully`); + + const entryPoint = path.join(appPath, 'Contents/Resources/app/bin/code'); + if (!fs.existsSync(entryPoint)) { + this.error(`Desktop entry point does not exist: ${entryPoint}`); + } + + this.log(`VS Code executable at: ${entryPoint}`); + return entryPoint; + } + + /** + * Installs a Linux RPM package. + * @param packagePath The path to the RPM file. + * @returns The path to the installed VS Code executable. + */ + public installRpm(packagePath: string): string { + this.log(`Installing ${packagePath} using RPM package manager`); + this.runNoErrors('sudo', 'rpm', '-i', packagePath); + this.log(`Installed ${packagePath} successfully`); + + const entryPoint = this.getEntryPoint('desktop', '/usr/bin'); + this.log(`Installed VS Code executable at: ${entryPoint}`); + return entryPoint; + } + + /** + * Installs a Linux DEB package. + * @param packagePath The path to the DEB file. + * @returns The path to the installed VS Code executable. + */ + public installDeb(packagePath: string): string { + this.log(`Installing ${packagePath} using DEB package manager`); + this.runNoErrors('sudo', 'dpkg', '-i', packagePath); + this.log(`Installed ${packagePath} successfully`); + + const entryPoint = this.getEntryPoint('desktop', '/usr/bin'); + this.log(`Installed VS Code executable at: ${entryPoint}`); + return entryPoint; + } + + /** + * Installs a Linux Snap package. + * @param packagePath The path to the Snap file. + * @returns The path to the installed VS Code executable. + */ + public installSnap(packagePath: string): string { + this.log(`Installing ${packagePath} using Snap package manager`); + this.runNoErrors('sudo', 'snap', 'install', packagePath, '--classic', '--dangerous'); + this.log(`Installed ${packagePath} successfully`); + + const entryPoint = this.getEntryPoint('desktop', '/snap/bin'); + this.log(`Installed VS Code executable at: ${entryPoint}`); + return entryPoint; + } + + /** + * Returns the entry point executable for the VS Code CLI or Desktop installation in the specified directory. + * @param dir The directory of the VS Code installation. + * @returns The path to the entry point executable. + */ + public getEntryPoint(type: 'cli' | 'desktop', dir: string): string { + let suffix: string; + switch (this.quality) { + case 'stable': + suffix = type === 'cli' ? '' : ''; + break; + case 'insider': + suffix = type === 'cli' ? '-insiders' : ' - Insiders'; + break; + case 'exploration': + suffix = type === 'cli' ? '-exploration' : ' - Exploration'; + break; + } + + const extension = os.platform() === 'win32' ? '.exe' : ''; + const filePath = path.join(dir, `code${suffix}${extension}`); + if (!fs.existsSync(filePath)) { + this.error(`CLI entry point does not exist: ${filePath}`); + } + + return filePath; + } + + /** + * Returns the tunnel URL for the VS Code server including vscode-version parameter. + * @param baseUrl The base URL for the VS Code server. + * @returns The tunnel URL with vscode-version parameter. + */ + public getTunnelUrl(baseUrl: string): string { + const url = new URL(baseUrl); + url.searchParams.set('vscode-version', this.commit); + return url.toString(); + } +} diff --git a/test/sanity/src/desktop.test.ts b/test/sanity/src/desktop.test.ts new file mode 100644 index 000000000000..4c5cd544e8f1 --- /dev/null +++ b/test/sanity/src/desktop.test.ts @@ -0,0 +1,236 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import fs from 'fs'; +import path from 'path'; +import { _electron } from 'playwright'; +import { TestContext } from './context'; + +export function setup(context: TestContext) { + describe('Desktop', () => { + if (context.platform === 'darwin-x64') { + it('darwin', async () => { + const dir = await context.downloadAndUnpack('darwin'); + const entryPoint = context.installMacApp(dir); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'darwin-arm64') { + it('darwin-arm64', async () => { + const dir = await context.downloadAndUnpack('darwin-arm64'); + const entryPoint = context.installMacApp(dir); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform.startsWith('darwin-')) { + it('darwin-universal', async () => { + const dir = await context.downloadAndUnpack('darwin-universal'); + const entryPoint = context.installMacApp(dir); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'linux-arm64') { + it('linux-arm64', async () => { + const dir = await context.downloadAndUnpack('linux-arm64'); + const entryPoint = context.getEntryPoint('desktop', dir); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'linux-arm') { + it('linux-armhf', async () => { + const dir = await context.downloadAndUnpack('linux-armhf'); + const entryPoint = context.getEntryPoint('desktop', dir); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'linux-arm64') { + it('linux-deb-arm64', async () => { + const packagePath = await context.downloadTarget('linux-deb-arm64'); + const entryPoint = context.installDeb(packagePath); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'linux-arm') { + it('linux-deb-armhf', async () => { + const packagePath = await context.downloadTarget('linux-deb-armhf'); + const entryPoint = context.installDeb(packagePath); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'linux-x64') { + it('linux-deb-x64', async () => { + const packagePath = await context.downloadTarget('linux-deb-x64'); + const entryPoint = context.installDeb(packagePath); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'linux-arm64') { + it('linux-rpm-arm64', async () => { + const packagePath = await context.downloadTarget('linux-rpm-arm64'); + const entryPoint = context.installRpm(packagePath); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'linux-arm') { + it('linux-rpm-armhf', async () => { + const packagePath = await context.downloadTarget('linux-rpm-armhf'); + const entryPoint = context.installRpm(packagePath); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'linux-x64') { + it('linux-rpm-x64', async () => { + const packagePath = await context.downloadTarget('linux-rpm-x64'); + const entryPoint = context.installRpm(packagePath); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'linux-x64') { + it('linux-snap-x64', async () => { + const packagePath = await context.downloadTarget('linux-snap-x64'); + const entryPoint = context.installSnap(packagePath); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'linux-x64') { + it('linux-x64', async () => { + const dir = await context.downloadAndUnpack('linux-x64'); + const entryPoint = context.getEntryPoint('desktop', dir); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'win32-arm64') { + it('win32-arm64', async () => { + const packagePath = await context.downloadTarget('win32-arm64'); + context.validateSignature(packagePath); + const entryPoint = context.installWindowsApp('system', packagePath); + context.validateAllSignatures(path.dirname(entryPoint)); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'win32-arm64') { + it('win32-arm64-archive', async () => { + const dir = await context.downloadAndUnpack('win32-arm64-archive'); + context.validateAllSignatures(dir); + const entryPoint = context.getEntryPoint('desktop', dir); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'win32-arm64') { + it('win32-arm64-user', async () => { + const packagePath = await context.downloadTarget('win32-arm64-user'); + context.validateSignature(packagePath); + const entryPoint = context.installWindowsApp('user', packagePath); + context.validateAllSignatures(path.dirname(entryPoint)); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'win32-x64') { + it('win32-x64', async () => { + const packagePath = await context.downloadTarget('win32-x64'); + context.validateSignature(packagePath); + const entryPoint = context.installWindowsApp('system', packagePath); + context.validateAllSignatures(path.dirname(entryPoint)); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'win32-x64') { + it('win32-x64-archive', async () => { + const dir = await context.downloadAndUnpack('win32-x64-archive'); + context.validateAllSignatures(dir); + const entryPoint = context.getEntryPoint('desktop', dir); + await testDesktopApp(entryPoint); + }); + } + + if (context.platform === 'win32-x64') { + it('win32-x64-user', async () => { + const packagePath = await context.downloadTarget('win32-x64-user'); + context.validateSignature(packagePath); + const entryPoint = context.installWindowsApp('user', packagePath); + context.validateAllSignatures(path.dirname(entryPoint)); + await testDesktopApp(entryPoint); + }); + } + + async function testDesktopApp(executablePath: string) { + const extensionsDir = context.createTempDir(); + const dataDir = context.createTempDir(); + const workspaceDir = context.createTempDir(); + const args = ['--extensions-dir', extensionsDir, '--user-data-dir', dataDir, workspaceDir]; + + context.log(`Start VS Code: ${executablePath} with args: ${args.join(' ')}`); + const app = await _electron.launch({ executablePath, args }); + const window = await app.firstWindow(); + + context.log('Dismiss workspace trust dialog'); + await window.getByText('Yes, I trust the authors').click(); + + context.log('Focus Explorer view'); + await window.keyboard.press('Control+Shift+E'); + + context.log('Click New File button'); + await window.getByLabel('New File...').click(); + + context.log('Type file name'); + await window.locator('input').fill('helloWorld.txt'); + await window.keyboard.press('Enter'); + + context.log('Focus the code editor'); + await window.getByText(/Start typing/).focus(); + + context.log('Type some content into the file'); + await window.keyboard.type('Hello, World!'); + + context.log('Save the file'); + await window.keyboard.press('Control+S'); + + context.log('Open Extensions view'); + await window.keyboard.press('Control+Shift+X'); + await window.waitForSelector('.extension-list-item'); + + context.log('Type extension name to search for'); + await window.keyboard.type('GitHub Pull Requests'); + await new Promise(resolve => setTimeout(resolve, 2000)); + + context.log('Click Install on the first extension in the list'); + await window.locator('.extension-action:not(.disabled)', { hasText: /Install/ }).first().click(); + + context.log('Wait for extension to be installed'); + await window.locator('.extension-action:not(.disabled)', { hasText: /Uninstall/ }).waitFor(); + + context.log('Close the application'); + await app.close(); + + context.log('Verify file contents'); + const filePath = `${workspaceDir}/helloWorld.txt`; + const fileContents = fs.readFileSync(filePath, 'utf-8'); + assert.strictEqual(fileContents, 'Hello, World!'); + + context.log('Verify extension is installed'); + const extensions = fs.readdirSync(extensionsDir); + const hasExtension = extensions.some(ext => ext.startsWith('github.vscode-pull-request-github')); + assert.strictEqual(hasExtension, true); + } + }); +} diff --git a/test/sanity/src/index.ts b/test/sanity/src/index.ts new file mode 100644 index 000000000000..8ebc7ca9c898 --- /dev/null +++ b/test/sanity/src/index.ts @@ -0,0 +1,40 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import minimist from 'minimist'; +import Mocha, { MochaOptions } from 'mocha'; + +const options = minimist(process.argv.slice(2), { + string: ['fgrep', 'grep'], + boolean: ['help'], + alias: { fgrep: 'f', grep: 'g', help: 'h' }, +}); + +if (options.help) { + console.info('Usage: npm run sanity-test -- [options]'); + console.info('Options:'); + console.info(' --commit, -c The commit to test (required)'); + console.info(` --quality, -q The quality to test (required, "stable", "insider" or "exploration")`); + console.info(' --no-cleanup Do not cleanup downloaded files after each test'); + console.info(' --grep, -g Only run tests matching the given '); + console.info(' --fgrep, -f Only run tests containing the given '); + console.info(' --verbose, -v Enable verbose logging'); + console.info(' --help, -h Show this help message'); + process.exit(0); +} + +const mochaOptions: MochaOptions = { + color: true, + timeout: 5 * 60 * 1000, + slow: 3 * 60 * 1000, + grep: options.grep, + fgrep: options.fgrep, +}; + +const mocha = new Mocha(mochaOptions); +mocha.addFile(require.resolve('./main.js')); +mocha.run(failures => { + process.exitCode = failures > 0 ? 1 : 0; +}); diff --git a/test/sanity/src/main.ts b/test/sanity/src/main.ts new file mode 100644 index 000000000000..f6a2b7d60842 --- /dev/null +++ b/test/sanity/src/main.ts @@ -0,0 +1,41 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import minimist from 'minimist'; +import { setup as setupCliTests } from './cli.test'; +import { TestContext } from './context'; +import { setup as setupDesktopTests } from './desktop.test'; +import { setup as setupServerTests } from './server.test'; +import { setup as setupServerWebTests } from './serverWeb.test'; + +const options = minimist(process.argv.slice(2), { + string: ['commit', 'quality'], + boolean: ['cleanup', 'verbose'], + alias: { commit: 'c', quality: 'q', verbose: 'v' }, + default: { cleanup: true, verbose: false }, +}); + +if (!options.commit) { + throw new Error('--commit is required'); +} + +if (!options.quality) { + throw new Error('--quality is required'); +} + +const context = new TestContext(options.quality, options.commit, options.verbose); + +describe('VS Code Sanity Tests', () => { + if (options.cleanup) { + afterEach(() => { + context.cleanup(); + }); + } + + setupCliTests(context); + setupDesktopTests(context); + setupServerTests(context); + setupServerWebTests(context); +}); diff --git a/test/sanity/src/server.test.ts b/test/sanity/src/server.test.ts new file mode 100644 index 000000000000..a79eded4994e --- /dev/null +++ b/test/sanity/src/server.test.ts @@ -0,0 +1,66 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { TestContext } from './context'; + +export function setup(context: TestContext) { + describe('Server', () => { + if (context.platform === 'linux-arm64') { + it('server-alpine-arm64', async () => { + await context.downloadAndUnpack('server-alpine-arm64'); + }); + } + + if (context.platform === 'linux-x64') { + it('server-alpine-x64', async () => { + await context.downloadAndUnpack('server-linux-alpine'); + }); + } + + if (context.platform === 'darwin-arm64') { + it('server-darwin-arm64', async () => { + await context.downloadAndUnpack('server-darwin-arm64'); + }); + } + + if (context.platform === 'darwin-x64') { + it('server-darwin-x64', async () => { + await context.downloadAndUnpack('server-darwin'); + }); + } + + if (context.platform === 'linux-arm64') { + it('server-linux-arm64', async () => { + await context.downloadAndUnpack('server-linux-arm64'); + }); + } + + if (context.platform === 'linux-arm') { + it('server-linux-armhf', async () => { + await context.downloadAndUnpack('server-linux-armhf'); + }); + } + + if (context.platform === 'linux-x64') { + it('server-linux-x64', async () => { + await context.downloadAndUnpack('server-linux-x64'); + }); + } + + if (context.platform === 'win32-arm64') { + it('server-win32-arm64', async () => { + const dir = await context.downloadAndUnpack('server-win32-arm64'); + context.validateAllSignatures(dir); + }); + } + + if (context.platform === 'win32-x64') { + it('server-win32-x64', async () => { + const dir = await context.downloadAndUnpack('server-win32-x64'); + context.validateAllSignatures(dir); + }); + } + }); +} diff --git a/test/sanity/src/serverWeb.test.ts b/test/sanity/src/serverWeb.test.ts new file mode 100644 index 000000000000..369fe90eb704 --- /dev/null +++ b/test/sanity/src/serverWeb.test.ts @@ -0,0 +1,66 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { TestContext } from './context'; + +export function setup(context: TestContext) { + describe('Server Web', () => { + if (context.platform === 'linux-arm64') { + it('server-alpine-arm64-web', async () => { + await context.downloadAndUnpack('server-alpine-arm64-web'); + }); + } + + if (context.platform === 'linux-x64') { + it('server-alpine-x64-web', async () => { + await context.downloadAndUnpack('server-linux-alpine-web'); + }); + } + + if (context.platform === 'darwin-arm64') { + it('server-darwin-arm64-web', async () => { + await context.downloadAndUnpack('server-darwin-arm64-web'); + }); + } + + if (context.platform === 'darwin-x64') { + it('server-darwin-x64-web', async () => { + await context.downloadAndUnpack('server-darwin-web'); + }); + } + + if (context.platform === 'linux-arm64') { + it('server-linux-arm64-web', async () => { + await context.downloadAndUnpack('server-linux-arm64-web'); + }); + } + + if (context.platform === 'linux-arm') { + it('server-linux-armhf-web', async () => { + await context.downloadAndUnpack('server-linux-armhf-web'); + }); + } + + if (context.platform === 'linux-x64') { + it('server-linux-x64-web', async () => { + await context.downloadAndUnpack('server-linux-x64-web'); + }); + } + + if (context.platform === 'win32-arm64') { + it('server-win32-arm64-web', async () => { + const dir = await context.downloadAndUnpack('server-win32-arm64-web'); + context.validateAllSignatures(dir); + }); + } + + if (context.platform === 'win32-x64') { + it('server-win32-x64-web', async () => { + const dir = await context.downloadAndUnpack('server-win32-x64-web'); + context.validateAllSignatures(dir); + }); + } + }); +} diff --git a/test/sanity/tsconfig.json b/test/sanity/tsconfig.json new file mode 100644 index 000000000000..1c04af2bba29 --- /dev/null +++ b/test/sanity/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "module": "commonjs", + "noImplicitAny": true, + "removeComments": false, + "preserveConstEnums": true, + "target": "es2024", + "strict": true, + "noUnusedParameters": false, + "noUnusedLocals": true, + "outDir": "out", + "sourceMap": true, + "skipLibCheck": true, + "esModuleInterop": true, + "lib": [ + "esnext", // for #201187 + "dom" + ] + }, + "exclude": [ + "node_modules" + ] +}