refactor: use sysroots on linux prod pipeline (#192513)

* ci: use sysroots on linux prod pipeline to build native modules

* ci: rebuild after electron update

build/linux/debian/install-sysroot.ts

@@ -3,18 +3,46 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { spawnSync } from 'child_process';
-import { createHash } from 'crypto';
+import { spawnSync, execSync } from 'child_process';
 import { tmpdir } from 'os';
 import * as fs from 'fs';
 import * as https from 'https';
 import * as path from 'path';
+import { createHash } from 'crypto';
 import { DebianArchString } from './types';
-import * as util from '../../lib/util';
+import * as ansiColors from 'ansi-colors';
 
 // Based on https://source.chromium.org/chromium/chromium/src/+/main:build/linux/sysroot_scripts/install-sysroot.py.
 const URL_PREFIX = 'https://msftelectron.blob.core.windows.net';
 const URL_PATH = 'sysroots/toolchain';
+const REPO_ROOT = path.dirname(path.dirname(path.dirname(__dirname)));
+
+const ghApiHeaders: Record<string, string> = {
+	Accept: 'application/vnd.github.v3+json',
+	'User-Agent': 'VSCode Build',
+};
+
+if (process.env.GITHUB_TOKEN) {
+	ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
+}
+
+const ghDownloadHeaders = {
+	...ghApiHeaders,
+	Accept: 'application/octet-stream',
+};
+
+interface IFetchOptions {
+	assetName: string;
+	checksumSha256?: string;
+	dest: string;
+}
+
+function getElectronVersion(): Record<string, string> {
+	const yarnrc = fs.readFileSync(path.join(REPO_ROOT, '.yarnrc'), 'utf8');
+	const electronVersion = /^target "(.*)"$/m.exec(yarnrc)![1];
+	const msBuildId = /^ms_build_id "(.*)"$/m.exec(yarnrc)![1];
+	return { electronVersion, msBuildId };
+}
 
 function getSha(filename: fs.PathLike): string {
 	const hash = createHash('sha1');
@@ -31,14 +59,121 @@ function getSha(filename: fs.PathLike): string {
 	return hash.digest('hex');
 }
 
+function getVSCodeSysrootChecksum(expectedName: string) {
+	const checksums = fs.readFileSync(path.join(REPO_ROOT, 'build', 'checksums', 'vscode-sysroot.txt'), 'utf8');
+	for (const line of checksums.split('\n')) {
+		const [checksum, name] = line.split(/\s+/);
+		if (name === expectedName) {
+			return checksum;
+		}
+	}
+	return undefined;
+}
+
+/*
+ * Do not use the fetch implementation from build/lib/fetch as it relies on vinyl streams
+ * and vinyl-fs breaks the symlinks in the compiler toolchain sysroot. We use the native
+ * tar implementation for that reason.
+ */
+async function fetchUrl(options: IFetchOptions, retries = 10, retryDelay = 1000): Promise<undefined> {
+	try {
+		const controller = new AbortController();
+		const timeout = setTimeout(() => controller.abort(), 30 * 1000);
+		const version = '20231122-245579';
+		try {
+			const response = await fetch(`https://api.github.com/repos/Microsoft/vscode-linux-build-agent/releases/tags/v${version}`, {
+				headers: ghApiHeaders,
+				signal: controller.signal as any /* Typings issue with lib.dom.d.ts */
+			});
+			if (response.ok && (response.status >= 200 && response.status < 300)) {
+				console.log(`Fetch completed: Status ${response.status}.`);
+				const contents = Buffer.from(await response.arrayBuffer());
+				const asset = JSON.parse(contents.toString()).assets.find((a: { name: string }) => a.name === options.assetName);
+				if (!asset) {
+					throw new Error(`Could not find asset in release of Microsoft/vscode-linux-build-agent @ ${version}`);
+				}
+				console.log(`Found asset ${options.assetName} @ ${asset.url}.`);
+				const assetResponse = await fetch(asset.url, {
+					headers: ghDownloadHeaders
+				});
+				if (assetResponse.ok && (assetResponse.status >= 200 && assetResponse.status < 300)) {
+					const assetContents = Buffer.from(await assetResponse.arrayBuffer());
+					console.log(`Fetched response body buffer: ${ansiColors.magenta(`${(assetContents as Buffer).byteLength} bytes`)}`);
+					if (options.checksumSha256) {
+						const actualSHA256Checksum = createHash('sha256').update(assetContents).digest('hex');
+						if (actualSHA256Checksum !== options.checksumSha256) {
+							throw new Error(`Checksum mismatch for ${ansiColors.cyan(asset.url)} (expected ${options.checksumSha256}, actual ${actualSHA256Checksum}))`);
+						}
+					}
+					console.log(`Verified SHA256 checksums match for ${ansiColors.cyan(asset.url)}`);
+					const tarCommand = `tar -xz -C ${options.dest}`;
+					execSync(tarCommand, { input: assetContents });
+					console.log(`Fetch complete!`);
+					return;
+				}
+				throw new Error(`Request ${ansiColors.magenta(asset.url)} failed with status code: ${assetResponse.status}`);
+			}
+			throw new Error(`Request ${ansiColors.magenta('https://api.github.com')} failed with status code: ${response.status}`);
+		} finally {
+			clearTimeout(timeout);
+		}
+	} catch (e) {
+		if (retries > 0) {
+			console.log(`Fetching failed: ${e}`);
+			await new Promise(resolve => setTimeout(resolve, retryDelay));
+			return fetchUrl(options, retries - 1, retryDelay);
+		}
+		throw e;
+	}
+}
+
 type SysrootDictEntry = {
 	Sha1Sum: string;
 	SysrootDir: string;
 	Tarball: string;
 };
 
-export async function getSysroot(arch: DebianArchString): Promise<string> {
-	const sysrootJSONUrl = `https://raw.githubusercontent.com/electron/electron/v${util.getElectronVersion().electronVersion}/script/sysroots.json`;
+export async function getVSCodeSysroot(arch: DebianArchString): Promise<string> {
+	let expectedName: string;
+	let triple: string;
+	switch (arch) {
+		case 'amd64':
+			expectedName = `x86_64-linux-gnu.tar.gz`;
+			triple = 'x86_64-linux-gnu';
+			break;
+		case 'arm64':
+			expectedName = `aarch64-linux-gnu.tar.gz`;
+			triple = 'aarch64-linux-gnu';
+			break;
+		case 'armhf':
+			expectedName = `arm-rpi-linux-gnueabihf.tar.gz`;
+			triple = 'arm-rpi-linux-gnueabihf';
+			break;
+	}
+	const checksumSha256 = getVSCodeSysrootChecksum(expectedName);
+	if (!checksumSha256) {
+		throw new Error(`Could not find checksum for ${expectedName}`);
+	}
+	const sysroot = process.env['VSCODE_SYSROOT_DIR'] ?? path.join(tmpdir(), `vscode-${arch}-sysroot`);
+	const stamp = path.join(sysroot, '.stamp');
+	const result = `${sysroot}/${triple}/${triple}/sysroot`;
+	if (fs.existsSync(stamp) && fs.readFileSync(stamp).toString() === expectedName) {
+		return result;
+	}
+	console.log(`Installing ${arch} root image: ${sysroot}`);
+	fs.rmSync(sysroot, { recursive: true, force: true });
+	fs.mkdirSync(sysroot);
+	await fetchUrl({
+		checksumSha256,
+		assetName: expectedName,
+		dest: sysroot
+	});
+	fs.writeFileSync(stamp, expectedName);
+	return result;
+}
+
+export async function getChromiumSysroot(arch: DebianArchString): Promise<string> {
+	const sysrootJSONUrl = `https://raw.githubusercontent.com/electron/electron/v${getElectronVersion().electronVersion}/script/sysroots.json`;
 	const sysrootDictLocation = `${tmpdir()}/sysroots.json`;
 	const result = spawnSync('curl', [sysrootJSONUrl, '-o', sysrootDictLocation]);
 	if (result.status !== 0) {