Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-04-25 11:08:51 +01:00
Code Issues Packages Projects Releases Wiki Activity

refactor: use sysroots on linux prod pipeline (#192513)

Browse Source 
* ci: use sysroots on linux prod pipeline to build native modules

* ci: rebuild after electron update
This commit is contained in:
Robo
2023-12-12 19:39:00 +09:00
committed by GitHub
parent 9c679f1e07
commit e7b5076dd7
19 changed files with 597 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files

137
build/linux/debian/install-sysroot.js
View File

@@ -4,17 +4,35 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.getSysroot = void 0;
exports.getChromiumSysroot = exports.getVSCodeSysroot = void 0;
const child_process_1 = require("child_process");
const crypto_1 = require("crypto");
const os_1 = require("os");
const fs = require("fs");
const https = require("https");
const path = require("path");
const util = require("../../lib/util");
const crypto_1 = require("crypto");
const ansiColors = require("ansi-colors");
// Based on https://source.chromium.org/chromium/chromium/src/+/main:build/linux/sysroot_scripts/install-sysroot.py.
const URL_PREFIX = 'https://msftelectron.blob.core.windows.net';
const URL_PATH = 'sysroots/toolchain';
const REPO_ROOT = path.dirname(path.dirname(path.dirname(__dirname)));
const ghApiHeaders = {
Accept: 'application/vnd.github.v3+json',
'User-Agent': 'VSCode Build',
};
if (process.env.GITHUB_TOKEN) {
ghApiHeaders.Authorization = 'Basic ' + Buffer.from(process.env.GITHUB_TOKEN).toString('base64');
}
const ghDownloadHeaders = {
...ghApiHeaders,
Accept: 'application/octet-stream',
};
function getElectronVersion() {
const yarnrc = fs.readFileSync(path.join(REPO_ROOT, '.yarnrc'), 'utf8');
const electronVersion = /^target "(.*)"$/m.exec(yarnrc)[1];
const msBuildId = /^ms_build_id "(.*)"$/m.exec(yarnrc)[1];
return { electronVersion, msBuildId };
}
function getSha(filename) {
const hash = (0, crypto_1.createHash)('sha1');
// Read file 1 MB at a time
@@ -29,8 +47,115 @@ function getSha(filename) {
hash.update(buffer.slice(0, bytesRead));
return hash.digest('hex');
}
async function getSysroot(arch) {
const sysrootJSONUrl = `https://raw.githubusercontent.com/electron/electron/v${util.getElectronVersion().electronVersion}/script/sysroots.json`;
function getVSCodeSysrootChecksum(expectedName) {
const checksums = fs.readFileSync(path.join(REPO_ROOT, 'build', 'checksums', 'vscode-sysroot.txt'), 'utf8');
for (const line of checksums.split('\n')) {
const [checksum, name] = line.split(/\s+/);
if (name === expectedName) {
return checksum;
}
}
return undefined;
}
/*
* Do not use the fetch implementation from build/lib/fetch as it relies on vinyl streams
* and vinyl-fs breaks the symlinks in the compiler toolchain sysroot. We use the native
* tar implementation for that reason.
*/
async function fetchUrl(options, retries = 10, retryDelay = 1000) {
try {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 30 * 1000);
const version = '20231122-245579';
try {
const response = await fetch(`https://api.github.com/repos/Microsoft/vscode-linux-build-agent/releases/tags/v${version}`, {
headers: ghApiHeaders,
signal: controller.signal /* Typings issue with lib.dom.d.ts */
});
if (response.ok && (response.status >= 200 && response.status < 300)) {
console.log(`Fetch completed: Status ${response.status}.`);
const contents = Buffer.from(await response.arrayBuffer());
const asset = JSON.parse(contents.toString()).assets.find((a) => a.name === options.assetName);
if (!asset) {
throw new Error(`Could not find asset in release of Microsoft/vscode-linux-build-agent @ ${version}`);
}
console.log(`Found asset ${options.assetName} @ ${asset.url}.`);
const assetResponse = await fetch(asset.url, {
headers: ghDownloadHeaders
});
if (assetResponse.ok && (assetResponse.status >= 200 && assetResponse.status < 300)) {
const assetContents = Buffer.from(await assetResponse.arrayBuffer());
console.log(`Fetched response body buffer: ${ansiColors.magenta(`${assetContents.byteLength} bytes`)}`);
if (options.checksumSha256) {
const actualSHA256Checksum = (0, crypto_1.createHash)('sha256').update(assetContents).digest('hex');
if (actualSHA256Checksum !== options.checksumSha256) {
throw new Error(`Checksum mismatch for ${ansiColors.cyan(asset.url)} (expected ${options.checksumSha256}, actual ${actualSHA256Checksum}))`);
}
}
console.log(`Verified SHA256 checksums match for ${ansiColors.cyan(asset.url)}`);
const tarCommand = `tar -xz -C ${options.dest}`;
(0, child_process_1.execSync)(tarCommand, { input: assetContents });
console.log(`Fetch complete!`);
return;
}
throw new Error(`Request ${ansiColors.magenta(asset.url)} failed with status code: ${assetResponse.status}`);
}
throw new Error(`Request ${ansiColors.magenta('https://api.github.com')} failed with status code: ${response.status}`);
}
finally {
clearTimeout(timeout);
}
}
catch (e) {
if (retries > 0) {
console.log(`Fetching failed: ${e}`);
await new Promise(resolve => setTimeout(resolve, retryDelay));
return fetchUrl(options, retries - 1, retryDelay);
}
throw e;
}
}
async function getVSCodeSysroot(arch) {
let expectedName;
let triple;
switch (arch) {
case 'amd64':
expectedName = `x86_64-linux-gnu.tar.gz`;
triple = 'x86_64-linux-gnu';
break;
case 'arm64':
expectedName = `aarch64-linux-gnu.tar.gz`;
triple = 'aarch64-linux-gnu';
break;
case 'armhf':
expectedName = `arm-rpi-linux-gnueabihf.tar.gz`;
triple = 'arm-rpi-linux-gnueabihf';
break;
}
const checksumSha256 = getVSCodeSysrootChecksum(expectedName);
if (!checksumSha256) {
throw new Error(`Could not find checksum for ${expectedName}`);
}
const sysroot = process.env['VSCODE_SYSROOT_DIR'] ?? path.join((0, os_1.tmpdir)(), `vscode-${arch}-sysroot`);
const stamp = path.join(sysroot, '.stamp');
const result = `${sysroot}/${triple}/${triple}/sysroot`;
if (fs.existsSync(stamp) && fs.readFileSync(stamp).toString() === expectedName) {
return result;
}
console.log(`Installing ${arch} root image: ${sysroot}`);
fs.rmSync(sysroot, { recursive: true, force: true });
fs.mkdirSync(sysroot);
await fetchUrl({
checksumSha256,
assetName: expectedName,
dest: sysroot
});
fs.writeFileSync(stamp, expectedName);
return result;
}
exports.getVSCodeSysroot = getVSCodeSysroot;
async function getChromiumSysroot(arch) {
const sysrootJSONUrl = `https://raw.githubusercontent.com/electron/electron/v${getElectronVersion().electronVersion}/script/sysroots.json`;
const sysrootDictLocation = `${(0, os_1.tmpdir)()}/sysroots.json`;
const result = (0, child_process_1.spawnSync)('curl', [sysrootJSONUrl, '-o', sysrootDictLocation]);
if (result.status !== 0) {
@@ -86,5 +211,5 @@ async function getSysroot(arch) {
fs.writeFileSync(stamp, url);
return sysroot;
}
exports.getSysroot = getSysroot;
exports.getChromiumSysroot = getChromiumSysroot;
//# sourceMappingURL=install-sysroot.js.map