Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-04-25 02:58:56 +01:00
Code Issues Packages Projects Releases Wiki Activity

Refactor package dependencies generators (#157845)

Browse Source
This commit is contained in:
Raymond Zhao
2022-08-11 17:09:00 -07:00
committed by GitHub
parent 7b38f89422
commit 17c169a7b7
18 changed files with 280 additions and 366 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
build/linux/debian/dependencies-generator.js → build/linux/debian/calculate-deps.js
View File

@@ -1,70 +1,23 @@
"use strict";
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
Object.defineProperty(exports, "__esModule", { value: true });
exports.getDependencies = void 0;
exports.generatePackageDeps = void 0;
const child_process_1 = require("child_process");
const fs_1 = require("fs");
const os_1 = require("os");
const path = require("path");
const dep_lists_1 = require("./dep-lists");
const manifests = require("../../../cgmanifest.json");
// A flag that can easily be toggled.
// Make sure to compile the build directory after toggling the value.
// If false, we warn about new dependencies if they show up
// while running the Debian prepare package task for a release.
// If true, we fail the build if there are new dependencies found during that task.
// The reference dependencies, which one has to update when the new dependencies
// are valid, are in dep-lists.ts
const FAIL_BUILD_FOR_NEW_DEPENDENCIES = true;
function getDependencies(buildDir, applicationName, arch, sysroot) {
// Get the files for which we want to find dependencies.
const nativeModulesPath = path.join(buildDir, 'resources', 'app', 'node_modules.asar.unpacked');
const findResult = (0, child_process_1.spawnSync)('find', [nativeModulesPath, '-name', '*.node']);
if (findResult.status) {
console.error('Error finding files:');
console.error(findResult.stderr.toString());
return [];
}
const files = findResult.stdout.toString().trimEnd().split('\n');
const appPath = path.join(buildDir, applicationName);
files.push(appPath);
// Add chrome sandbox and crashpad handler.
files.push(path.join(buildDir, 'chrome-sandbox'));
files.push(path.join(buildDir, 'chrome_crashpad_handler'));
// Generate the dependencies.
const dependencies = files.map((file) => calculatePackageDeps(file, arch, sysroot));
// Add additional dependencies.
const dep_lists_1 = require("./dep-lists");
function generatePackageDeps(files, arch, sysroot) {
const dependencies = files.map(file => calculatePackageDeps(file, arch, sysroot));
const additionalDepsSet = new Set(dep_lists_1.additionalDeps);
dependencies.push(additionalDepsSet);
// Merge all the dependencies.
const mergedDependencies = mergePackageDeps(dependencies);
let sortedDependencies = [];
for (const dependency of mergedDependencies) {
sortedDependencies.push(dependency);
}
sortedDependencies.sort();
// Exclude bundled dependencies
sortedDependencies = sortedDependencies.filter(dependency => {
return !dep_lists_1.bundledDeps.some(bundledDep => dependency.startsWith(bundledDep));
});
const referenceGeneratedDeps = dep_lists_1.referenceGeneratedDepsByArch[arch];
if (JSON.stringify(sortedDependencies) !== JSON.stringify(referenceGeneratedDeps)) {
const failMessage = 'The dependencies list has changed.'
+ '\nOld:\n' + referenceGeneratedDeps.join('\n')
+ '\nNew:\n' + sortedDependencies.join('\n');
if (FAIL_BUILD_FOR_NEW_DEPENDENCIES) {
throw new Error(failMessage);
}
else {
console.warn(failMessage);
}
}
return sortedDependencies;
return dependencies;
}
exports.getDependencies = getDependencies;
exports.generatePackageDeps = generatePackageDeps;
// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/calculate_package_deps.py.
function calculatePackageDeps(binaryPath, arch, sysroot) {
try {
@@ -97,8 +50,6 @@ function calculatePackageDeps(binaryPath, arch, sysroot) {
case 'arm64':
cmd.push(`-l${sysroot}/usr/lib/aarch64-linux-gnu`, `-l${sysroot}/lib/aarch64-linux-gnu`);
break;
default:
throw new Error('Unsupported architecture ' + arch);
}
cmd.push(`-l${sysroot}/usr/lib`);
cmd.push('-O', '-e', path.resolve(binaryPath));
@@ -117,17 +68,3 @@ function calculatePackageDeps(binaryPath, arch, sysroot) {
const requires = new Set(depsStr.split(', ').sort());
return requires;
}
// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/rpm/merge_package_deps.py.
function mergePackageDeps(inputDeps) {
// For now, see if directly appending the dependencies helps.
const requires = new Set();
for (const depSet of inputDeps) {
for (const dep of depSet) {
const trimmedDependency = dep.trim();
if (trimmedDependency.length && !trimmedDependency.startsWith('#')) {
requires.add(trimmedDependency);
}
}
}
return requires;
}

75
build/linux/debian/calculate-deps.ts Normal file
View File

@@ -0,0 +1,75 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { spawnSync } from 'child_process';
import { constants, statSync } from 'fs';
import { tmpdir } from 'os';
import path = require('path');
import * as manifests from '../../../cgmanifest.json';
import { additionalDeps } from './dep-lists';
import { DebianArchString } from './types';
export function generatePackageDeps(files: string[], arch: DebianArchString, sysroot: string): Set<string>[] {
const dependencies: Set<string>[] = files.map(file => calculatePackageDeps(file, arch, sysroot));
const additionalDepsSet = new Set(additionalDeps);
dependencies.push(additionalDepsSet);
return dependencies;
}
// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/calculate_package_deps.py.
function calculatePackageDeps(binaryPath: string, arch: DebianArchString, sysroot: string): Set<string> {
try {
if (!(statSync(binaryPath).mode & constants.S_IXUSR)) {
throw new Error(`Binary ${binaryPath} needs to have an executable bit set.`);
}
} catch (e) {
// The package might not exist. Don't re-throw the error here.
console.error('Tried to stat ' + binaryPath + ' but failed.');
}
// Get the Chromium dpkg-shlibdeps file.
const chromiumManifest = manifests.registrations.filter(registration => {
return registration.component.type === 'git' && registration.component.git!.name === 'chromium';
});
const dpkgShlibdepsUrl = `https://raw.githubusercontent.com/chromium/chromium/${chromiumManifest[0].version}/third_party/dpkg-shlibdeps/dpkg-shlibdeps.pl`;
const dpkgShlibdepsScriptLocation = `${tmpdir()}/dpkg-shlibdeps.pl`;
const result = spawnSync('curl', [dpkgShlibdepsUrl, '-o', dpkgShlibdepsScriptLocation]);
if (result.status !== 0) {
throw new Error('Cannot retrieve dpkg-shlibdeps. Stderr:\n' + result.stderr);
}
const cmd = [dpkgShlibdepsScriptLocation, '--ignore-weak-undefined'];
switch (arch) {
case 'amd64':
cmd.push(`-l${sysroot}/usr/lib/x86_64-linux-gnu`,
`-l${sysroot}/lib/x86_64-linux-gnu`);
break;
case 'armhf':
cmd.push(`-l${sysroot}/usr/lib/arm-linux-gnueabihf`,
`-l${sysroot}/lib/arm-linux-gnueabihf`);
break;
case 'arm64':
cmd.push(`-l${sysroot}/usr/lib/aarch64-linux-gnu`,
`-l${sysroot}/lib/aarch64-linux-gnu`);
break;
}
cmd.push(`-l${sysroot}/usr/lib`);
cmd.push('-O', '-e', path.resolve(binaryPath));
const dpkgShlibdepsResult = spawnSync('perl', cmd, { cwd: sysroot });
if (dpkgShlibdepsResult.status !== 0) {
throw new Error(`dpkg-shlibdeps failed with exit code ${dpkgShlibdepsResult.status}. stderr:\n${dpkgShlibdepsResult.stderr} `);
}
const shlibsDependsPrefix = 'shlibs:Depends=';
const requiresList = dpkgShlibdepsResult.stdout.toString('utf-8').trimEnd().split('\n');
let depsStr = '';
for (const line of requiresList) {
if (line.startsWith(shlibsDependsPrefix)) {
depsStr = line.substring(shlibsDependsPrefix.length);
}
}
const requires = new Set(depsStr.split(', ').sort());
return requires;
}

14
build/linux/debian/dep-lists.js
View File

@@ -4,7 +4,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.referenceGeneratedDepsByArch = exports.bundledDeps = exports.recommendedDeps = exports.additionalDeps = void 0;
exports.referenceGeneratedDepsByArch = exports.recommendedDeps = exports.additionalDeps = void 0;
// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/additional_deps
// Additional dependencies not in the dpkg-shlibdeps output.
exports.additionalDeps = [
@@ -20,18 +20,6 @@ exports.additionalDeps = [
exports.recommendedDeps = [
'libvulkan1' // Move to additionalDeps once support for Trusty and Jessie are dropped.
];
// Based on https://source.chromium.org/chromium/chromium/src/+/refs/tags/98.0.4758.109:chrome/installer/linux/BUILD.gn;l=64-80
// and the Linux Archive build
// Shared library dependencies that we already bundle.
exports.bundledDeps = [
'libEGL.so',
'libGLESv2.so',
'libvulkan.so.1',
'swiftshader_libEGL.so',
'swiftshader_libGLESv2.so',
'libvk_swiftshader.so',
'libffmpeg.so'
];
exports.referenceGeneratedDepsByArch = {
'amd64': [
'ca-certificates',

13
build/linux/debian/dep-lists.ts
View File

@@ -20,19 +20,6 @@ export const recommendedDeps = [
'libvulkan1' // Move to additionalDeps once support for Trusty and Jessie are dropped.
];
// Based on https://source.chromium.org/chromium/chromium/src/+/refs/tags/98.0.4758.109:chrome/installer/linux/BUILD.gn;l=64-80
// and the Linux Archive build
// Shared library dependencies that we already bundle.
export const bundledDeps = [
'libEGL.so',
'libGLESv2.so',
'libvulkan.so.1',
'swiftshader_libEGL.so',
'swiftshader_libGLESv2.so',
'libvk_swiftshader.so',
'libffmpeg.so'
];
export const referenceGeneratedDepsByArch = {
'amd64': [
'ca-certificates',

149
build/linux/debian/dependencies-generator.ts
View File

@@ -1,149 +0,0 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
'use strict';
import { spawnSync } from 'child_process';
import { constants, statSync } from 'fs';
import { tmpdir } from 'os';
import path = require('path');
import { additionalDeps, bundledDeps, referenceGeneratedDepsByArch } from './dep-lists';
import { ArchString } from './types';
import * as manifests from '../../../cgmanifest.json';
// A flag that can easily be toggled.
// Make sure to compile the build directory after toggling the value.
// If false, we warn about new dependencies if they show up
// while running the Debian prepare package task for a release.
// If true, we fail the build if there are new dependencies found during that task.
// The reference dependencies, which one has to update when the new dependencies
// are valid, are in dep-lists.ts
const FAIL_BUILD_FOR_NEW_DEPENDENCIES: boolean = true;
export function getDependencies(buildDir: string, applicationName: string, arch: ArchString, sysroot: string): string[] {
// Get the files for which we want to find dependencies.
const nativeModulesPath = path.join(buildDir, 'resources', 'app', 'node_modules.asar.unpacked');
const findResult = spawnSync('find', [nativeModulesPath, '-name', '*.node']);
if (findResult.status) {
console.error('Error finding files:');
console.error(findResult.stderr.toString());
return [];
}
const files = findResult.stdout.toString().trimEnd().split('\n');
const appPath = path.join(buildDir, applicationName);
files.push(appPath);
// Add chrome sandbox and crashpad handler.
files.push(path.join(buildDir, 'chrome-sandbox'));
files.push(path.join(buildDir, 'chrome_crashpad_handler'));
// Generate the dependencies.
const dependencies: Set<string>[] = files.map((file) => calculatePackageDeps(file, arch, sysroot));
// Add additional dependencies.
const additionalDepsSet = new Set(additionalDeps);
dependencies.push(additionalDepsSet);
// Merge all the dependencies.
const mergedDependencies = mergePackageDeps(dependencies);
let sortedDependencies: string[] = [];
for (const dependency of mergedDependencies) {
sortedDependencies.push(dependency);
}
sortedDependencies.sort();
// Exclude bundled dependencies
sortedDependencies = sortedDependencies.filter(dependency => {
return !bundledDeps.some(bundledDep => dependency.startsWith(bundledDep));
});
const referenceGeneratedDeps = referenceGeneratedDepsByArch[arch];
if (JSON.stringify(sortedDependencies) !== JSON.stringify(referenceGeneratedDeps)) {
const failMessage = 'The dependencies list has changed.'
+ '\nOld:\n' + referenceGeneratedDeps.join('\n')
+ '\nNew:\n' + sortedDependencies.join('\n');
if (FAIL_BUILD_FOR_NEW_DEPENDENCIES) {
throw new Error(failMessage);
} else {
console.warn(failMessage);
}
}
return sortedDependencies;
}
// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/calculate_package_deps.py.
function calculatePackageDeps(binaryPath: string, arch: ArchString, sysroot: string): Set<string> {
try {
if (!(statSync(binaryPath).mode & constants.S_IXUSR)) {
throw new Error(`Binary ${binaryPath} needs to have an executable bit set.`);
}
} catch (e) {
// The package might not exist. Don't re-throw the error here.
console.error('Tried to stat ' + binaryPath + ' but failed.');
}
// Get the Chromium dpkg-shlibdeps file.
const chromiumManifest = manifests.registrations.filter(registration => {
return registration.component.type === 'git' && registration.component.git!.name === 'chromium';
});
const dpkgShlibdepsUrl = `https://raw.githubusercontent.com/chromium/chromium/${chromiumManifest[0].version}/third_party/dpkg-shlibdeps/dpkg-shlibdeps.pl`;
const dpkgShlibdepsScriptLocation = `${tmpdir()}/dpkg-shlibdeps.pl`;
const result = spawnSync('curl', [dpkgShlibdepsUrl, '-o', dpkgShlibdepsScriptLocation]);
if (result.status !== 0) {
throw new Error('Cannot retrieve dpkg-shlibdeps. Stderr:\n' + result.stderr);
}
const cmd = [dpkgShlibdepsScriptLocation, '--ignore-weak-undefined'];
switch (arch) {
case 'amd64':
cmd.push(`-l${sysroot}/usr/lib/x86_64-linux-gnu`,
`-l${sysroot}/lib/x86_64-linux-gnu`);
break;
case 'armhf':
cmd.push(`-l${sysroot}/usr/lib/arm-linux-gnueabihf`,
`-l${sysroot}/lib/arm-linux-gnueabihf`);
break;
case 'arm64':
cmd.push(`-l${sysroot}/usr/lib/aarch64-linux-gnu`,
`-l${sysroot}/lib/aarch64-linux-gnu`);
break;
default:
throw new Error('Unsupported architecture ' + arch);
}
cmd.push(`-l${sysroot}/usr/lib`);
cmd.push('-O', '-e', path.resolve(binaryPath));
const dpkgShlibdepsResult = spawnSync('perl', cmd, { cwd: sysroot });
if (dpkgShlibdepsResult.status !== 0) {
throw new Error(`dpkg-shlibdeps failed with exit code ${dpkgShlibdepsResult.status}. stderr:\n${dpkgShlibdepsResult.stderr} `);
}
const shlibsDependsPrefix = 'shlibs:Depends=';
const requiresList = dpkgShlibdepsResult.stdout.toString('utf-8').trimEnd().split('\n');
let depsStr = '';
for (const line of requiresList) {
if (line.startsWith(shlibsDependsPrefix)) {
depsStr = line.substring(shlibsDependsPrefix.length);
}
}
const requires = new Set(depsStr.split(', ').sort());
return requires;
}
// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/rpm/merge_package_deps.py.
function mergePackageDeps(inputDeps: Set<string>[]): Set<string> {
// For now, see if directly appending the dependencies helps.
const requires = new Set<string>();
for (const depSet of inputDeps) {
for (const dep of depSet) {
const trimmedDependency = dep.trim();
if (trimmedDependency.length && !trimmedDependency.startsWith('#')) {
requires.add(trimmedDependency);
}
}
}
return requires;
}

6
build/linux/debian/install-sysroot.js
View File

@@ -54,14 +54,13 @@ async function getSysroot(arch) {
console.log(`Downloading ${url}`);
let downloadSuccess = false;
for (let i = 0; i < 3 && !downloadSuccess; i++) {
fs.writeFileSync(tarball, '');
await new Promise((c) => {
https.get(url, (res) => {
const chunks = [];
res.on('data', (chunk) => {
chunks.push(chunk);
fs.appendFileSync(tarball, chunk);
});
res.on('end', () => {
fs.writeFileSync(tarball, Buffer.concat(chunks));
downloadSuccess = true;
c();
});
@@ -72,6 +71,7 @@ async function getSysroot(arch) {
});
}
if (!downloadSuccess) {
fs.rmSync(tarball);
throw new Error('Failed to download ' + url);
}
const sha = getSha(tarball);

10
build/linux/debian/install-sysroot.ts
View File

@@ -9,7 +9,7 @@ import { tmpdir } from 'os';
import * as fs from 'fs';
import * as https from 'https';
import * as path from 'path';
import { ArchString } from './types';
import { DebianArchString } from './types';
import * as util from '../../lib/util';
// Based on https://source.chromium.org/chromium/chromium/src/+/main:build/linux/sysroot_scripts/install-sysroot.py.
@@ -37,7 +37,7 @@ type SysrootDictEntry = {
Tarball: string;
};
export async function getSysroot(arch: ArchString): Promise<string> {
export async function getSysroot(arch: DebianArchString): Promise<string> {
const sysrootJSONUrl = `https://raw.githubusercontent.com/electron/electron/v${util.getElectronVersion()}/script/sysroots.json`;
const sysrootDictLocation = `${tmpdir()}/sysroots.json`;
const result = spawnSync('curl', [sysrootJSONUrl, '-o', sysrootDictLocation]);
@@ -63,14 +63,13 @@ export async function getSysroot(arch: ArchString): Promise<string> {
console.log(`Downloading ${url}`);
let downloadSuccess = false;
for (let i = 0; i < 3 && !downloadSuccess; i++) {
fs.writeFileSync(tarball, '');
await new Promise<void>((c) => {
https.get(url, (res) => {
const chunks: Uint8Array[] = [];
res.on('data', (chunk) => {
chunks.push(chunk);
fs.appendFileSync(tarball, chunk);
});
res.on('end', () => {
fs.writeFileSync(tarball, Buffer.concat(chunks));
downloadSuccess = true;
c();
});
@@ -81,6 +80,7 @@ export async function getSysroot(arch: ArchString): Promise<string> {
});
}
if (!downloadSuccess) {
fs.rmSync(tarball);
throw new Error('Failed to download ' + url);
}
const sha = getSha(tarball);

5
build/linux/debian/types.js
View File

@@ -4,3 +4,8 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.isDebianArchString = void 0;
function isDebianArchString(s) {
return ['amd64', 'armhf', 'arm64'].includes(s);
}
exports.isDebianArchString = isDebianArchString;

6
build/linux/debian/types.ts
View File

@@ -3,4 +3,8 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
export type ArchString = 'amd64' | 'armhf' | 'arm64';
export type DebianArchString = 'amd64' | 'armhf' | 'arm64';
export function isDebianArchString(s: string): s is DebianArchString {
return ['amd64', 'armhf', 'arm64'].includes(s);
}