mirror of
https://github.com/microsoft/vscode.git
synced 2026-02-15 07:28:05 +00:00
Merge pull request #294145 from microsoft/joh/esbuild-the-things
esbuild for transpile and bundle
This commit is contained in:
Johannes Rieken
GitHub
52
.vscode/tasks.json
vendored
52
.vscode/tasks.json
vendored
@@ -1,10 +1,40 @@
|
||||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"type": "npm",
|
||||
"script": "watch-client-transpiled",
|
||||
"label": "Core - Transpile",
|
||||
"isBackground": true,
|
||||
"presentation": {
|
||||
"reveal": "never",
|
||||
"group": "buildWatchers",
|
||||
"close": false
|
||||
},
|
||||
"problemMatcher": {
|
||||
"owner": "esbuild",
|
||||
"applyTo": "closedDocuments",
|
||||
"fileLocation": [
|
||||
"relative",
|
||||
"${workspaceFolder}/src"
|
||||
],
|
||||
"pattern": {
|
||||
"regexp": "^(.+?):(\\d+):(\\d+): ERROR: (.+)$",
|
||||
"file": 1,
|
||||
"line": 2,
|
||||
"column": 3,
|
||||
"message": 4
|
||||
},
|
||||
"background": {
|
||||
"beginsPattern": "Starting transpilation...",
|
||||
"endsPattern": "Finished transpilation with"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "npm",
|
||||
"script": "watch-clientd",
|
||||
"label": "Core - Build",
|
||||
"label": "Core - Typecheck",
|
||||
"isBackground": true,
|
||||
"presentation": {
|
||||
"reveal": "never",
|
||||
@@ -60,7 +90,8 @@
|
||||
{
|
||||
"label": "VS Code - Build",
|
||||
"dependsOn": [
|
||||
"Core - Build",
|
||||
"Core - Transpile",
|
||||
"Core - Typecheck",
|
||||
"Ext - Build"
|
||||
],
|
||||
"group": {
|
||||
@@ -69,10 +100,22 @@
|
||||
},
|
||||
"problemMatcher": []
|
||||
},
|
||||
{
|
||||
"type": "npm",
|
||||
"script": "kill-watch-client-transpiled",
|
||||
"label": "Kill Core - Transpile",
|
||||
"group": "build",
|
||||
"presentation": {
|
||||
"reveal": "never",
|
||||
"group": "buildKillers",
|
||||
"close": true
|
||||
},
|
||||
"problemMatcher": "$tsc"
|
||||
},
|
||||
{
|
||||
"type": "npm",
|
||||
"script": "kill-watch-clientd",
|
||||
"label": "Kill Core - Build",
|
||||
"label": "Kill Core - Typecheck",
|
||||
"group": "build",
|
||||
"presentation": {
|
||||
"reveal": "never",
|
||||
@@ -96,7 +139,8 @@
|
||||
{
|
||||
"label": "Kill VS Code - Build",
|
||||
"dependsOn": [
|
||||
"Kill Core - Build",
|
||||
"Kill Core - Transpile",
|
||||
"Kill Core - Typecheck",
|
||||
"Kill Ext - Build"
|
||||
],
|
||||
"group": "build",
|
||||
|
||||
12
build/buildConfig.ts
Normal file
12
build/buildConfig.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
/**
|
||||
* When `true`, self-hosting uses esbuild for fast transpilation (build/next)
|
||||
* and gulp-tsb only for type-checking (`noEmit`).
|
||||
*
|
||||
* When `false`, gulp-tsb does both transpilation and type-checking (old behavior).
|
||||
*/
|
||||
export const useEsbuildTranspile = true;
|
||||
@@ -13,6 +13,7 @@ import { compileExtensionMediaTask, compileExtensionsTask, watchExtensionsTask }
|
||||
import * as compilation from './lib/compilation.ts';
|
||||
import * as task from './lib/task.ts';
|
||||
import * as util from './lib/util.ts';
|
||||
import { useEsbuildTranspile } from './buildConfig.ts';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
@@ -32,7 +33,9 @@ gulp.task(transpileClientTask);
|
||||
const compileClientTask = task.define('compile-client', task.series(util.rimraf('out'), compilation.copyCodiconsTask, compilation.compileApiProposalNamesTask, compilation.compileTask('src', 'out', false)));
|
||||
gulp.task(compileClientTask);
|
||||
|
||||
const watchClientTask = task.define('watch-client', task.series(util.rimraf('out'), task.parallel(compilation.watchTask('out', false), compilation.watchApiProposalNamesTask, compilation.watchCodiconsTask)));
|
||||
const watchClientTask = useEsbuildTranspile
|
||||
? task.define('watch-client', task.parallel(compilation.watchTask('out', false, 'src', { noEmit: true }), compilation.watchApiProposalNamesTask, compilation.watchCodiconsTask))
|
||||
: task.define('watch-client', task.series(util.rimraf('out'), task.parallel(compilation.watchTask('out', false), compilation.watchApiProposalNamesTask, compilation.watchCodiconsTask)));
|
||||
gulp.task(watchClientTask);
|
||||
|
||||
// All
|
||||
|
||||
@@ -30,9 +30,12 @@ import { createAsar } from './lib/asar.ts';
|
||||
import minimist from 'minimist';
|
||||
import { compileBuildWithoutManglingTask, compileBuildWithManglingTask } from './gulpfile.compile.ts';
|
||||
import { compileNonNativeExtensionsBuildTask, compileNativeExtensionsBuildTask, compileAllExtensionsBuildTask, compileExtensionMediaBuildTask, cleanExtensionsBuildTask } from './gulpfile.extensions.ts';
|
||||
import { copyCodiconsTask } from './lib/compilation.ts';
|
||||
import { useEsbuildTranspile } from './buildConfig.ts';
|
||||
import { promisify } from 'util';
|
||||
import globCallback from 'glob';
|
||||
import rceditCallback from 'rcedit';
|
||||
import * as cp from 'child_process';
|
||||
|
||||
|
||||
const glob = promisify(globCallback);
|
||||
@@ -152,6 +155,81 @@ const bundleVSCodeTask = task.define('bundle-vscode', task.series(
|
||||
));
|
||||
gulp.task(bundleVSCodeTask);
|
||||
|
||||
// esbuild-based bundle tasks (drop-in replacement for bundle-vscode / minify-vscode)
|
||||
function runEsbuildTranspile(outDir: string, excludeTests: boolean): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const scriptPath = path.join(root, 'build/next/index.ts');
|
||||
const args = [scriptPath, 'transpile', '--out', outDir];
|
||||
if (excludeTests) {
|
||||
args.push('--exclude-tests');
|
||||
}
|
||||
|
||||
const proc = cp.spawn(process.execPath, args, {
|
||||
cwd: root,
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
proc.on('error', reject);
|
||||
proc.on('close', code => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`esbuild transpile failed with exit code ${code} (outDir: ${outDir})`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function runEsbuildBundle(outDir: string, minify: boolean, nls: boolean, target: 'desktop' | 'server' | 'server-web' = 'desktop', sourceMapBaseUrl?: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
// const tsxPath = path.join(root, 'build/node_modules/tsx/dist/cli.mjs');
|
||||
const scriptPath = path.join(root, 'build/next/index.ts');
|
||||
const args = [scriptPath, 'bundle', '--out', outDir, '--target', target];
|
||||
if (minify) {
|
||||
args.push('--minify');
|
||||
}
|
||||
if (nls) {
|
||||
args.push('--nls');
|
||||
}
|
||||
if (sourceMapBaseUrl) {
|
||||
args.push('--source-map-base-url', sourceMapBaseUrl);
|
||||
}
|
||||
|
||||
const proc = cp.spawn(process.execPath, args, {
|
||||
cwd: root,
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
proc.on('error', reject);
|
||||
proc.on('close', code => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`esbuild bundle failed with exit code ${code} (outDir: ${outDir}, minify: ${minify}, nls: ${nls}, target: ${target})`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function runTsGoTypeCheck(): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const proc = cp.spawn('tsgo', ['--project', 'src/tsconfig.json', '--noEmit', '--skipLibCheck'], {
|
||||
cwd: root,
|
||||
stdio: 'inherit',
|
||||
shell: true
|
||||
});
|
||||
|
||||
proc.on('error', reject);
|
||||
proc.on('close', code => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`tsgo typecheck failed with exit code ${code}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const sourceMappingURLBase = `https://main.vscode-cdn.net/sourcemaps/${commit}`;
|
||||
const minifyVSCodeTask = task.define('minify-vscode', task.series(
|
||||
bundleVSCodeTask,
|
||||
@@ -160,7 +238,7 @@ const minifyVSCodeTask = task.define('minify-vscode', task.series(
|
||||
));
|
||||
gulp.task(minifyVSCodeTask);
|
||||
|
||||
const coreCI = task.define('core-ci', task.series(
|
||||
const coreCIOld = task.define('core-ci-old', task.series(
|
||||
gulp.task('compile-build-with-mangling') as task.Task,
|
||||
task.parallel(
|
||||
gulp.task('minify-vscode') as task.Task,
|
||||
@@ -168,7 +246,27 @@ const coreCI = task.define('core-ci', task.series(
|
||||
gulp.task('minify-vscode-reh-web') as task.Task,
|
||||
)
|
||||
));
|
||||
gulp.task(coreCI);
|
||||
gulp.task(coreCIOld);
|
||||
|
||||
const coreCIEsbuild = task.define('core-ci-esbuild', task.series(
|
||||
copyCodiconsTask,
|
||||
cleanExtensionsBuildTask,
|
||||
compileNonNativeExtensionsBuildTask,
|
||||
compileExtensionMediaBuildTask,
|
||||
// Type-check with tsgo (no emit)
|
||||
task.define('tsgo-typecheck', () => runTsGoTypeCheck()),
|
||||
// Transpile individual files to out-build first (for unit tests)
|
||||
task.define('esbuild-out-build', () => runEsbuildTranspile('out-build', false)),
|
||||
// Then bundle for shipping (bundles also write NLS files to out-build)
|
||||
task.parallel(
|
||||
task.define('esbuild-vscode-min', () => runEsbuildBundle('out-vscode-min', true, true, 'desktop', `${sourceMappingURLBase}/core`)),
|
||||
task.define('esbuild-vscode-reh-min', () => runEsbuildBundle('out-vscode-reh-min', true, true, 'server', `${sourceMappingURLBase}/core`)),
|
||||
task.define('esbuild-vscode-reh-web-min', () => runEsbuildBundle('out-vscode-reh-web-min', true, true, 'server-web', `${sourceMappingURLBase}/core`)),
|
||||
)
|
||||
));
|
||||
gulp.task(coreCIEsbuild);
|
||||
|
||||
gulp.task(task.define('core-ci', useEsbuildTranspile ? coreCIEsbuild : coreCIOld));
|
||||
|
||||
const coreCIPR = task.define('core-ci-pr', task.series(
|
||||
gulp.task('compile-build-without-mangling') as task.Task,
|
||||
@@ -516,27 +614,43 @@ BUILD_TARGETS.forEach(buildTarget => {
|
||||
const sourceFolderName = `out-vscode${dashed(minified)}`;
|
||||
const destinationFolderName = `VSCode${dashed(platform)}${dashed(arch)}`;
|
||||
|
||||
const tasks = [
|
||||
const packageTasks: task.Task[] = [
|
||||
compileNativeExtensionsBuildTask,
|
||||
util.rimraf(path.join(buildRoot, destinationFolderName)),
|
||||
packageTask(platform, arch, sourceFolderName, destinationFolderName, opts)
|
||||
];
|
||||
|
||||
if (platform === 'win32') {
|
||||
tasks.push(patchWin32DependenciesTask(destinationFolderName));
|
||||
packageTasks.push(patchWin32DependenciesTask(destinationFolderName));
|
||||
}
|
||||
|
||||
const vscodeTaskCI = task.define(`vscode${dashed(platform)}${dashed(arch)}${dashed(minified)}-ci`, task.series(...tasks));
|
||||
const vscodeTaskCI = task.define(`vscode${dashed(platform)}${dashed(arch)}${dashed(minified)}-ci`, task.series(...packageTasks));
|
||||
gulp.task(vscodeTaskCI);
|
||||
|
||||
const vscodeTask = task.define(`vscode${dashed(platform)}${dashed(arch)}${dashed(minified)}`, task.series(
|
||||
minified ? compileBuildWithManglingTask : compileBuildWithoutManglingTask,
|
||||
cleanExtensionsBuildTask,
|
||||
compileNonNativeExtensionsBuildTask,
|
||||
compileExtensionMediaBuildTask,
|
||||
minified ? minifyVSCodeTask : bundleVSCodeTask,
|
||||
vscodeTaskCI
|
||||
));
|
||||
let vscodeTask: task.Task;
|
||||
if (useEsbuildTranspile) {
|
||||
const esbuildBundleTask = task.define(
|
||||
`esbuild-bundle${dashed(platform)}${dashed(arch)}${dashed(minified)}`,
|
||||
() => runEsbuildBundle(sourceFolderName, !!minified, true, 'desktop', minified ? `${sourceMappingURLBase}/core` : undefined)
|
||||
);
|
||||
vscodeTask = task.define(`vscode${dashed(platform)}${dashed(arch)}${dashed(minified)}`, task.series(
|
||||
copyCodiconsTask,
|
||||
cleanExtensionsBuildTask,
|
||||
compileNonNativeExtensionsBuildTask,
|
||||
compileExtensionMediaBuildTask,
|
||||
esbuildBundleTask,
|
||||
vscodeTaskCI
|
||||
));
|
||||
} else {
|
||||
vscodeTask = task.define(`vscode${dashed(platform)}${dashed(arch)}${dashed(minified)}`, task.series(
|
||||
minified ? compileBuildWithManglingTask : compileBuildWithoutManglingTask,
|
||||
cleanExtensionsBuildTask,
|
||||
compileNonNativeExtensionsBuildTask,
|
||||
compileExtensionMediaBuildTask,
|
||||
minified ? minifyVSCodeTask : bundleVSCodeTask,
|
||||
vscodeTaskCI
|
||||
));
|
||||
}
|
||||
gulp.task(vscodeTask);
|
||||
|
||||
return vscodeTask;
|
||||
@@ -568,7 +682,7 @@ const innoSetupConfig: Record<string, { codePage: string; defaultInfo?: { name:
|
||||
gulp.task(task.define(
|
||||
'vscode-translations-export',
|
||||
task.series(
|
||||
coreCI,
|
||||
gulp.task('core-ci') as task.Task,
|
||||
compileAllExtensionsBuildTask,
|
||||
function () {
|
||||
const pathToMetadata = './out-build/nls.metadata.json';
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
|
||||
import gulp from 'gulp';
|
||||
import * as path from 'path';
|
||||
import * as cp from 'child_process';
|
||||
import es from 'event-stream';
|
||||
import * as util from './lib/util.ts';
|
||||
import { getVersion } from './lib/getVersion.ts';
|
||||
@@ -18,6 +19,7 @@ import { getProductionDependencies } from './lib/dependencies.ts';
|
||||
import vfs from 'vinyl-fs';
|
||||
import packageJson from '../package.json' with { type: 'json' };
|
||||
import { compileBuildWithManglingTask } from './gulpfile.compile.ts';
|
||||
import { copyCodiconsTask } from './lib/compilation.ts';
|
||||
import * as extensions from './lib/extensions.ts';
|
||||
import jsonEditor from 'gulp-json-editor';
|
||||
import buildfile from './buildfile.ts';
|
||||
@@ -30,6 +32,34 @@ const commit = getVersion(REPO_ROOT);
|
||||
const quality = (product as { quality?: string }).quality;
|
||||
const version = (quality && quality !== 'stable') ? `${packageJson.version}-${quality}` : packageJson.version;
|
||||
|
||||
// esbuild-based bundle for standalone web
|
||||
function runEsbuildBundle(outDir: string, minify: boolean, nls: boolean): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const scriptPath = path.join(REPO_ROOT, 'build/next/index.ts');
|
||||
const args = [scriptPath, 'bundle', '--out', outDir, '--target', 'web'];
|
||||
if (minify) {
|
||||
args.push('--minify');
|
||||
}
|
||||
if (nls) {
|
||||
args.push('--nls');
|
||||
}
|
||||
|
||||
const proc = cp.spawn(process.execPath, args, {
|
||||
cwd: REPO_ROOT,
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
proc.on('error', reject);
|
||||
proc.on('close', code => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`esbuild web bundle failed with exit code ${code} (outDir: ${outDir}, minify: ${minify}, nls: ${nls})`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export const vscodeWebResourceIncludes = [
|
||||
|
||||
// NLS
|
||||
@@ -110,7 +140,7 @@ export const createVSCodeWebFileContentMapper = (extensionsRoot: string, product
|
||||
};
|
||||
};
|
||||
|
||||
const bundleVSCodeWebTask = task.define('bundle-vscode-web', task.series(
|
||||
const bundleVSCodeWebTask = task.define('bundle-vscode-web-OLD', task.series(
|
||||
util.rimraf('out-vscode-web'),
|
||||
optimize.bundleTask(
|
||||
{
|
||||
@@ -125,13 +155,17 @@ const bundleVSCodeWebTask = task.define('bundle-vscode-web', task.series(
|
||||
)
|
||||
));
|
||||
|
||||
const minifyVSCodeWebTask = task.define('minify-vscode-web', task.series(
|
||||
const minifyVSCodeWebTask = task.define('minify-vscode-web-OLD', task.series(
|
||||
bundleVSCodeWebTask,
|
||||
util.rimraf('out-vscode-web-min'),
|
||||
optimize.minifyTask('out-vscode-web', `https://main.vscode-cdn.net/sourcemaps/${commit}/core`)
|
||||
));
|
||||
gulp.task(minifyVSCodeWebTask);
|
||||
|
||||
// esbuild-based tasks (new)
|
||||
const esbuildBundleVSCodeWebTask = task.define('esbuild-vscode-web', () => runEsbuildBundle('out-vscode-web', false, true));
|
||||
const esbuildBundleVSCodeWebMinTask = task.define('esbuild-vscode-web-min', () => runEsbuildBundle('out-vscode-web-min', true, true));
|
||||
|
||||
function packageTask(sourceFolderName: string, destinationFolderName: string) {
|
||||
const destination = path.join(BUILD_ROOT, destinationFolderName);
|
||||
|
||||
@@ -197,8 +231,9 @@ const dashed = (str: string) => (str ? `-${str}` : ``);
|
||||
const destinationFolderName = `vscode-web`;
|
||||
|
||||
const vscodeWebTaskCI = task.define(`vscode-web${dashed(minified)}-ci`, task.series(
|
||||
copyCodiconsTask,
|
||||
compileWebExtensionsBuildTask,
|
||||
minified ? minifyVSCodeWebTask : bundleVSCodeWebTask,
|
||||
minified ? esbuildBundleVSCodeWebMinTask : esbuildBundleVSCodeWebTask,
|
||||
util.rimraf(path.join(BUILD_ROOT, destinationFolderName)),
|
||||
packageTask(sourceFolderName, destinationFolderName)
|
||||
));
|
||||
|
||||
@@ -49,14 +49,18 @@ interface ICompileTaskOptions {
|
||||
readonly emitError: boolean;
|
||||
readonly transpileOnly: boolean | { esbuild: boolean };
|
||||
readonly preserveEnglish: boolean;
|
||||
readonly noEmit?: boolean;
|
||||
}
|
||||
|
||||
export function createCompile(src: string, { build, emitError, transpileOnly, preserveEnglish }: ICompileTaskOptions) {
|
||||
export function createCompile(src: string, { build, emitError, transpileOnly, preserveEnglish, noEmit }: ICompileTaskOptions) {
|
||||
const projectPath = path.join(import.meta.dirname, '../../', src, 'tsconfig.json');
|
||||
const overrideOptions = { ...getTypeScriptCompilerOptions(src), inlineSources: Boolean(build) };
|
||||
if (!build) {
|
||||
overrideOptions.inlineSourceMap = true;
|
||||
}
|
||||
if (noEmit) {
|
||||
overrideOptions.noEmit = true;
|
||||
}
|
||||
|
||||
const compilation = tsb.create(projectPath, overrideOptions, {
|
||||
verbose: false,
|
||||
@@ -163,10 +167,10 @@ export function compileTask(src: string, out: string, build: boolean, options: {
|
||||
return task;
|
||||
}
|
||||
|
||||
export function watchTask(out: string, build: boolean, srcPath: string = 'src'): task.StreamTask {
|
||||
export function watchTask(out: string, build: boolean, srcPath: string = 'src', options?: { noEmit?: boolean }): task.StreamTask {
|
||||
|
||||
const task = () => {
|
||||
const compile = createCompile(srcPath, { build, emitError: false, transpileOnly: false, preserveEnglish: false });
|
||||
const compile = createCompile(srcPath, { build, emitError: false, transpileOnly: false, preserveEnglish: false, noEmit: options?.noEmit });
|
||||
|
||||
const src = gulp.src(`${srcPath}/**`, { base: srcPath });
|
||||
const watchSrc = watch(`${srcPath}/**`, { base: srcPath, readDelay: 200 });
|
||||
|
||||
317
build/lib/nls-analysis.ts
Normal file
317
build/lib/nls-analysis.ts
Normal file
@@ -0,0 +1,317 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as ts from 'typescript';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
export interface ISpan {
|
||||
start: ts.LineAndCharacter;
|
||||
end: ts.LineAndCharacter;
|
||||
}
|
||||
|
||||
export interface ILocalizeCall {
|
||||
keySpan: ISpan;
|
||||
key: string;
|
||||
valueSpan: ISpan;
|
||||
value: string;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// AST Collection
|
||||
// ============================================================================
|
||||
|
||||
export const CollectStepResult = Object.freeze({
|
||||
Yes: 'Yes',
|
||||
YesAndRecurse: 'YesAndRecurse',
|
||||
No: 'No',
|
||||
NoAndRecurse: 'NoAndRecurse'
|
||||
});
|
||||
|
||||
export type CollectStepResult = typeof CollectStepResult[keyof typeof CollectStepResult];
|
||||
|
||||
export function collect(node: ts.Node, fn: (node: ts.Node) => CollectStepResult): ts.Node[] {
|
||||
const result: ts.Node[] = [];
|
||||
|
||||
function loop(node: ts.Node) {
|
||||
const stepResult = fn(node);
|
||||
|
||||
if (stepResult === CollectStepResult.Yes || stepResult === CollectStepResult.YesAndRecurse) {
|
||||
result.push(node);
|
||||
}
|
||||
|
||||
if (stepResult === CollectStepResult.YesAndRecurse || stepResult === CollectStepResult.NoAndRecurse) {
|
||||
ts.forEachChild(node, loop);
|
||||
}
|
||||
}
|
||||
|
||||
loop(node);
|
||||
return result;
|
||||
}
|
||||
|
||||
export function isImportNode(node: ts.Node): boolean {
|
||||
return node.kind === ts.SyntaxKind.ImportDeclaration || node.kind === ts.SyntaxKind.ImportEqualsDeclaration;
|
||||
}
|
||||
|
||||
export function isCallExpressionWithinTextSpanCollectStep(textSpan: ts.TextSpan, node: ts.Node): CollectStepResult {
|
||||
if (!ts.textSpanContainsTextSpan({ start: node.pos, length: node.end - node.pos }, textSpan)) {
|
||||
return CollectStepResult.No;
|
||||
}
|
||||
|
||||
return node.kind === ts.SyntaxKind.CallExpression ? CollectStepResult.YesAndRecurse : CollectStepResult.NoAndRecurse;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Language Service Host
|
||||
// ============================================================================
|
||||
|
||||
export class SingleFileServiceHost implements ts.LanguageServiceHost {
|
||||
private file: ts.IScriptSnapshot;
|
||||
private lib: ts.IScriptSnapshot;
|
||||
private options: ts.CompilerOptions;
|
||||
private filename: string;
|
||||
|
||||
constructor(options: ts.CompilerOptions, filename: string, contents: string) {
|
||||
this.options = options;
|
||||
this.filename = filename;
|
||||
this.file = ts.ScriptSnapshot.fromString(contents);
|
||||
this.lib = ts.ScriptSnapshot.fromString('');
|
||||
}
|
||||
|
||||
getCompilationSettings = () => this.options;
|
||||
getScriptFileNames = () => [this.filename];
|
||||
getScriptVersion = () => '1';
|
||||
getScriptSnapshot = (name: string) => name === this.filename ? this.file : this.lib;
|
||||
getCurrentDirectory = () => '';
|
||||
getDefaultLibFileName = () => 'lib.d.ts';
|
||||
|
||||
readFile(path: string): string | undefined {
|
||||
if (path === this.filename) {
|
||||
return this.file.getText(0, this.file.getLength());
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
fileExists(path: string): boolean {
|
||||
return path === this.filename;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Analysis
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Analyzes TypeScript source code to find localize() or localize2() calls.
|
||||
*/
|
||||
export function analyzeLocalizeCalls(
|
||||
contents: string,
|
||||
functionName: 'localize' | 'localize2'
|
||||
): ILocalizeCall[] {
|
||||
const filename = 'file.ts';
|
||||
const options: ts.CompilerOptions = { noResolve: true };
|
||||
const serviceHost = new SingleFileServiceHost(options, filename, contents);
|
||||
const service = ts.createLanguageService(serviceHost);
|
||||
const sourceFile = ts.createSourceFile(filename, contents, ts.ScriptTarget.ES5, true);
|
||||
|
||||
// Find all imports
|
||||
const imports = collect(sourceFile, n => isImportNode(n) ? CollectStepResult.YesAndRecurse : CollectStepResult.NoAndRecurse);
|
||||
|
||||
// import nls = require('vs/nls');
|
||||
const importEqualsDeclarations = imports
|
||||
.filter(n => n.kind === ts.SyntaxKind.ImportEqualsDeclaration)
|
||||
.map(n => n as ts.ImportEqualsDeclaration)
|
||||
.filter(d => d.moduleReference.kind === ts.SyntaxKind.ExternalModuleReference)
|
||||
.filter(d => {
|
||||
const text = (d.moduleReference as ts.ExternalModuleReference).expression.getText();
|
||||
return text.endsWith(`/nls'`) || text.endsWith(`/nls"`) || text.endsWith(`/nls.js'`) || text.endsWith(`/nls.js"`);
|
||||
});
|
||||
|
||||
// import ... from 'vs/nls';
|
||||
const importDeclarations = imports
|
||||
.filter(n => n.kind === ts.SyntaxKind.ImportDeclaration)
|
||||
.map(n => n as ts.ImportDeclaration)
|
||||
.filter(d => d.moduleSpecifier.kind === ts.SyntaxKind.StringLiteral)
|
||||
.filter(d => {
|
||||
const text = d.moduleSpecifier.getText();
|
||||
return text.endsWith(`/nls'`) || text.endsWith(`/nls"`) || text.endsWith(`/nls.js'`) || text.endsWith(`/nls.js"`);
|
||||
})
|
||||
.filter(d => !!d.importClause && !!d.importClause.namedBindings);
|
||||
|
||||
// `nls.localize(...)` calls via namespace import
|
||||
const nlsLocalizeCallExpressions: ts.CallExpression[] = [];
|
||||
|
||||
const namespaceImports = importDeclarations
|
||||
.filter(d => d.importClause?.namedBindings?.kind === ts.SyntaxKind.NamespaceImport)
|
||||
.map(d => (d.importClause!.namedBindings as ts.NamespaceImport).name);
|
||||
|
||||
const importEqualsNames = importEqualsDeclarations.map(d => d.name);
|
||||
|
||||
for (const name of [...namespaceImports, ...importEqualsNames]) {
|
||||
const refs = service.getReferencesAtPosition(filename, name.pos + 1) ?? [];
|
||||
for (const ref of refs) {
|
||||
if (ref.isWriteAccess) {
|
||||
continue;
|
||||
}
|
||||
const calls = collect(sourceFile, n => isCallExpressionWithinTextSpanCollectStep(ref.textSpan, n));
|
||||
const lastCall = calls[calls.length - 1] as ts.CallExpression | undefined;
|
||||
if (lastCall &&
|
||||
lastCall.expression.kind === ts.SyntaxKind.PropertyAccessExpression &&
|
||||
(lastCall.expression as ts.PropertyAccessExpression).name.getText() === functionName) {
|
||||
nlsLocalizeCallExpressions.push(lastCall);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// `localize` named imports
|
||||
const namedImports = importDeclarations
|
||||
.filter(d => d.importClause?.namedBindings?.kind === ts.SyntaxKind.NamedImports)
|
||||
.flatMap(d => Array.from((d.importClause!.namedBindings! as ts.NamedImports).elements));
|
||||
|
||||
const localizeCallExpressions: ts.CallExpression[] = [];
|
||||
|
||||
// Direct named import: import { localize } from 'vs/nls'
|
||||
for (const namedImport of namedImports) {
|
||||
const isTarget = namedImport.name.getText() === functionName ||
|
||||
(namedImport.propertyName && namedImport.propertyName.getText() === functionName);
|
||||
|
||||
if (!isTarget) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const searchName = namedImport.propertyName ? namedImport.name : namedImport.name;
|
||||
const refs = service.getReferencesAtPosition(filename, searchName.pos + 1) ?? [];
|
||||
|
||||
for (const ref of refs) {
|
||||
if (ref.isWriteAccess) {
|
||||
continue;
|
||||
}
|
||||
const calls = collect(sourceFile, n => isCallExpressionWithinTextSpanCollectStep(ref.textSpan, n));
|
||||
const lastCall = calls[calls.length - 1] as ts.CallExpression | undefined;
|
||||
if (lastCall) {
|
||||
localizeCallExpressions.push(lastCall);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Combine and deduplicate
|
||||
const allCalls = [...nlsLocalizeCallExpressions, ...localizeCallExpressions];
|
||||
const seen = new Set<number>();
|
||||
const uniqueCalls = allCalls.filter(call => {
|
||||
const start = call.getStart();
|
||||
if (seen.has(start)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(start);
|
||||
return true;
|
||||
});
|
||||
|
||||
// Convert to ILocalizeCall
|
||||
return uniqueCalls
|
||||
.filter(e => e.arguments.length > 1)
|
||||
.sort((a, b) => a.arguments[0].getStart() - b.arguments[0].getStart())
|
||||
.map(e => {
|
||||
const args = e.arguments;
|
||||
return {
|
||||
keySpan: {
|
||||
start: ts.getLineAndCharacterOfPosition(sourceFile, args[0].getStart()),
|
||||
end: ts.getLineAndCharacterOfPosition(sourceFile, args[0].getEnd())
|
||||
},
|
||||
key: args[0].getText(),
|
||||
valueSpan: {
|
||||
start: ts.getLineAndCharacterOfPosition(sourceFile, args[1].getStart()),
|
||||
end: ts.getLineAndCharacterOfPosition(sourceFile, args[1].getEnd())
|
||||
},
|
||||
value: args[1].getText()
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Text Model for patching
|
||||
// ============================================================================
|
||||
|
||||
export class TextModel {
|
||||
private lines: string[];
|
||||
private lineEndings: string[];
|
||||
|
||||
constructor(contents: string) {
|
||||
const regex = /\r\n|\r|\n/g;
|
||||
let index = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
this.lines = [];
|
||||
this.lineEndings = [];
|
||||
|
||||
while (match = regex.exec(contents)) {
|
||||
this.lines.push(contents.substring(index, match.index));
|
||||
this.lineEndings.push(match[0]);
|
||||
index = regex.lastIndex;
|
||||
}
|
||||
|
||||
if (contents.length > 0) {
|
||||
this.lines.push(contents.substring(index, contents.length));
|
||||
this.lineEndings.push('');
|
||||
}
|
||||
}
|
||||
|
||||
get(index: number): string {
|
||||
return this.lines[index];
|
||||
}
|
||||
|
||||
set(index: number, line: string): void {
|
||||
this.lines[index] = line;
|
||||
}
|
||||
|
||||
get lineCount(): number {
|
||||
return this.lines.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies patch(es) to the model.
|
||||
* Multiple patches must be ordered.
|
||||
* Does not support patches spanning multiple lines.
|
||||
*/
|
||||
apply(span: ISpan, content: string): void {
|
||||
const startLineNumber = span.start.line;
|
||||
const endLineNumber = span.end.line;
|
||||
|
||||
const startLine = this.lines[startLineNumber] || '';
|
||||
const endLine = this.lines[endLineNumber] || '';
|
||||
|
||||
this.lines[startLineNumber] = [
|
||||
startLine.substring(0, span.start.character),
|
||||
content,
|
||||
endLine.substring(span.end.character)
|
||||
].join('');
|
||||
|
||||
for (let i = startLineNumber + 1; i <= endLineNumber; i++) {
|
||||
this.lines[i] = '';
|
||||
}
|
||||
}
|
||||
|
||||
toString(): string {
|
||||
let result = '';
|
||||
for (let i = 0; i < this.lines.length; i++) {
|
||||
result += this.lines[i] + this.lineEndings[i];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Utilities
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Parses a localize key or value expression.
|
||||
* sourceExpression can be "foo", 'foo', `foo` or { key: 'foo', comment: [...] }
|
||||
*/
|
||||
export function parseLocalizeKeyOrValue(sourceExpression: string): string | { key: string; comment?: string[] } {
|
||||
// eslint-disable-next-line no-eval
|
||||
return eval(`(${sourceExpression})`);
|
||||
}
|
||||
277
build/lib/nls.ts
277
build/lib/nls.ts
@@ -10,45 +10,10 @@ import File from 'vinyl';
|
||||
import sm from 'source-map';
|
||||
import path from 'path';
|
||||
import sort from 'gulp-sort';
|
||||
import { type ISpan, analyzeLocalizeCalls, TextModel, parseLocalizeKeyOrValue } from './nls-analysis.ts';
|
||||
|
||||
type FileWithSourcemap = File & { sourceMap: sm.RawSourceMap };
|
||||
|
||||
const CollectStepResult = Object.freeze({
|
||||
Yes: 'Yes',
|
||||
YesAndRecurse: 'YesAndRecurse',
|
||||
No: 'No',
|
||||
NoAndRecurse: 'NoAndRecurse'
|
||||
});
|
||||
|
||||
type CollectStepResult = typeof CollectStepResult[keyof typeof CollectStepResult];
|
||||
|
||||
function collect(ts: typeof import('typescript'), node: ts.Node, fn: (node: ts.Node) => CollectStepResult): ts.Node[] {
|
||||
const result: ts.Node[] = [];
|
||||
|
||||
function loop(node: ts.Node) {
|
||||
const stepResult = fn(node);
|
||||
|
||||
if (stepResult === CollectStepResult.Yes || stepResult === CollectStepResult.YesAndRecurse) {
|
||||
result.push(node);
|
||||
}
|
||||
|
||||
if (stepResult === CollectStepResult.YesAndRecurse || stepResult === CollectStepResult.NoAndRecurse) {
|
||||
ts.forEachChild(node, loop);
|
||||
}
|
||||
}
|
||||
|
||||
loop(node);
|
||||
return result;
|
||||
}
|
||||
|
||||
function clone<T extends object>(object: T): T {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const id in object) {
|
||||
result[id] = object[id];
|
||||
}
|
||||
return result as T;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a stream containing the patched JavaScript and source maps.
|
||||
*/
|
||||
@@ -117,10 +82,6 @@ globalThis._VSCODE_NLS_MESSAGES=${JSON.stringify(_nls.allNLSMessages)};`),
|
||||
return eventStream.duplex(input, output);
|
||||
}
|
||||
|
||||
function isImportNode(ts: typeof import('typescript'), node: ts.Node): boolean {
|
||||
return node.kind === ts.SyntaxKind.ImportDeclaration || node.kind === ts.SyntaxKind.ImportEqualsDeclaration;
|
||||
}
|
||||
|
||||
const _nls = (() => {
|
||||
|
||||
const moduleToNLSKeys: { [name: string /* module ID */]: ILocalizeKey[] /* keys */ } = {};
|
||||
@@ -138,22 +99,6 @@ const _nls = (() => {
|
||||
nlsKeys?: ILocalizeKey[];
|
||||
}
|
||||
|
||||
interface ISpan {
|
||||
start: ts.LineAndCharacter;
|
||||
end: ts.LineAndCharacter;
|
||||
}
|
||||
|
||||
interface ILocalizeCall {
|
||||
keySpan: ISpan;
|
||||
key: string;
|
||||
valueSpan: ISpan;
|
||||
value: string;
|
||||
}
|
||||
|
||||
interface ILocalizeAnalysisResult {
|
||||
localizeCalls: ILocalizeCall[];
|
||||
}
|
||||
|
||||
interface IPatch {
|
||||
span: ISpan;
|
||||
content: string;
|
||||
@@ -176,212 +121,11 @@ const _nls = (() => {
|
||||
return { line: position.line - 1, character: position.column };
|
||||
}
|
||||
|
||||
class SingleFileServiceHost implements ts.LanguageServiceHost {
|
||||
|
||||
private file: ts.IScriptSnapshot;
|
||||
private lib: ts.IScriptSnapshot;
|
||||
private options: ts.CompilerOptions;
|
||||
private filename: string;
|
||||
|
||||
constructor(ts: typeof import('typescript'), options: ts.CompilerOptions, filename: string, contents: string) {
|
||||
this.options = options;
|
||||
this.filename = filename;
|
||||
this.file = ts.ScriptSnapshot.fromString(contents);
|
||||
this.lib = ts.ScriptSnapshot.fromString('');
|
||||
}
|
||||
|
||||
getCompilationSettings = () => this.options;
|
||||
getScriptFileNames = () => [this.filename];
|
||||
getScriptVersion = () => '1';
|
||||
getScriptSnapshot = (name: string) => name === this.filename ? this.file : this.lib;
|
||||
getCurrentDirectory = () => '';
|
||||
getDefaultLibFileName = () => 'lib.d.ts';
|
||||
|
||||
readFile(path: string, _encoding?: string): string | undefined {
|
||||
if (path === this.filename) {
|
||||
return this.file.getText(0, this.file.getLength());
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
fileExists(path: string): boolean {
|
||||
return path === this.filename;
|
||||
}
|
||||
}
|
||||
|
||||
function isCallExpressionWithinTextSpanCollectStep(ts: typeof import('typescript'), textSpan: ts.TextSpan, node: ts.Node): CollectStepResult {
|
||||
if (!ts.textSpanContainsTextSpan({ start: node.pos, length: node.end - node.pos }, textSpan)) {
|
||||
return CollectStepResult.No;
|
||||
}
|
||||
|
||||
return node.kind === ts.SyntaxKind.CallExpression ? CollectStepResult.YesAndRecurse : CollectStepResult.NoAndRecurse;
|
||||
}
|
||||
|
||||
function analyze(
|
||||
ts: typeof import('typescript'),
|
||||
contents: string,
|
||||
functionName: 'localize' | 'localize2',
|
||||
options: ts.CompilerOptions = {}
|
||||
): ILocalizeAnalysisResult {
|
||||
const filename = 'file.ts';
|
||||
const serviceHost = new SingleFileServiceHost(ts, Object.assign(clone(options), { noResolve: true }), filename, contents);
|
||||
const service = ts.createLanguageService(serviceHost);
|
||||
const sourceFile = ts.createSourceFile(filename, contents, ts.ScriptTarget.ES5, true);
|
||||
|
||||
// all imports
|
||||
const imports = lazy(collect(ts, sourceFile, n => isImportNode(ts, n) ? CollectStepResult.YesAndRecurse : CollectStepResult.NoAndRecurse));
|
||||
|
||||
// import nls = require('vs/nls');
|
||||
const importEqualsDeclarations = imports
|
||||
.filter(n => n.kind === ts.SyntaxKind.ImportEqualsDeclaration)
|
||||
.map(n => n as ts.ImportEqualsDeclaration)
|
||||
.filter(d => d.moduleReference.kind === ts.SyntaxKind.ExternalModuleReference)
|
||||
.filter(d => (d.moduleReference as ts.ExternalModuleReference).expression.getText().endsWith(`/nls.js'`));
|
||||
|
||||
// import ... from 'vs/nls';
|
||||
const importDeclarations = imports
|
||||
.filter(n => n.kind === ts.SyntaxKind.ImportDeclaration)
|
||||
.map(n => n as ts.ImportDeclaration)
|
||||
.filter(d => d.moduleSpecifier.kind === ts.SyntaxKind.StringLiteral)
|
||||
.filter(d => d.moduleSpecifier.getText().endsWith(`/nls.js'`))
|
||||
.filter(d => !!d.importClause && !!d.importClause.namedBindings);
|
||||
|
||||
// `nls.localize(...)` calls
|
||||
const nlsLocalizeCallExpressions = importDeclarations
|
||||
.filter(d => !!(d.importClause && d.importClause.namedBindings && d.importClause.namedBindings.kind === ts.SyntaxKind.NamespaceImport))
|
||||
.map(d => (d.importClause!.namedBindings as ts.NamespaceImport).name)
|
||||
.concat(importEqualsDeclarations.map(d => d.name))
|
||||
|
||||
// find read-only references to `nls`
|
||||
.map(n => service.getReferencesAtPosition(filename, n.pos + 1) ?? [])
|
||||
.flatten()
|
||||
.filter(r => !r.isWriteAccess)
|
||||
|
||||
// find the deepest call expressions AST nodes that contain those references
|
||||
.map(r => collect(ts, sourceFile, n => isCallExpressionWithinTextSpanCollectStep(ts, r.textSpan, n)))
|
||||
.map(a => lazy(a).last())
|
||||
.filter(n => !!n)
|
||||
.map(n => n as ts.CallExpression)
|
||||
|
||||
// only `localize` calls
|
||||
.filter(n => n.expression.kind === ts.SyntaxKind.PropertyAccessExpression && (n.expression as ts.PropertyAccessExpression).name.getText() === functionName);
|
||||
|
||||
// `localize` named imports
|
||||
const allLocalizeImportDeclarations = importDeclarations
|
||||
.filter(d => !!(d.importClause && d.importClause.namedBindings && d.importClause.namedBindings.kind === ts.SyntaxKind.NamedImports))
|
||||
.map(d => (d.importClause!.namedBindings! as ts.NamedImports).elements)
|
||||
.flatten();
|
||||
|
||||
// `localize` read-only references
|
||||
const localizeReferences = allLocalizeImportDeclarations
|
||||
.filter(d => d.name.getText() === functionName)
|
||||
.map(n => service.getReferencesAtPosition(filename, n.pos + 1) ?? [])
|
||||
.flatten()
|
||||
.filter(r => !r.isWriteAccess);
|
||||
|
||||
// custom named `localize` read-only references
|
||||
const namedLocalizeReferences = allLocalizeImportDeclarations
|
||||
.filter(d => !!d.propertyName && d.propertyName.getText() === functionName)
|
||||
.map(n => service.getReferencesAtPosition(filename, n.name.pos + 1) ?? [])
|
||||
.flatten()
|
||||
.filter(r => !r.isWriteAccess);
|
||||
|
||||
// find the deepest call expressions AST nodes that contain those references
|
||||
const localizeCallExpressions = localizeReferences
|
||||
.concat(namedLocalizeReferences)
|
||||
.map(r => collect(ts, sourceFile, n => isCallExpressionWithinTextSpanCollectStep(ts, r.textSpan, n)))
|
||||
.map(a => lazy(a).last())
|
||||
.filter(n => !!n)
|
||||
.map(n => n as ts.CallExpression);
|
||||
|
||||
// collect everything
|
||||
const localizeCalls = nlsLocalizeCallExpressions
|
||||
.concat(localizeCallExpressions)
|
||||
.map(e => e.arguments)
|
||||
.filter(a => a.length > 1)
|
||||
.sort((a, b) => a[0].getStart() - b[0].getStart())
|
||||
.map<ILocalizeCall>(a => ({
|
||||
keySpan: { start: ts.getLineAndCharacterOfPosition(sourceFile, a[0].getStart()), end: ts.getLineAndCharacterOfPosition(sourceFile, a[0].getEnd()) },
|
||||
key: a[0].getText(),
|
||||
valueSpan: { start: ts.getLineAndCharacterOfPosition(sourceFile, a[1].getStart()), end: ts.getLineAndCharacterOfPosition(sourceFile, a[1].getEnd()) },
|
||||
value: a[1].getText()
|
||||
}));
|
||||
|
||||
return {
|
||||
localizeCalls: localizeCalls.toArray()
|
||||
};
|
||||
}
|
||||
|
||||
class TextModel {
|
||||
|
||||
private lines: string[];
|
||||
private lineEndings: string[];
|
||||
|
||||
constructor(contents: string) {
|
||||
const regex = /\r\n|\r|\n/g;
|
||||
let index = 0;
|
||||
let match: RegExpExecArray | null;
|
||||
|
||||
this.lines = [];
|
||||
this.lineEndings = [];
|
||||
|
||||
while (match = regex.exec(contents)) {
|
||||
this.lines.push(contents.substring(index, match.index));
|
||||
this.lineEndings.push(match[0]);
|
||||
index = regex.lastIndex;
|
||||
}
|
||||
|
||||
if (contents.length > 0) {
|
||||
this.lines.push(contents.substring(index, contents.length));
|
||||
this.lineEndings.push('');
|
||||
}
|
||||
}
|
||||
|
||||
public get(index: number): string {
|
||||
return this.lines[index];
|
||||
}
|
||||
|
||||
public set(index: number, line: string): void {
|
||||
this.lines[index] = line;
|
||||
}
|
||||
|
||||
public get lineCount(): number {
|
||||
return this.lines.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies patch(es) to the model.
|
||||
* Multiple patches must be ordered.
|
||||
* Does not support patches spanning multiple lines.
|
||||
*/
|
||||
public apply(patch: IPatch): void {
|
||||
const startLineNumber = patch.span.start.line;
|
||||
const endLineNumber = patch.span.end.line;
|
||||
|
||||
const startLine = this.lines[startLineNumber] || '';
|
||||
const endLine = this.lines[endLineNumber] || '';
|
||||
|
||||
this.lines[startLineNumber] = [
|
||||
startLine.substring(0, patch.span.start.character),
|
||||
patch.content,
|
||||
endLine.substring(patch.span.end.character)
|
||||
].join('');
|
||||
|
||||
for (let i = startLineNumber + 1; i <= endLineNumber; i++) {
|
||||
this.lines[i] = '';
|
||||
}
|
||||
}
|
||||
|
||||
public toString(): string {
|
||||
return lazy(this.lines).zip(this.lineEndings)
|
||||
.flatten().toArray().join('');
|
||||
}
|
||||
}
|
||||
|
||||
function patchJavascript(patches: IPatch[], contents: string): string {
|
||||
const model = new TextModel(contents);
|
||||
|
||||
// patch the localize calls
|
||||
lazy(patches).reverse().each(p => model.apply(p));
|
||||
lazy(patches).reverse().each(p => model.apply(p.span, p.content));
|
||||
|
||||
return model.toString();
|
||||
}
|
||||
@@ -431,24 +175,16 @@ const _nls = (() => {
|
||||
return JSON.parse(smg.toString());
|
||||
}
|
||||
|
||||
function parseLocalizeKeyOrValue(sourceExpression: string) {
|
||||
// sourceValue can be "foo", 'foo', `foo` or { .... }
|
||||
// in its evalulated form
|
||||
// we want to return either the string or the object
|
||||
// eslint-disable-next-line no-eval
|
||||
return eval(`(${sourceExpression})`);
|
||||
}
|
||||
|
||||
function patch(ts: typeof import('typescript'), typescript: string, javascript: string, sourcemap: sm.RawSourceMap, options: { preserveEnglish: boolean }): INlsPatchResult {
|
||||
const { localizeCalls } = analyze(ts, typescript, 'localize');
|
||||
const { localizeCalls: localize2Calls } = analyze(ts, typescript, 'localize2');
|
||||
function patch(typescript: string, javascript: string, sourcemap: sm.RawSourceMap, options: { preserveEnglish: boolean }): INlsPatchResult {
|
||||
const localizeCalls = analyzeLocalizeCalls(typescript, 'localize');
|
||||
const localize2Calls = analyzeLocalizeCalls(typescript, 'localize2');
|
||||
|
||||
if (localizeCalls.length === 0 && localize2Calls.length === 0) {
|
||||
return { javascript, sourcemap };
|
||||
}
|
||||
|
||||
const nlsKeys = localizeCalls.map(lc => parseLocalizeKeyOrValue(lc.key)).concat(localize2Calls.map(lc => parseLocalizeKeyOrValue(lc.key)));
|
||||
const nlsMessages = localizeCalls.map(lc => parseLocalizeKeyOrValue(lc.value)).concat(localize2Calls.map(lc => parseLocalizeKeyOrValue(lc.value)));
|
||||
const nlsMessages = localizeCalls.map(lc => parseLocalizeKeyOrValue(lc.value) as string).concat(localize2Calls.map(lc => parseLocalizeKeyOrValue(lc.value) as string));
|
||||
const smc = new sm.SourceMapConsumer(sourcemap);
|
||||
const positionFrom = mappedPositionFrom.bind(null, sourcemap.sources[0]);
|
||||
|
||||
@@ -505,7 +241,6 @@ const _nls = (() => {
|
||||
.replace(/\\/g, '/');
|
||||
|
||||
const { javascript, sourcemap, nlsKeys, nlsMessages } = patch(
|
||||
ts,
|
||||
typescript,
|
||||
javascriptFile.contents!.toString(),
|
||||
javascriptFile.sourceMap,
|
||||
|
||||
1134
build/next/index.ts
Normal file
1134
build/next/index.ts
Normal file
File diff suppressed because it is too large
Load Diff
319
build/next/nls-plugin.ts
Normal file
319
build/next/nls-plugin.ts
Normal file
@@ -0,0 +1,319 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as esbuild from 'esbuild';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import {
|
||||
TextModel,
|
||||
analyzeLocalizeCalls,
|
||||
parseLocalizeKeyOrValue
|
||||
} from '../lib/nls-analysis.ts';
|
||||
|
||||
// ============================================================================
|
||||
// Types
|
||||
// ============================================================================
|
||||
|
||||
interface NLSEntry {
|
||||
moduleId: string;
|
||||
key: string | { key: string; comment: string[] };
|
||||
message: string;
|
||||
placeholder: string;
|
||||
}
|
||||
|
||||
export interface NLSPluginOptions {
|
||||
/**
|
||||
* Base path for computing module IDs (e.g., 'src')
|
||||
*/
|
||||
baseDir: string;
|
||||
|
||||
/**
|
||||
* Shared collector for NLS entries across multiple builds.
|
||||
* Create with createNLSCollector() and pass to multiple plugin instances.
|
||||
*/
|
||||
collector: NLSCollector;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collector for NLS entries across multiple esbuild builds.
|
||||
*/
|
||||
export interface NLSCollector {
|
||||
entries: Map<string, NLSEntry>;
|
||||
add(entry: NLSEntry): void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a shared NLS collector that can be passed to multiple plugin instances.
|
||||
*/
|
||||
export function createNLSCollector(): NLSCollector {
|
||||
const entries = new Map<string, NLSEntry>();
|
||||
return {
|
||||
entries,
|
||||
add(entry: NLSEntry) {
|
||||
entries.set(entry.placeholder, entry);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Finalizes NLS collection and writes output files.
|
||||
* Call this after all esbuild builds have completed.
|
||||
*/
|
||||
export async function finalizeNLS(
|
||||
collector: NLSCollector,
|
||||
outDir: string,
|
||||
alsoWriteTo?: string[]
|
||||
): Promise<{ indexMap: Map<string, number>; messageCount: number }> {
|
||||
if (collector.entries.size === 0) {
|
||||
return { indexMap: new Map(), messageCount: 0 };
|
||||
}
|
||||
|
||||
// Sort entries by moduleId, then by key for stable indices
|
||||
const sortedEntries = [...collector.entries.values()].sort((a, b) => {
|
||||
const aKey = typeof a.key === 'string' ? a.key : a.key.key;
|
||||
const bKey = typeof b.key === 'string' ? b.key : b.key.key;
|
||||
const moduleCompare = a.moduleId.localeCompare(b.moduleId);
|
||||
if (moduleCompare !== 0) {
|
||||
return moduleCompare;
|
||||
}
|
||||
return aKey.localeCompare(bKey);
|
||||
});
|
||||
|
||||
// Create index map
|
||||
const indexMap = new Map<string, number>();
|
||||
sortedEntries.forEach((entry, idx) => {
|
||||
indexMap.set(entry.placeholder, idx);
|
||||
});
|
||||
|
||||
// Build NLS metadata
|
||||
const allMessages: string[] = [];
|
||||
const moduleToKeys: Map<string, (string | { key: string; comment: string[] })[]> = new Map();
|
||||
const moduleToMessages: Map<string, string[]> = new Map();
|
||||
|
||||
for (const entry of sortedEntries) {
|
||||
allMessages.push(entry.message);
|
||||
|
||||
if (!moduleToKeys.has(entry.moduleId)) {
|
||||
moduleToKeys.set(entry.moduleId, []);
|
||||
moduleToMessages.set(entry.moduleId, []);
|
||||
}
|
||||
moduleToKeys.get(entry.moduleId)!.push(entry.key);
|
||||
moduleToMessages.get(entry.moduleId)!.push(entry.message);
|
||||
}
|
||||
|
||||
// nls.keys.json: [["moduleId", ["key1", "key2"]], ...]
|
||||
const nlsKeysJson: [string, string[]][] = [];
|
||||
for (const [moduleId, keys] of moduleToKeys) {
|
||||
nlsKeysJson.push([moduleId, keys.map(k => typeof k === 'string' ? k : k.key)]);
|
||||
}
|
||||
|
||||
// nls.metadata.json: { keys: {...}, messages: {...} }
|
||||
const nlsMetadataJson = {
|
||||
keys: Object.fromEntries(moduleToKeys),
|
||||
messages: Object.fromEntries(moduleToMessages)
|
||||
};
|
||||
|
||||
// Write NLS files
|
||||
const allOutDirs = [outDir, ...(alsoWriteTo ?? [])];
|
||||
for (const dir of allOutDirs) {
|
||||
await fs.promises.mkdir(dir, { recursive: true });
|
||||
}
|
||||
|
||||
await Promise.all(allOutDirs.flatMap(dir => [
|
||||
fs.promises.writeFile(
|
||||
path.join(dir, 'nls.messages.json'),
|
||||
JSON.stringify(allMessages)
|
||||
),
|
||||
fs.promises.writeFile(
|
||||
path.join(dir, 'nls.keys.json'),
|
||||
JSON.stringify(nlsKeysJson)
|
||||
),
|
||||
fs.promises.writeFile(
|
||||
path.join(dir, 'nls.metadata.json'),
|
||||
JSON.stringify(nlsMetadataJson, null, '\t')
|
||||
),
|
||||
fs.promises.writeFile(
|
||||
path.join(dir, 'nls.messages.js'),
|
||||
`/*---------------------------------------------------------\n * Copyright (C) Microsoft Corporation. All rights reserved.\n *--------------------------------------------------------*/\nglobalThis._VSCODE_NLS_MESSAGES=${JSON.stringify(allMessages)};`
|
||||
),
|
||||
]));
|
||||
|
||||
console.log(`[nls] Extracted ${allMessages.length} messages from ${moduleToKeys.size} modules`);
|
||||
|
||||
return { indexMap, messageCount: allMessages.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-processes a JavaScript file to replace NLS placeholders with indices.
|
||||
*/
|
||||
export function postProcessNLS(
|
||||
content: string,
|
||||
indexMap: Map<string, number>,
|
||||
preserveEnglish: boolean
|
||||
): string {
|
||||
return replaceInOutput(content, indexMap, preserveEnglish);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Transformation
|
||||
// ============================================================================
|
||||
|
||||
function transformToPlaceholders(
|
||||
source: string,
|
||||
moduleId: string
|
||||
): { code: string; entries: NLSEntry[] } {
|
||||
const localizeCalls = analyzeLocalizeCalls(source, 'localize');
|
||||
const localize2Calls = analyzeLocalizeCalls(source, 'localize2');
|
||||
|
||||
// Tag calls with their type so we can handle them differently later
|
||||
const taggedLocalize = localizeCalls.map(call => ({ call, isLocalize2: false }));
|
||||
const taggedLocalize2 = localize2Calls.map(call => ({ call, isLocalize2: true }));
|
||||
const allCalls = [...taggedLocalize, ...taggedLocalize2].sort(
|
||||
(a, b) => a.call.keySpan.start.line - b.call.keySpan.start.line ||
|
||||
a.call.keySpan.start.character - b.call.keySpan.start.character
|
||||
);
|
||||
|
||||
if (allCalls.length === 0) {
|
||||
return { code: source, entries: [] };
|
||||
}
|
||||
|
||||
const entries: NLSEntry[] = [];
|
||||
const model = new TextModel(source);
|
||||
|
||||
// Process in reverse order to preserve positions
|
||||
for (const { call, isLocalize2 } of allCalls.reverse()) {
|
||||
const keyParsed = parseLocalizeKeyOrValue(call.key) as string | { key: string; comment: string[] };
|
||||
const messageParsed = parseLocalizeKeyOrValue(call.value);
|
||||
const keyString = typeof keyParsed === 'string' ? keyParsed : keyParsed.key;
|
||||
|
||||
// Use different placeholder prefix for localize vs localize2
|
||||
// localize: message will be replaced with null
|
||||
// localize2: message will be preserved (only key replaced)
|
||||
const prefix = isLocalize2 ? 'NLS2' : 'NLS';
|
||||
const placeholder = `%%${prefix}:${moduleId}#${keyString}%%`;
|
||||
|
||||
entries.push({
|
||||
moduleId,
|
||||
key: keyParsed,
|
||||
message: String(messageParsed),
|
||||
placeholder
|
||||
});
|
||||
|
||||
// Replace the key with the placeholder string
|
||||
model.apply(call.keySpan, `"${placeholder}"`);
|
||||
}
|
||||
|
||||
// Reverse entries to match source order
|
||||
entries.reverse();
|
||||
|
||||
return { code: model.toString(), entries };
|
||||
}
|
||||
|
||||
function replaceInOutput(
|
||||
content: string,
|
||||
indexMap: Map<string, number>,
|
||||
preserveEnglish: boolean
|
||||
): string {
|
||||
// Replace all placeholders in a single pass using regex
|
||||
// Two types of placeholders:
|
||||
// - %%NLS:moduleId#key%% for localize() - message replaced with null
|
||||
// - %%NLS2:moduleId#key%% for localize2() - message preserved
|
||||
// Note: esbuild may use single or double quotes, so we handle both
|
||||
|
||||
if (preserveEnglish) {
|
||||
// Just replace the placeholder with the index (both NLS and NLS2)
|
||||
return content.replace(/["']%%NLS2?:([^%]+)%%["']/g, (match, inner) => {
|
||||
// Try NLS first, then NLS2
|
||||
let placeholder = `%%NLS:${inner}%%`;
|
||||
let index = indexMap.get(placeholder);
|
||||
if (index === undefined) {
|
||||
placeholder = `%%NLS2:${inner}%%`;
|
||||
index = indexMap.get(placeholder);
|
||||
}
|
||||
if (index !== undefined) {
|
||||
return String(index);
|
||||
}
|
||||
// Placeholder not found in map, leave as-is (shouldn't happen)
|
||||
return match;
|
||||
});
|
||||
} else {
|
||||
// For NLS (localize): replace placeholder with index AND replace message with null
|
||||
// For NLS2 (localize2): replace placeholder with index, keep message
|
||||
// Note: Use (?:[^"\\]|\\.)* to properly handle escaped quotes like \" or \\
|
||||
// Note: esbuild may use single or double quotes, so we handle both
|
||||
|
||||
// First handle NLS (localize) - replace both key and message
|
||||
content = content.replace(
|
||||
/["']%%NLS:([^%]+)%%["'](\s*,\s*)(?:"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*'|`(?:[^`\\]|\\.)*`)/g,
|
||||
(match, inner, comma) => {
|
||||
const placeholder = `%%NLS:${inner}%%`;
|
||||
const index = indexMap.get(placeholder);
|
||||
if (index !== undefined) {
|
||||
return `${index}${comma}null`;
|
||||
}
|
||||
return match;
|
||||
}
|
||||
);
|
||||
|
||||
// Then handle NLS2 (localize2) - replace only key, keep message
|
||||
content = content.replace(
|
||||
/["']%%NLS2:([^%]+)%%["']/g,
|
||||
(match, inner) => {
|
||||
const placeholder = `%%NLS2:${inner}%%`;
|
||||
const index = indexMap.get(placeholder);
|
||||
if (index !== undefined) {
|
||||
return String(index);
|
||||
}
|
||||
return match;
|
||||
}
|
||||
);
|
||||
|
||||
return content;
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Plugin
|
||||
// ============================================================================
|
||||
|
||||
export function nlsPlugin(options: NLSPluginOptions): esbuild.Plugin {
|
||||
const { collector } = options;
|
||||
|
||||
return {
|
||||
name: 'nls',
|
||||
setup(build) {
|
||||
// Transform TypeScript files to replace localize() calls with placeholders
|
||||
build.onLoad({ filter: /\.ts$/ }, async (args) => {
|
||||
// Skip .d.ts files
|
||||
if (args.path.endsWith('.d.ts')) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const source = await fs.promises.readFile(args.path, 'utf-8');
|
||||
|
||||
// Compute module ID (e.g., "vs/editor/editor" from "src/vs/editor/editor.ts")
|
||||
const relativePath = path.relative(options.baseDir, args.path);
|
||||
const moduleId = relativePath
|
||||
.replace(/\\/g, '/')
|
||||
.replace(/\.ts$/, '');
|
||||
|
||||
// Transform localize() calls to placeholders
|
||||
const { code, entries: fileEntries } = transformToPlaceholders(source, moduleId);
|
||||
|
||||
// Collect entries
|
||||
for (const entry of fileEntries) {
|
||||
collector.add(entry);
|
||||
}
|
||||
|
||||
if (fileEntries.length > 0) {
|
||||
return { contents: code, loader: 'ts' };
|
||||
}
|
||||
|
||||
// No NLS calls, return undefined to let esbuild handle normally
|
||||
return undefined;
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
235
build/next/working.md
Normal file
235
build/next/working.md
Normal file
@@ -0,0 +1,235 @@
|
||||
# Working Notes: New esbuild-based Build System
|
||||
|
||||
> These notes are for AI agents to help with context in new or summarized sessions.
|
||||
|
||||
## Important: Validating Changes
|
||||
|
||||
**The `VS Code - Build` task is NOT needed to validate changes in the `build/` folder!**
|
||||
|
||||
Build scripts in `build/` are TypeScript files that run directly with `tsx` (e.g., `npx tsx build/next/index.ts`). They are not compiled by the main VS Code build.
|
||||
|
||||
To test changes:
|
||||
```bash
|
||||
# Test transpile
|
||||
npx tsx build/next/index.ts transpile --out out-test
|
||||
|
||||
# Test bundle (server-web target to test the auth fix)
|
||||
npx tsx build/next/index.ts bundle --nls --target server-web --out out-vscode-reh-web-test
|
||||
|
||||
# Verify product config was injected
|
||||
grep -l "serverLicense" out-vscode-reh-web-test/vs/code/browser/workbench/workbench.js
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Architecture Overview
|
||||
|
||||
### Files
|
||||
|
||||
- **[index.ts](index.ts)** - Main build orchestrator
|
||||
- `transpile` command: Fast TS → JS using `esbuild.transform()`
|
||||
- `bundle` command: TS → bundled JS using `esbuild.build()`
|
||||
- **[nls-plugin.ts](nls-plugin.ts)** - NLS (localization) esbuild plugin
|
||||
|
||||
### Integration with Old Build
|
||||
|
||||
In [gulpfile.vscode.ts](../gulpfile.vscode.ts#L228-L242), the `core-ci` task uses these new scripts:
|
||||
- `runEsbuildTranspile()` → transpile command
|
||||
- `runEsbuildBundle()` → bundle command
|
||||
|
||||
Old gulp-based bundling renamed to `core-ci-OLD`.
|
||||
|
||||
---
|
||||
|
||||
## Key Learnings
|
||||
|
||||
### 1. Comment Stripping by esbuild
|
||||
|
||||
**Problem:** esbuild strips comments like `/*BUILD->INSERT_PRODUCT_CONFIGURATION*/` during bundling.
|
||||
|
||||
**Solution:** Use an `onLoad` plugin to transform source files BEFORE esbuild processes them. See `fileContentMapperPlugin()` in index.ts.
|
||||
|
||||
**Why post-processing doesn't work:** By the time we post-process the bundled output, the comment placeholder has already been stripped.
|
||||
|
||||
### 2. Authorization Error: "Unauthorized client refused"
|
||||
|
||||
**Root cause:** Missing product configuration in browser bundle.
|
||||
|
||||
**Flow:**
|
||||
1. Browser loads with empty product config (placeholder was stripped)
|
||||
2. `productService.serverLicense` is empty/undefined
|
||||
3. Browser's `SignService.vsda()` can't decrypt vsda WASM (needs serverLicense as key)
|
||||
4. Browser's `sign()` returns original challenge instead of signed value
|
||||
5. Server validates signature → fails
|
||||
6. Server is in built mode (no `VSCODE_DEV`) → rejects connection
|
||||
|
||||
**Fix:** The `fileContentMapperPlugin` now runs during `onLoad`, replacing placeholders before esbuild strips them.
|
||||
|
||||
### 3. Build-Time Placeholders
|
||||
|
||||
Two placeholders that need injection:
|
||||
|
||||
| Placeholder | Location | Purpose |
|
||||
|-------------|----------|---------|
|
||||
| `/*BUILD->INSERT_PRODUCT_CONFIGURATION*/` | `src/vs/platform/product/common/product.ts` | Product config (commit, version, serverLicense, etc.) |
|
||||
| `/*BUILD->INSERT_BUILTIN_EXTENSIONS*/` | `src/vs/workbench/services/extensionManagement/browser/builtinExtensionsScannerService.ts` | List of built-in extensions |
|
||||
|
||||
### 4. Server-web Target Specifics
|
||||
|
||||
- Removes `webEndpointUrlTemplate` from product config (see `tweakProductForServerWeb` in old build)
|
||||
- Uses `.build/extensions` for builtin extensions (not `.build/web/extensions`)
|
||||
|
||||
### 5. Entry Point Parity with Old Build
|
||||
|
||||
**Problem:** The desktop target had `keyboardMapEntryPoints` as separate esbuild entry points, producing `layout.contribution.darwin.js`, `layout.contribution.linux.js`, and `layout.contribution.win.js` as standalone files in the output.
|
||||
|
||||
**Root cause:** In the old build (`gulpfile.vscode.ts`), `vscodeEntryPoints` does NOT include `buildfile.keyboardMaps`. These files are only separate entry points for server-web (`gulpfile.reh.ts`) and web (`gulpfile.vscode.web.ts`). For desktop, they're imported as dependencies of `workbench.desktop.main` and get bundled into it.
|
||||
|
||||
**Fix:** Removed `...keyboardMapEntryPoints` from the `desktop` case in `getEntryPointsForTarget()`. Keep for `server-web` and `web`.
|
||||
|
||||
**Lesson:** Always verify new build entry points against the old build's per-target definitions in `buildfile.ts` and the respective gulpfiles.
|
||||
|
||||
### 6. NLS Output File Parity
|
||||
|
||||
**Problem:** `finalizeNLS()` was generating `nls.messages.js` (with `globalThis._VSCODE_NLS_MESSAGES=...`) in addition to the standard `.json` files. The old build only produces `nls.messages.json`, `nls.keys.json`, and `nls.metadata.json`.
|
||||
|
||||
**Fix:** Removed `nls.messages.js` generation from `finalizeNLS()` in `nls-plugin.ts`.
|
||||
|
||||
**Lesson:** Don't add new output file formats that create parity differences with the old build. The old build is the reference.
|
||||
|
||||
---
|
||||
|
||||
## Testing the Fix
|
||||
|
||||
```bash
|
||||
# Build server-web with new system
|
||||
npx tsx build/next/index.ts bundle --nls --target server-web --out out-vscode-reh-web-min
|
||||
|
||||
# Package it (uses gulp task)
|
||||
npm run gulp vscode-reh-web-darwin-arm64-min
|
||||
|
||||
# Run server
|
||||
./vscode-server-darwin-arm64-web/bin/code-server-oss --connection-token dev-token
|
||||
|
||||
# Open browser - should connect without "Unauthorized client refused"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Open Items / Future Work
|
||||
|
||||
1. **`BUILD_INSERT_PACKAGE_CONFIGURATION`** - Server bootstrap files ([bootstrap-meta.ts](../../src/bootstrap-meta.ts)) have this marker for package.json injection. Currently handled by [inlineMeta.ts](../lib/inlineMeta.ts) in the old build's packaging step.
|
||||
|
||||
2. **Mangling** - The new build doesn't do TypeScript-based mangling yet. Old `core-ci` with mangling is now `core-ci-OLD`.
|
||||
|
||||
3. **Entry point duplication** - Entry points are duplicated between [buildfile.ts](../buildfile.ts) and [index.ts](index.ts). Consider consolidating.
|
||||
|
||||
---
|
||||
|
||||
## Build Comparison: OLD (gulp-tsb) vs NEW (esbuild) — Desktop Build
|
||||
|
||||
### Summary
|
||||
|
||||
| Metric | OLD | NEW | Delta |
|
||||
|--------|-----|-----|-------|
|
||||
| Total files in `out/` | 3993 | 4301 | +309 extra, 1 missing |
|
||||
| Total size of `out/` | 25.8 MB | 64.6 MB | +38.8 MB (2.5×) |
|
||||
| `workbench.desktop.main.js` | 13.0 MB | 15.5 MB | +2.5 MB |
|
||||
|
||||
### 1 Missing File (in OLD, not in NEW)
|
||||
|
||||
| File | Why Missing | Fix |
|
||||
|------|-------------|-----|
|
||||
| `out/vs/platform/browserView/electron-browser/preload-browserView.js` | Not listed in `desktopStandaloneFiles` in index.ts. Only `preload.ts` and `preload-aux.ts` are compiled as standalone files. | **Add** `'vs/platform/browserView/electron-browser/preload-browserView.ts'` to the `desktopStandaloneFiles` array in `index.ts`. |
|
||||
|
||||
### 309 Extra Files (in NEW, not in OLD) — Breakdown
|
||||
|
||||
| Category | Count | Explanation |
|
||||
|----------|-------|-------------|
|
||||
| **CSS files** | 291 | `copyCssFiles()` copies ALL `.css` from `src/` to the output. The old bundler inlines CSS into the main `.css` bundle (e.g., `workbench.desktop.main.css`) and never ships individual CSS files. These individual files ARE needed at runtime because the new ESM system uses `import './foo.css'` resolved by an import map. |
|
||||
| **Vendor JS files** | 3 | `dompurify.js`, `marked.js`, `semver.js` — listed in `commonResourcePatterns`. The old bundler inlines these into the main bundle. The new system keeps them as separate files because they're plain JS (not TS). They're needed. |
|
||||
| **Web workbench bundle** | 1 | `vs/code/browser/workbench/workbench.js` (15.4 MB). This is the web workbench entry point bundle. It should NOT be in a desktop build — the old build explicitly excludes `out-build/vs/code/browser/**`. The `desktopResourcePatterns` in index.ts includes `vs/code/browser/workbench/*.html` and `callback.html` which is correct, but the actual bundle gets written by the esbuild desktop bundle step because the desktop entry points include web entry points. |
|
||||
| **Web workbench internal** | 1 | `vs/workbench/workbench.web.main.internal.js` (15.4 MB). Similar: shouldn't ship in a desktop build. It's output by the esbuild bundler. |
|
||||
| **Keyboard layout contributions** | 3 | `layout.contribution.{darwin,linux,win}.js` — the old bundler inlines these into the main bundle. These are new separate files from the esbuild bundler. |
|
||||
| **NLS files** | 2 | `nls.messages.js` (new) and `nls.metadata.json` (new). The old build has `nls.messages.json` and `nls.keys.json` but not a `.js` version or metadata. The `.js` version is produced by the NLS plugin. |
|
||||
| **HTML files** | 2 | `vs/code/browser/workbench/workbench.html` and `callback.html` — correctly listed in `desktopResourcePatterns` (these are needed for desktop's built-in web server). |
|
||||
| **SVG loading spinners** | 3 | `loading-dark.svg`, `loading-hc.svg`, `loading.svg` in `vs/workbench/contrib/extensions/browser/media/`. The old build only copies `theme-icon.png` and `language-icon.svg` from that folder; the new build's `desktopResourcePatterns` uses `*.svg` which is broader. |
|
||||
| **codicon.ttf (duplicate)** | 1 | At `vs/base/browser/ui/codicons/codicon/codicon.ttf`. The old build copies this to `out/media/codicon.ttf` only. The new build has BOTH: the copy in `out/media/` (from esbuild's `file` loader) AND the original path (from `commonResourcePatterns`). Duplicate. |
|
||||
| **PSReadLine.psm1** | 1 | `vs/workbench/contrib/terminal/common/scripts/psreadline/PSReadLine.psm1` — the old build uses `*.psm1` in `terminal/common/scripts/` (non-recursive?). The new build uses `**/*.psm1` (recursive), picking up this subdirectory file. Check if it's needed. |
|
||||
| **date file** | 1 | `out/date` — build timestamp, produced by the new build's `bundle()` function. The old build doesn't write this; it reads `package.json.date` instead. |
|
||||
|
||||
### Size Increase Breakdown by Area
|
||||
|
||||
| Area | OLD | NEW | Delta | Why |
|
||||
|------|-----|-----|-------|-----|
|
||||
| `vs/code` | 1.5 MB | 17.4 MB | +15.9 MB | Web workbench bundle (15.4 MB) shouldn't be in desktop build |
|
||||
| `vs/workbench` | 18.9 MB | 38.7 MB | +19.8 MB | `workbench.web.main.internal.js` (15.4 MB) + unmangled desktop bundle (+2.5 MB) + individual CSS files (~1 MB) |
|
||||
| `vs/base` | 0 MB | 0.4 MB | +0.4 MB | Individual CSS files + vendor JS |
|
||||
| `vs/editor` | 0.3 MB | 0.5 MB | +0.1 MB | Individual CSS files |
|
||||
| `vs/platform` | 1.7 MB | 1.9 MB | +0.2 MB | Individual CSS files |
|
||||
|
||||
### JS Files with >2× Size Change
|
||||
|
||||
| File | OLD | NEW | Ratio | Reason |
|
||||
|------|-----|-----|-------|--------|
|
||||
| `vs/workbench/contrib/webview/browser/pre/service-worker.js` | 7 KB | 15 KB | 2.2× | Not minified / includes more inlined code |
|
||||
| `vs/code/electron-browser/workbench/workbench.js` | 10 KB | 28 KB | 2.75× | OLD is minified to 6 lines; NEW is 380 lines (not compressed, includes tslib banner) |
|
||||
|
||||
### Action Items
|
||||
|
||||
1. **[CRITICAL] Missing `preload-browserView.ts`** — Add to `desktopStandaloneFiles` in index.ts. Without it, BrowserView (used for Simple Browser) may fail.
|
||||
2. **[SIZE] Web bundles in desktop build** — `workbench.web.main.internal.js` and `vs/code/browser/workbench/workbench.js` together add ~31 MB. These are written by the esbuild bundler and not filtered out. Consider: either don't bundle web entry points for the desktop target, or ensure the packaging step excludes them (currently `packageTask` takes `out-vscode-min/**` without filtering).
|
||||
3. **[SIZE] No mangling** — The desktop main bundle is 2.5 MB larger due to no property mangling. Known open item.
|
||||
4. **[MINOR] Duplicate codicon.ttf** — Exists at both `out/media/codicon.ttf` (from esbuild `file` loader) and `out/vs/base/browser/ui/codicons/codicon/codicon.ttf` (from `commonResourcePatterns`). Consider removing from `commonResourcePatterns` if it's already handled by the loader.
|
||||
5. **[MINOR] Extra SVGs** — `desktopResourcePatterns` uses `*.svg` for extensions media but old build only ships `language-icon.svg`. The loading spinners may be unused in the desktop build.
|
||||
6. **[MINOR] Extra PSReadLine.psm1** from recursive glob — verify if needed.
|
||||
|
||||
---
|
||||
|
||||
## Source Maps
|
||||
|
||||
### Fixes Applied
|
||||
|
||||
1. **`sourcesContent: true`** — Production bundles now embed original TypeScript source content in `.map` files, matching the old build's `includeContent: true` behavior. Without this, crash reports from CDN-hosted source maps can't show original source.
|
||||
|
||||
2. **`--source-map-base-url` option** — The `bundle` command accepts an optional `--source-map-base-url <url>` flag. When set, post-processing rewrites `sourceMappingURL` comments in `.js` and `.css` output files to point to the CDN (e.g., `https://main.vscode-cdn.net/sourcemaps/<commit>/core/vs/...`). This matches the old build's `sourceMappingURL` function in `minifyTask()`. Wired up in `gulpfile.vscode.ts` for `core-ci-esbuild` and `vscode-esbuild-min` tasks.
|
||||
|
||||
### NLS Source Map Accuracy (Decision: Accept Imprecision)
|
||||
|
||||
**Problem:** `postProcessNLS()` replaces `"%%NLS:moduleId#key%%"` placeholders (~40 chars) with short index values like `null` (4 chars) in the final JS output. This shifts column positions without updating the `.map` files.
|
||||
|
||||
**Options considered:**
|
||||
|
||||
| Option | Description | Effort | Accuracy |
|
||||
|--------|-------------|--------|----------|
|
||||
| A. Fixed-width placeholders | Pad placeholders to match replacement length | Hard — indices unknown until all modules are collected across parallel bundles | Perfect |
|
||||
| B. Post-process source map | Parse `.map`, track replacement offsets per line, adjust VLQ mappings | Medium | Perfect |
|
||||
| C. Two-pass build | Assign NLS indices during plugin phase | Not feasible with parallel bundling | N/A |
|
||||
| **D. Accept imprecision** | NLS replacements only affect column positions; line-level debugging works | Zero | Line-level |
|
||||
|
||||
**Decision: Option D — accept imprecision.** Rationale:
|
||||
|
||||
- NLS replacements only shift **columns**, never lines — line-level stack traces and breakpoints remain correct.
|
||||
- Production crash reporting (the primary consumer of CDN source maps) uses line numbers; column-level accuracy is rarely needed.
|
||||
- The old gulp build had the same fundamental issue in its `nls.nls()` step and used `SourceMapConsumer`/`SourceMapGenerator` to fix it — but that approach was fragile and slow.
|
||||
- If column-level precision becomes important later (e.g., for minified+NLS bundles), Option B can be implemented: after NLS replacement, re-parse the source map, walk replacement sites, and adjust column offsets. This is a localized change in the post-processing loop.
|
||||
|
||||
---
|
||||
|
||||
## Self-hosting Setup
|
||||
|
||||
The default `VS Code - Build` task now runs three parallel watchers:
|
||||
|
||||
| Task | What it does | Script |
|
||||
|------|-------------|--------|
|
||||
| **Core - Transpile** | esbuild single-file TS→JS (fast, no type checking) | `watch-client-transpiled` → `npx tsx build/next/index.ts transpile --watch` |
|
||||
| **Core - Typecheck** | gulp-tsb `noEmit` watch (type errors only, no output) | `watch-clientd` → `gulp watch-client` (with `noEmit: true`) |
|
||||
| **Ext - Build** | Extension compilation (unchanged) | `watch-extensionsd` |
|
||||
|
||||
### Key Changes
|
||||
|
||||
- **`compilation.ts`**: `ICompileTaskOptions` gained `noEmit?: boolean`. When set, `overrideOptions.noEmit = true` is passed to tsb. `watchTask()` accepts an optional 4th parameter `{ noEmit?: boolean }`.
|
||||
- **`gulpfile.ts`**: `watchClientTask` no longer runs `rimraf('out')` (the transpiler owns that). Passes `{ noEmit: true }` to `watchTask`.
|
||||
- **`index.ts`**: Watch mode emits `Starting transpilation...` / `Finished transpilation with N errors after X ms` for VS Code problem matcher.
|
||||
- **`tasks.json`**: Old "Core - Build" split into "Core - Transpile" + "Core - Typecheck" with separate problem matchers (owners: `esbuild` vs `typescript`).
|
||||
- **`package.json`**: Added `watch-client-transpile`, `watch-client-transpiled`, `kill-watch-client-transpiled` scripts.
|
||||
@@ -20,7 +20,7 @@
|
||||
"postinstall": "node build/npm/postinstall.ts",
|
||||
"compile": "npm run gulp compile",
|
||||
"compile-check-ts-native": "tsgo --project ./src/tsconfig.json --noEmit --skipLibCheck",
|
||||
"watch": "npm-run-all2 -lp watch-client watch-extensions",
|
||||
"watch": "npm-run-all2 -lp watch-client-transpile watch-client watch-extensions",
|
||||
"watchd": "deemon npm run watch",
|
||||
"watch-webd": "deemon npm run watch-web",
|
||||
"kill-watchd": "deemon --kill npm run watch",
|
||||
@@ -30,6 +30,9 @@
|
||||
"watch-client": "npm run gulp watch-client",
|
||||
"watch-clientd": "deemon npm run watch-client",
|
||||
"kill-watch-clientd": "deemon --kill npm run watch-client",
|
||||
"watch-client-transpile": "npx tsx build/next/index.ts transpile --watch",
|
||||
"watch-client-transpiled": "deemon npm run watch-client-transpile",
|
||||
"kill-watch-client-transpiled": "deemon --kill npm run watch-client-transpile",
|
||||
"watch-extensions": "npm run gulp watch-extensions watch-extension-media",
|
||||
"watch-extensionsd": "deemon npm run watch-extensions",
|
||||
"kill-watch-extensionsd": "deemon --kill npm run watch-extensions",
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
@font-face {
|
||||
font-family: "codicon";
|
||||
font-display: block;
|
||||
src: url("./codicon.ttf?5d4d76ab2ce5108968ad644d591a16a6") format("truetype");
|
||||
src: url("./codicon.ttf") format("truetype");
|
||||
}
|
||||
|
||||
.codicon[class*='codicon-'] {
|
||||
|
||||
Reference in New Issue
Block a user