From 18456cc05f7c4dca4fd5e9b2316180bcb681870b Mon Sep 17 00:00:00 2001 From: Johannes Date: Tue, 10 Feb 2026 11:10:12 +0100 Subject: [PATCH] source maps: sourcesContent, CDN URL rewriting, replace @parcel/watcher --- build/gulpfile.vscode.ts | 13 ++++--- build/next/index.ts | 79 ++++++++++++++++++++++------------------ build/next/working.md | 30 +++++++++++++++ 3 files changed, 82 insertions(+), 40 deletions(-) diff --git a/build/gulpfile.vscode.ts b/build/gulpfile.vscode.ts index 16a9ed44173..c3489e0931e 100644 --- a/build/gulpfile.vscode.ts +++ b/build/gulpfile.vscode.ts @@ -180,7 +180,7 @@ function runEsbuildTranspile(outDir: string, excludeTests: boolean): Promise { +function runEsbuildBundle(outDir: string, minify: boolean, nls: boolean, target: 'desktop' | 'server' | 'server-web' = 'desktop', sourceMapBaseUrl?: string): Promise { return new Promise((resolve, reject) => { // const tsxPath = path.join(root, 'build/node_modules/tsx/dist/cli.mjs'); const scriptPath = path.join(root, 'build/next/index.ts'); @@ -191,6 +191,9 @@ function runEsbuildBundle(outDir: string, minify: boolean, nls: boolean, target: if (nls) { args.push('--nls'); } + if (sourceMapBaseUrl) { + args.push('--source-map-base-url', sourceMapBaseUrl); + } const proc = cp.spawn(process.execPath, args, { cwd: root, @@ -256,9 +259,9 @@ const coreCIEsbuild = task.define('core-ci-esbuild', task.series( task.define('esbuild-out-build', () => runEsbuildTranspile('out-build', false)), // Then bundle for shipping (bundles also write NLS files to out-build) task.parallel( - task.define('esbuild-vscode-min', () => runEsbuildBundle('out-vscode-min', true, true, 'desktop')), - task.define('esbuild-vscode-reh-min', () => runEsbuildBundle('out-vscode-reh-min', true, true, 'server')), - task.define('esbuild-vscode-reh-web-min', () => runEsbuildBundle('out-vscode-reh-web-min', true, true, 'server-web')), + task.define('esbuild-vscode-min', () => runEsbuildBundle('out-vscode-min', true, true, 'desktop', `${sourceMappingURLBase}/core`)), + task.define('esbuild-vscode-reh-min', () => runEsbuildBundle('out-vscode-reh-min', true, true, 'server', `${sourceMappingURLBase}/core`)), + task.define('esbuild-vscode-reh-web-min', () => runEsbuildBundle('out-vscode-reh-web-min', true, true, 'server-web', `${sourceMappingURLBase}/core`)), ) )); gulp.task(coreCIEsbuild); @@ -646,7 +649,7 @@ BUILD_TARGETS.forEach(buildTarget => { const esbuildBundleTask = task.define( `esbuild-bundle${dashed(platform)}${dashed(arch)}${dashed(minified)}`, - () => runEsbuildBundle(sourceFolderName, !!minified, true) + () => runEsbuildBundle(sourceFolderName, !!minified, true, 'desktop', minified ? `${sourceMappingURLBase}/core` : undefined) ); const tasks = [ diff --git a/build/next/index.ts b/build/next/index.ts index 62a3c2c177b..9ddf11f4be3 100644 --- a/build/next/index.ts +++ b/build/next/index.ts @@ -8,7 +8,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { promisify } from 'util'; import glob from 'glob'; -import * as watcher from '@parcel/watcher'; +import gulpWatch from '../lib/watch/index.ts'; import { nlsPlugin, createNLSCollector, finalizeNLS, postProcessNLS } from './nls-plugin.ts'; import { getVersion } from '../lib/getVersion.ts'; import product from '../../product.json' with { type: 'json' }; @@ -44,6 +44,7 @@ const options = { excludeTests: process.argv.includes('--exclude-tests'), out: getArgValue('--out'), target: getArgValue('--target') ?? 'desktop', // 'desktop' | 'server' | 'server-web' | 'web' + sourceMapBaseUrl: getArgValue('--source-map-base-url'), }; // Build targets @@ -736,7 +737,7 @@ async function transpile(outDir: string, excludeTests: boolean): Promise { // Bundle (Goal 2: JS → bundled JS) // ============================================================================ -async function bundle(outDir: string, doMinify: boolean, doNls: boolean, target: BuildTarget): Promise { +async function bundle(outDir: string, doMinify: boolean, doNls: boolean, target: BuildTarget, sourceMapBaseUrl?: string): Promise { await cleanDir(outDir); // Write build date file (used by packaging to embed in product.json) @@ -816,7 +817,7 @@ ${tslib}`, target: ['es2024'], packages: 'external', sourcemap: 'external', - sourcesContent: false, + sourcesContent: true, minify: doMinify, treeShaking: true, banner, @@ -868,7 +869,7 @@ ${tslib}`, target: ['es2024'], packages: 'external', sourcemap: 'external', - sourcesContent: false, + sourcesContent: true, minify: doMinify, treeShaking: true, banner, @@ -906,17 +907,30 @@ ${tslib}`, for (const file of result.outputFiles) { await fs.promises.mkdir(path.dirname(file.path), { recursive: true }); - if (file.path.endsWith('.js')) { + if (file.path.endsWith('.js') || file.path.endsWith('.css')) { let content = file.text; - // Apply NLS post-processing if enabled - if (doNls && indexMap.size > 0) { + // Apply NLS post-processing if enabled (JS only) + if (file.path.endsWith('.js') && doNls && indexMap.size > 0) { content = postProcessNLS(content, indexMap, preserveEnglish); } + // Rewrite sourceMappingURL to CDN URL if configured + if (sourceMapBaseUrl) { + const relativePath = path.relative(path.join(REPO_ROOT, outDir), file.path); + content = content.replace( + /\/\/# sourceMappingURL=.+$/m, + `//# sourceMappingURL=${sourceMapBaseUrl}/${relativePath}.map` + ); + content = content.replace( + /\/\*# sourceMappingURL=.+\*\/$/m, + `/*# sourceMappingURL=${sourceMapBaseUrl}/${relativePath}.map*/` + ); + } + await fs.promises.writeFile(file.path, content); } else { - // Write other files (source maps, etc.) as-is + // Write other files (source maps, assets) as-is await fs.promises.writeFile(file.path, file.contents); } } @@ -1011,40 +1025,34 @@ async function watch(): Promise { // Extensions to watch and copy (non-TypeScript resources) const copyExtensions = ['.css', '.html', '.js', '.json', '.ttf', '.svg', '.png', '.mp3', '.scm', '.sh', '.ps1', '.psm1', '.fish', '.zsh', '.scpt']; - // Watch src directory - const subscription = await watcher.subscribe( - path.join(REPO_ROOT, SRC_DIR), - (err, events) => { - if (err) { - console.error('[watch] Watcher error:', err); - return; - } + // Watch src directory using existing gulp-watch based watcher + let debounceTimer: ReturnType | undefined; + const srcDir = path.join(REPO_ROOT, SRC_DIR); + const watchStream = gulpWatch('src/**', { base: srcDir, readDelay: 200 }); - for (const event of events) { - if (event.path.includes('/test/')) { - continue; - } + watchStream.on('data', (file: { path: string }) => { + if (file.path.includes('/test/')) { + return; + } - if (event.path.endsWith('.ts') && !event.path.endsWith('.d.ts')) { - pendingTsFiles.add(event.path); - } else if (copyExtensions.some(ext => event.path.endsWith(ext))) { - pendingCopyFiles.add(event.path); - } - } + if (file.path.endsWith('.ts') && !file.path.endsWith('.d.ts')) { + pendingTsFiles.add(file.path); + } else if (copyExtensions.some(ext => file.path.endsWith(ext))) { + pendingCopyFiles.add(file.path); + } - if (pendingTsFiles.size > 0 || pendingCopyFiles.size > 0) { - processChanges(); - } - }, - { ignore: ['**/test/**', '**/node_modules/**'] } - ); + if (pendingTsFiles.size > 0 || pendingCopyFiles.size > 0) { + clearTimeout(debounceTimer); + debounceTimer = setTimeout(processChanges, 200); + } + }); console.log('[watch] Watching src/**/*.{ts,css,...} (Ctrl+C to stop)'); // Keep process alive - process.on('SIGINT', async () => { + process.on('SIGINT', () => { console.log('\n[watch] Stopping...'); - await subscription.unsubscribe(); + watchStream.end(); process.exit(0); }); } @@ -1070,6 +1078,7 @@ Options for 'bundle': --nls Process NLS (localization) strings --out Output directory (default: out-vscode) --target Build target: desktop (default), server, server-web, web + --source-map-base-url Rewrite sourceMappingURL to CDN URL Examples: npx tsx build/next/index.ts transpile @@ -1110,7 +1119,7 @@ async function main(): Promise { break; case 'bundle': - await bundle(options.out ?? OUT_VSCODE_DIR, options.minify, options.nls, options.target as BuildTarget); + await bundle(options.out ?? OUT_VSCODE_DIR, options.minify, options.nls, options.target as BuildTarget, options.sourceMapBaseUrl); break; default: diff --git a/build/next/working.md b/build/next/working.md index 56a6aafbad0..ac466fce548 100644 --- a/build/next/working.md +++ b/build/next/working.md @@ -168,6 +168,36 @@ npm run gulp vscode-reh-web-darwin-arm64-min --- +## Source Maps + +### Fixes Applied + +1. **`sourcesContent: true`** — Production bundles now embed original TypeScript source content in `.map` files, matching the old build's `includeContent: true` behavior. Without this, crash reports from CDN-hosted source maps can't show original source. + +2. **`--source-map-base-url` option** — The `bundle` command accepts an optional `--source-map-base-url ` flag. When set, post-processing rewrites `sourceMappingURL` comments in `.js` and `.css` output files to point to the CDN (e.g., `https://main.vscode-cdn.net/sourcemaps//core/vs/...`). This matches the old build's `sourceMappingURL` function in `minifyTask()`. Wired up in `gulpfile.vscode.ts` for `core-ci-esbuild` and `vscode-esbuild-min` tasks. + +### NLS Source Map Accuracy (Decision: Accept Imprecision) + +**Problem:** `postProcessNLS()` replaces `"%%NLS:moduleId#key%%"` placeholders (~40 chars) with short index values like `null` (4 chars) in the final JS output. This shifts column positions without updating the `.map` files. + +**Options considered:** + +| Option | Description | Effort | Accuracy | +|--------|-------------|--------|----------| +| A. Fixed-width placeholders | Pad placeholders to match replacement length | Hard — indices unknown until all modules are collected across parallel bundles | Perfect | +| B. Post-process source map | Parse `.map`, track replacement offsets per line, adjust VLQ mappings | Medium | Perfect | +| C. Two-pass build | Assign NLS indices during plugin phase | Not feasible with parallel bundling | N/A | +| **D. Accept imprecision** | NLS replacements only affect column positions; line-level debugging works | Zero | Line-level | + +**Decision: Option D — accept imprecision.** Rationale: + +- NLS replacements only shift **columns**, never lines — line-level stack traces and breakpoints remain correct. +- Production crash reporting (the primary consumer of CDN source maps) uses line numbers; column-level accuracy is rarely needed. +- The old gulp build had the same fundamental issue in its `nls.nls()` step and used `SourceMapConsumer`/`SourceMapGenerator` to fix it — but that approach was fragile and slow. +- If column-level precision becomes important later (e.g., for minified+NLS bundles), Option B can be implemented: after NLS replacement, re-parse the source map, walk replacement sites, and adjust column offsets. This is a localized change in the post-processing loop. + +--- + ## Self-hosting Setup The default `VS Code - Build` task now runs three parallel watchers: