feat: reenable asar support (#272552)

build/gulpfile.vscode.js

@@ -288,14 +288,17 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
 		const productionDependencies = getProductionDependencies(root);
 		const dependenciesSrc = productionDependencies.map(d => path.relative(root, d)).map(d => [`${d}/**`, `!${d}/**/{test,tests}/**`]).flat().concat('!**/*.mk');
 
-		const deps = gulp.src(dependenciesSrc, { base: '.', dot: true })
+		const deps = es.merge(
+			gulp.src(dependenciesSrc, { base: '.', dot: true }),
+			gulp.src(['node_modules/vsda/**'], { base: 'node_modules', dot: true }) // retain vsda at root level of asar for backward compatibility
+		)
 			.pipe(filter(['**', `!**/${config.version}/**`, '!**/bin/darwin-arm64-87/**', '!**/package-lock.json', '!**/yarn.lock', '!**/*.{js,css}.map']))
 			.pipe(util.cleanNodeModules(path.join(__dirname, '.moduleignore')))
 			.pipe(util.cleanNodeModules(path.join(__dirname, `.moduleignore.${process.platform}`)))
 			.pipe(jsFilter)
 			.pipe(util.rewriteSourceMappingURL(sourceMappingURLBase))
 			.pipe(jsFilter.restore)
-			.pipe(createAsar(path.join(process.cwd(), 'node_modules'), [
+			.pipe(createAsar(process.cwd(), [
 				'**/*.node',
 				'**/@vscode/ripgrep/bin/*',
 				'**/node-pty/build/Release/*',
@@ -306,9 +309,8 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
 				'**/@vscode/vsce-sign/bin/*',
 			], [
 				'**/*.mk',
-				'!node_modules/vsda/**' // stay compatible with extensions that depend on us shipping `vsda` into ASAR
 			], [
-				'node_modules/vsda/**' // retain copy of `vsda` in node_modules for internal use
+				'node_modules/vsda/**', // duplicate vsda in node_modules.asar.unpacked for backward compatibility
 			], 'node_modules.asar'));
 
 		let all = es.merge(

build/lib/asar.js

@@ -32,7 +32,7 @@ function createAsar(folderPath, unpackGlobs, skipGlobs, duplicateGlobs, destFile
         return false;
     };
     // Files that should be duplicated between
-    // node_modules.asar and node_modules
+    // node_modules.asar.unpacked/node_modules and node_modules.asar.unpacked
     const shouldDuplicateFile = (file) => {
         for (const duplicateGlob of duplicateGlobs) {
             if ((0, minimatch_1.default)(file.relative, duplicateGlob)) {
@@ -94,14 +94,6 @@ function createAsar(folderPath, unpackGlobs, skipGlobs, duplicateGlobs, destFile
             }));
             return;
         }
-        if (shouldDuplicateFile(file)) {
-            this.queue(new vinyl_1.default({
-                base: '.',
-                path: file.path,
-                stat: file.stat,
-                contents: file.contents
-            }));
-        }
         const shouldUnpack = shouldUnpackFile(file);
         insertFile(file.relative, { size: file.contents.length, mode: file.stat.mode }, shouldUnpack);
         if (shouldUnpack) {
@@ -113,6 +105,16 @@ function createAsar(folderPath, unpackGlobs, skipGlobs, duplicateGlobs, destFile
                 stat: file.stat,
                 contents: file.contents
             }));
+            const shouldDuplicate = shouldDuplicateFile(file);
+            if (shouldDuplicate) {
+                const rootRelative = file.relative.replace(/^node_modules\//, '');
+                this.queue(new vinyl_1.default({
+                    base: '.',
+                    path: path_1.default.join(destFilename + '.unpacked', rootRelative),
+                    stat: file.stat,
+                    contents: file.contents
+                }));
+            }
         }
         else {
             // The file goes inside of xx.asar

build/lib/asar.ts

@@ -38,7 +38,7 @@ export function createAsar(folderPath: string, unpackGlobs: string[], skipGlobs:
 	};
 
 	// Files that should be duplicated between
-	// node_modules.asar and node_modules
+	// node_modules.asar.unpacked/node_modules and node_modules.asar.unpacked
 	const shouldDuplicateFile = (file: VinylFile): boolean => {
 		for (const duplicateGlob of duplicateGlobs) {
 			if (minimatch(file.relative, duplicateGlob)) {
@@ -107,14 +107,7 @@ export function createAsar(folderPath: string, unpackGlobs: string[], skipGlobs:
 			}));
 			return;
 		}
-		if (shouldDuplicateFile(file)) {
-			this.queue(new VinylFile({
-				base: '.',
-				path: file.path,
-				stat: file.stat,
-				contents: file.contents
-			}));
-		}
+
 		const shouldUnpack = shouldUnpackFile(file);
 		insertFile(file.relative, { size: file.contents.length, mode: file.stat.mode }, shouldUnpack);
 
@@ -127,6 +120,17 @@ export function createAsar(folderPath: string, unpackGlobs: string[], skipGlobs:
 				stat: file.stat,
 				contents: file.contents
 			}));
+
+			const shouldDuplicate = shouldDuplicateFile(file);
+			if (shouldDuplicate) {
+				const rootRelative = file.relative.replace(/^node_modules\//, '');
+				this.queue(new VinylFile({
+					base: '.',
+					path: path.join(destFilename + '.unpacked', rootRelative),
+					stat: file.stat,
+					contents: file.contents
+				}));
+			}
 		} else {
 			// The file goes inside of xx.asar
 			out.push(file.contents);

build/linux/dependencies-generator.js

@@ -46,8 +46,7 @@ async function getDependencies(packageType, buildDir, applicationName, arch) {
         throw new Error('Invalid RPM arch string ' + arch);
     }
     // Get the files for which we want to find dependencies.
-    const canAsar = false; // TODO@esm ASAR disabled in ESM
-    const nativeModulesPath = path_1.default.join(buildDir, 'resources', 'app', canAsar ? 'node_modules.asar.unpacked' : 'node_modules');
+    const nativeModulesPath = path_1.default.join(buildDir, 'resources', 'app', 'node_modules.asar.unpacked');
     const findResult = (0, child_process_1.spawnSync)('find', [nativeModulesPath, '-name', '*.node']);
     if (findResult.status) {
         console.error('Error finding files:');

build/linux/dependencies-generator.ts

@@ -47,8 +47,7 @@ export async function getDependencies(packageType: 'deb' | 'rpm', buildDir: stri
 	}
 
 	// Get the files for which we want to find dependencies.
-	const canAsar = false; // TODO@esm ASAR disabled in ESM
-	const nativeModulesPath = path.join(buildDir, 'resources', 'app', canAsar ? 'node_modules.asar.unpacked' : 'node_modules');
+	const nativeModulesPath = path.join(buildDir, 'resources', 'app', 'node_modules.asar.unpacked');
 	const findResult = spawnSync('find', [nativeModulesPath, '-name', '*.node']);
 	if (findResult.status) {
 		console.error('Error finding files:');

src/bootstrap-esm.ts

@@ -5,34 +5,74 @@
 
 import * as fs from 'node:fs';
 import { register } from 'node:module';
+import { sep } from 'node:path';
 import { product, pkg } from './bootstrap-meta.js';
 import './bootstrap-node.js';
 import * as performance from './vs/base/common/performance.js';
 import { INLSConfiguration } from './vs/nls.js';
 
-// Install a hook to module resolution to map 'fs' to 'original-fs'
-if (process.env['ELECTRON_RUN_AS_NODE'] || process.versions['electron']) {
+// Prepare globals that are needed for running
+globalThis._VSCODE_PRODUCT_JSON = { ...product };
+globalThis._VSCODE_PACKAGE_JSON = { ...pkg };
+globalThis._VSCODE_FILE_ROOT = import.meta.dirname;
+
+// Install a hook to module resolution to map dependencies into the asar archive
+function enableASARSupport() {
+	if (!process.env['ELECTRON_RUN_AS_NODE'] && !process.versions['electron']) {
+		return;
+	}
+
+	if (process.env['VSCODE_DEV']) {
+		return;
+	}
+
 	const jsCode = `
+	import { pathToFileURL, fileURLToPath } from 'node:url';
+	function isRelativeSpecifier(specifier) {
+		if (specifier[0] === '.') {
+			if (specifier.length === 1 || specifier[1] === '/') { return true; }
+			if (specifier[1] === '.') {
+				if (specifier.length === 2 || specifier[2] === '/') { return true; }
+			}
+		}
+		return false;
+	}
+	function normalizeDriveLetter(path) {
+		if (process.platform === 'win32'
+			&& path.length >= 2
+			&& (path.charCodeAt(0) >= 65 && path.charCodeAt(0) <= 90 || path.charCodeAt(0) >= 97 && path.charCodeAt(0) <= 122)
+			&& path.charCodeAt(1) === 58) {
+			return path[0].toLowerCase() + path.slice(1);
+		}
+		return path;
+	}
+	export async function initialize({ resourcesPath, asarPath }) {
+		globalThis.__resourcesPath = normalizeDriveLetter(resourcesPath);
+		globalThis.__asarPath = asarPath;
+	}
 	export async function resolve(specifier, context, nextResolve) {
-		if (specifier === 'fs') {
-			return {
-				format: 'builtin',
-				shortCircuit: true,
-				url: 'node:original-fs'
-			};
+		if (!isRelativeSpecifier(specifier) && context.parentURL) {
+			const currentPath = fileURLToPath(context.parentURL);
+			const normalizedCurrentPath = normalizeDriveLetter(currentPath);
+			if (normalizedCurrentPath.startsWith(globalThis.__resourcesPath)) {
+				const asarPath = normalizedCurrentPath.replace(globalThis.__resourcesPath, globalThis.__asarPath);
+				context.parentURL = pathToFileURL(asarPath);
+			}
 		}
 
 		// Defer to the next hook in the chain, which would be the
 		// Node.js default resolve if this is the last user-specified loader.
 		return nextResolve(specifier, context);
 	}`;
-	register(`data:text/javascript;base64,${Buffer.from(jsCode).toString('base64')}`, import.meta.url);
+	register(`data:text/javascript;base64,${Buffer.from(jsCode).toString('base64')}`, import.meta.url, {
+		data: {
+			resourcesPath: `${process.resourcesPath}${sep}app`,
+			asarPath: `${process.resourcesPath}${sep}app${sep}node_modules.asar`,
+		}
+	});
 }
 
-// Prepare globals that are needed for running
-globalThis._VSCODE_PRODUCT_JSON = { ...product };
-globalThis._VSCODE_PACKAGE_JSON = { ...pkg };
-globalThis._VSCODE_FILE_ROOT = import.meta.dirname;
+enableASARSupport();
 
 //#region NLS helpers
 

src/bootstrap-node.ts

@@ -29,6 +29,34 @@ if (!process.env['VSCODE_HANDLES_SIGPIPE']) {
 	});
 }
 
+/**
+ * Helper to enable ASAR support.
+ */
+function enableASARSupport(): void {
+	if (process.env['ELECTRON_RUN_AS_NODE'] || process.versions['electron']) {
+		const Module = require('node:module');
+		const NODE_MODULES_PATH = path.join(import.meta.dirname, '../node_modules');
+		const NODE_MODULES_ASAR_PATH = path.join(import.meta.dirname, '../node_modules.asar');
+		// @ts-ignore
+		const originalResolveLookupPaths = Module._resolveLookupPaths;
+		// @ts-ignore
+		Module._resolveLookupPaths = function (request, parent) {
+			const paths = originalResolveLookupPaths(request, parent);
+			if (Array.isArray(paths)) {
+				for (let i = 0, len = paths.length; i < len; i++) {
+					if (paths[i] === NODE_MODULES_PATH) {
+						paths.splice(i, 0, NODE_MODULES_ASAR_PATH);
+						break;
+					}
+				}
+			}
+			return paths;
+		};
+	}
+}
+
+enableASARSupport();
+
 // Setup current working directory in all our node & electron processes
 // - Windows: call `process.chdir()` to always set application folder as cwd
 // -  all OS: store the `process.cwd()` inside `VSCODE_CWD` for consistent lookups

src/vs/amdX.ts

@@ -9,8 +9,6 @@ import { IProductConfiguration } from './base/common/product.js';
 import { URI } from './base/common/uri.js';
 import { generateUuid } from './base/common/uuid.js';
 
-export const canASAR = false; // TODO@esm: ASAR disabled in ESM
-
 declare const window: any;
 declare const document: any;
 declare const self: any;
@@ -218,7 +216,7 @@ export async function importAMDNodeModule<T>(nodeModuleName: string, pathInsideN
 		// bit of a special case for: src/vs/workbench/services/languageDetection/browser/languageDetectionWebWorker.ts
 		scriptSrc = nodeModulePath;
 	} else {
-		const useASAR = (canASAR && isBuilt && !platform.isWeb);
+		const useASAR = (isBuilt && (platform.isElectron || (platform.isWebWorker && platform.hasElectronUserAgent)));
 		const actualNodeModulesPath = (useASAR ? nodeModulesAsarPath : nodeModulesPath);
 		const resourcePath: AppResourcePath = `${actualNodeModulesPath}/${nodeModulePath}`;
 		scriptSrc = FileAccess.asBrowserUri(resourcePath).toString(true);
@@ -231,7 +229,7 @@ export async function importAMDNodeModule<T>(nodeModuleName: string, pathInsideN
 export function resolveAmdNodeModulePath(nodeModuleName: string, pathInsideNodeModule: string): string {
 	const product = globalThis._VSCODE_PRODUCT_JSON as unknown as IProductConfiguration;
 	const isBuilt = Boolean((product ?? globalThis.vscode?.context?.configuration()?.product)?.commit);
-	const useASAR = (canASAR && isBuilt && !platform.isWeb);
+	const useASAR = (isBuilt && (platform.isElectron || (platform.isWebWorker && platform.hasElectronUserAgent)));
 
 	const nodeModulePath = `${nodeModuleName}/${pathInsideNodeModule}`;
 	const actualNodeModulesPath = (useASAR ? nodeModulesAsarPath : nodeModulesPath);

src/vs/base/common/network.ts

@@ -257,8 +257,8 @@ export type AppResourcePath = (
 
 export const builtinExtensionsPath: AppResourcePath = 'vs/../../extensions';
 export const nodeModulesPath: AppResourcePath = 'vs/../../node_modules';
-export const nodeModulesAsarPath: AppResourcePath = 'vs/../../node_modules.asar';
-export const nodeModulesAsarUnpackedPath: AppResourcePath = 'vs/../../node_modules.asar.unpacked';
+export const nodeModulesAsarPath: AppResourcePath = 'vs/../../node_modules.asar/node_modules';
+export const nodeModulesAsarUnpackedPath: AppResourcePath = 'vs/../../node_modules.asar.unpacked/node_modules';
 
 export const VSCODE_AUTHORITY = 'vscode-app';
 

src/vs/base/common/platform.ts

@@ -274,6 +274,7 @@ export const isFirefox = !!(userAgent && userAgent.indexOf('Firefox') >= 0);
 export const isSafari = !!(!isChrome && (userAgent && userAgent.indexOf('Safari') >= 0));
 export const isEdge = !!(userAgent && userAgent.indexOf('Edg/') >= 0);
 export const isAndroid = !!(userAgent && userAgent.indexOf('Android') >= 0);
+export const hasElectronUserAgent = !!(userAgent && userAgent.indexOf('Electron') >= 0);
 
 export function isBigSurOrNewer(osVersion: string): boolean {
 	return parseFloat(osVersion) >= 20;

src/vs/workbench/services/languageDetection/browser/languageDetectionWorkerServiceImpl.ts

@@ -21,7 +21,6 @@ import { IEditorService } from '../../editor/common/editorService.js';
 import { IStorageService, StorageScope, StorageTarget } from '../../../../platform/storage/common/storage.js';
 import { LRUCache } from '../../../../base/common/map.js';
 import { ILogService } from '../../../../platform/log/common/log.js';
-import { canASAR } from '../../../../amdX.js';
 import { createWebWorker } from '../../../../base/browser/webWorkerFactory.js';
 import { WorkerTextModelSyncClient } from '../../../../editor/common/services/textModelSync/textModelSync.impl.js';
 import { ILanguageDetectionWorker, LanguageDetectionWorkerHost } from './languageDetectionWorker.protocol.js';
@@ -66,7 +65,7 @@ export class LanguageDetectionService extends Disposable implements ILanguageDet
 	) {
 		super();
 
-		const useAsar = canASAR && this._environmentService.isBuilt && !isWeb;
+		const useAsar = this._environmentService.isBuilt && !isWeb;
 		this._languageDetectionWorkerClient = this._register(new LanguageDetectionWorkerClient(
 			modelService,
 			languageService,

src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.ts

@@ -3,7 +3,6 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { canASAR } from '../../../../../amdX.js';
 import { DisposableStore, IDisposable, toDisposable } from '../../../../../base/common/lifecycle.js';
 import { AppResourcePath, FileAccess, nodeModulesAsarPath, nodeModulesPath } from '../../../../../base/common/network.js';
 import { IObservable } from '../../../../../base/common/observable.js';
@@ -129,7 +128,7 @@ export class ThreadedBackgroundTokenizerFactory implements IDisposable {
 		const onigurumaModuleLocation: AppResourcePath = `${nodeModulesPath}/vscode-oniguruma`;
 		const onigurumaModuleLocationAsar: AppResourcePath = `${nodeModulesAsarPath}/vscode-oniguruma`;
 
-		const useAsar = canASAR && this._environmentService.isBuilt && !isWeb;
+		const useAsar = this._environmentService.isBuilt && !isWeb;
 		const onigurumaLocation: AppResourcePath = useAsar ? onigurumaModuleLocationAsar : onigurumaModuleLocation;
 		const onigurumaWASM: AppResourcePath = `${onigurumaLocation}/release/onig.wasm`;
 

src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { canASAR, importAMDNodeModule, resolveAmdNodeModulePath } from '../../../../amdX.js';
+import { importAMDNodeModule, resolveAmdNodeModulePath } from '../../../../amdX.js';
 import * as domStylesheets from '../../../../base/browser/domStylesheets.js';
 import { equals as equalArray } from '../../../../base/common/arrays.js';
 import { Color } from '../../../../base/common/color.js';
@@ -390,7 +390,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate
 			// We therefore use the non-streaming compiler :(.
 			return await response.arrayBuffer();
 		} else {
-			const response = await fetch(canASAR && this._environmentService.isBuilt
+			const response = await fetch(this._environmentService.isBuilt
 				? FileAccess.asBrowserUri(`${nodeModulesAsarUnpackedPath}/vscode-oniguruma/release/onig.wasm`).toString(true)
 				: FileAccess.asBrowserUri(`${nodeModulesPath}/vscode-oniguruma/release/onig.wasm`).toString(true));
 			return response;

src/vs/workbench/services/treeSitter/browser/treeSitterLibraryService.ts

@@ -7,7 +7,7 @@
 import type { Parser, Language, Query } from '@vscode/tree-sitter-wasm';
 import { IReader, ObservablePromise } from '../../../../base/common/observable.js';
 import { ITreeSitterLibraryService } from '../../../../editor/common/services/treeSitter/treeSitterLibraryService.js';
-import { canASAR, importAMDNodeModule } from '../../../../amdX.js';
+import { importAMDNodeModule } from '../../../../amdX.js';
 import { Lazy } from '../../../../base/common/lazy.js';
 import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js';
 import { FileOperationResult, IFileContent, IFileService, toFileOperationResult } from '../../../../platform/files/common/files.js';
@@ -25,7 +25,7 @@ const MODULE_LOCATION_SUBPATH = `@vscode/tree-sitter-wasm/wasm`;
 const FILENAME_TREESITTER_WASM = `tree-sitter.wasm`;
 
 export function getModuleLocation(environmentService: IEnvironmentService): AppResourcePath {
-	return `${(canASAR && environmentService.isBuilt) ? nodeModulesAsarUnpackedPath : nodeModulesPath}/${MODULE_LOCATION_SUBPATH}`;
+	return `${environmentService.isBuilt ? nodeModulesAsarUnpackedPath : nodeModulesPath}/${MODULE_LOCATION_SUBPATH}`;
 }
 
 export class TreeSitterLibraryService extends Disposable implements ITreeSitterLibraryService {