JS/TS package acquisition (#184438)

* Experiment with adding ata using `@types` packages shipped in an extension

* Use own file system instead of `https`

* JS/TS type support on web

* Tsconfig needs esModuleInterop not module:nodenext

We actually want webpack to emit commonjs, but need to write ES default
imports to use node-maintainer

* fix package.json indentation

* Adding setting to disable web type acquisition

* Fix merge of yarn lock

* Fixing merge errors

* Fixing errors

* Pick up package externally

* Fixing conflicts

* Bump version

---------

Co-authored-by: Kat Marchán <kzm@zkat.tech>
Co-authored-by: Nathan Shively-Sanders <293473+sandersn@users.noreply.github.com>

extensions/typescript-language-features/src/configuration/configuration.ts

@@ -112,6 +112,7 @@ export interface TypeScriptServiceConfiguration {
 	readonly useSyntaxServer: SyntaxServerConfiguration;
 	readonly webProjectWideIntellisenseEnabled: boolean;
 	readonly webProjectWideIntellisenseSuppressSemanticErrors: boolean;
+	readonly webExperimentalTypeAcquisition: boolean;
 	readonly enableDiagnosticsTelemetry: boolean;
 	readonly enableProjectDiagnostics: boolean;
 	readonly maxTsServerMemory: number;
@@ -145,6 +146,7 @@ export abstract class BaseServiceConfigurationProvider implements ServiceConfigu
 			useSyntaxServer: this.readUseSyntaxServer(configuration),
 			webProjectWideIntellisenseEnabled: this.readWebProjectWideIntellisenseEnable(configuration),
 			webProjectWideIntellisenseSuppressSemanticErrors: this.readWebProjectWideIntellisenseSuppressSemanticErrors(configuration),
+			webExperimentalTypeAcquisition: this.readWebExperimentalTypeAcquisition(configuration),
 			enableDiagnosticsTelemetry: this.readEnableDiagnosticsTelemetry(configuration),
 			enableProjectDiagnostics: this.readEnableProjectDiagnostics(configuration),
 			maxTsServerMemory: this.readMaxTsServerMemory(configuration),
@@ -175,6 +177,10 @@ export abstract class BaseServiceConfigurationProvider implements ServiceConfigu
 		return configuration.get<boolean>('typescript.disableAutomaticTypeAcquisition', false);
 	}
 
+	protected readWebExperimentalTypeAcquisition(configuration: vscode.WorkspaceConfiguration): boolean {
+		return configuration.get<boolean>('typescript.experimental.tsserver.web.typeAcquisition.enabled', false);
+	}
+
 	protected readLocale(configuration: vscode.WorkspaceConfiguration): string | null {
 		const value = configuration.get<string>('typescript.locale', 'auto');
 		return !value || value === 'auto' ? null : value;

extensions/typescript-language-features/src/extension.browser.ts

@@ -8,22 +8,24 @@ import * as vscode from 'vscode';
 import { Api, getExtensionApi } from './api';
 import { CommandManager } from './commands/commandManager';
 import { registerBaseCommands } from './commands/index';
+import { TypeScriptServiceConfiguration } from './configuration/configuration';
+import { BrowserServiceConfigurationProvider } from './configuration/configuration.browser';
 import { ExperimentationTelemetryReporter, IExperimentationTelemetryReporter } from './experimentTelemetryReporter';
+import { AutoInstallerFs } from './filesystems/autoInstallerFs';
+import { MemFs } from './filesystems/memFs';
 import { createLazyClientHost, lazilyActivateClient } from './lazyClientHost';
+import { Logger } from './logging/logger';
 import RemoteRepositories from './remoteRepositories.browser';
 import { API } from './tsServer/api';
 import { noopRequestCancellerFactory } from './tsServer/cancellation';
 import { noopLogDirectoryProvider } from './tsServer/logDirectoryProvider';
+import { PluginManager } from './tsServer/plugins';
 import { WorkerServerProcessFactory } from './tsServer/serverProcess.browser';
 import { ITypeScriptVersionProvider, TypeScriptVersion, TypeScriptVersionSource } from './tsServer/versionProvider';
 import { ActiveJsTsEditorTracker } from './ui/activeJsTsEditorTracker';
-import { TypeScriptServiceConfiguration } from './configuration/configuration';
-import { BrowserServiceConfigurationProvider } from './configuration/configuration.browser';
-import { Logger } from './logging/logger';
+import { Disposable } from './utils/dispose';
 import { getPackageInfo } from './utils/packageInfo';
 import { isWebAndHasSharedArrayBuffers } from './utils/platform';
-import { PluginManager } from './tsServer/plugins';
-import { Disposable } from './utils/dispose';
 
 class StaticVersionProvider implements ITypeScriptVersionProvider {
 
@@ -99,6 +101,14 @@ export async function activate(context: vscode.ExtensionContext): Promise<Api> {
 	context.subscriptions.push(lazilyActivateClient(lazyClientHost, pluginManager, activeJsTsEditorTracker, async () => {
 		await startPreloadWorkspaceContentsIfNeeded(context, logger);
 	}));
+	context.subscriptions.push(vscode.workspace.registerFileSystemProvider('vscode-global-typings', new MemFs(), {
+		isCaseSensitive: true,
+		isReadonly: false
+	}));
+	context.subscriptions.push(vscode.workspace.registerFileSystemProvider('vscode-node-modules', new AutoInstallerFs(), {
+		isCaseSensitive: true,
+		isReadonly: false
+	}));
 
 	return getExtensionApi(onCompletionAccepted.event, pluginManager);
 }

extensions/typescript-language-features/src/filesystems/autoInstallerFs.ts

@@ -0,0 +1,252 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import { MemFs } from './memFs';
+import { URI } from 'vscode-uri';
+import { PackageManager, FileSystem, packagePath } from '@vscode/ts-package-manager';
+import { join, basename, dirname } from 'path';
+
+const TEXT_DECODER = new TextDecoder('utf-8');
+const TEXT_ENCODER = new TextEncoder();
+
+export class AutoInstallerFs implements vscode.FileSystemProvider {
+
+	private readonly memfs = new MemFs();
+	private readonly fs: FileSystem;
+	private readonly projectCache = new Map<string, Set<string>>();
+	private readonly watcher: vscode.FileSystemWatcher;
+	private readonly _emitter = new vscode.EventEmitter<vscode.FileChangeEvent[]>();
+
+	readonly onDidChangeFile: vscode.Event<vscode.FileChangeEvent[]> = this._emitter.event;
+
+	constructor() {
+		this.watcher = vscode.workspace.createFileSystemWatcher('**/{package.json,package-lock.json,package-lock.kdl}');
+		const handler = (uri: URI) => {
+			const root = dirname(uri.path);
+			if (this.projectCache.delete(root)) {
+				(async () => {
+					const pm = new PackageManager(this.fs);
+					const opts = await this.getInstallOpts(uri, root);
+					const proj = await pm.resolveProject(root, opts);
+					proj.pruneExtraneous();
+					// TODO: should this fire on vscode-node-modules instead?
+					// NB(kmarchan): This should tell TSServer that there's
+					// been changes inside node_modules and it needs to
+					// re-evaluate things.
+					this._emitter.fire([{
+						type: vscode.FileChangeType.Changed,
+						uri: uri.with({ path: join(root, 'node_modules') })
+					}]);
+				})();
+			}
+		};
+		this.watcher.onDidChange(handler);
+		this.watcher.onDidCreate(handler);
+		this.watcher.onDidDelete(handler);
+		const memfs = this.memfs;
+		memfs.onDidChangeFile((e) => {
+			this._emitter.fire(e.map(ev => ({
+				type: ev.type,
+				// TODO: we're gonna need a MappedUri dance...
+				uri: ev.uri.with({ scheme: 'memfs' })
+			})));
+		});
+		this.fs = {
+			readDirectory(path: string, _extensions?: readonly string[], _exclude?: readonly string[], _include?: readonly string[], _depth?: number): string[] {
+				return memfs.readDirectory(URI.file(path)).map(([name, _]) => name);
+			},
+
+			deleteFile(path: string): void {
+				memfs.delete(URI.file(path));
+			},
+
+			createDirectory(path: string): void {
+				memfs.createDirectory(URI.file(path));
+			},
+
+			writeFile(path: string, data: string, _writeByteOrderMark?: boolean): void {
+				memfs.writeFile(URI.file(path), TEXT_ENCODER.encode(data), { overwrite: true, create: true });
+			},
+
+			directoryExists(path: string): boolean {
+				try {
+					const stat = memfs.stat(URI.file(path));
+					return stat.type === vscode.FileType.Directory;
+				} catch (e) {
+					return false;
+				}
+			},
+
+			readFile(path: string, _encoding?: string): string | undefined {
+				try {
+					return TEXT_DECODER.decode(memfs.readFile(URI.file(path)));
+				} catch (e) {
+					return undefined;
+				}
+			}
+		};
+	}
+
+	watch(resource: vscode.Uri): vscode.Disposable {
+		const mapped = URI.file(new MappedUri(resource).path);
+		console.log('watching', mapped);
+		return this.memfs.watch(mapped);
+	}
+
+	async stat(uri: vscode.Uri): Promise<vscode.FileStat> {
+		// console.log('stat', uri.toString());
+		const mapped = new MappedUri(uri);
+
+		// TODO: case sensitivity configuration
+
+		// We pretend every single node_modules or @types directory ever actually
+		// exists.
+		if (basename(mapped.path) === 'node_modules' || basename(mapped.path) === '@types') {
+			return {
+				mtime: 0,
+				ctime: 0,
+				type: vscode.FileType.Directory,
+				size: 0
+			};
+		}
+
+		await this.ensurePackageContents(mapped);
+
+		return this.memfs.stat(URI.file(mapped.path));
+	}
+
+	async readDirectory(uri: vscode.Uri): Promise<[string, vscode.FileType][]> {
+		// console.log('readDirectory', uri.toString());
+		const mapped = new MappedUri(uri);
+		await this.ensurePackageContents(mapped);
+
+		return this.memfs.readDirectory(URI.file(mapped.path));
+	}
+
+	async readFile(uri: vscode.Uri): Promise<Uint8Array> {
+		// console.log('readFile', uri.toString());
+		const mapped = new MappedUri(uri);
+		await this.ensurePackageContents(mapped);
+
+		return this.memfs.readFile(URI.file(mapped.path));
+	}
+
+	writeFile(_uri: vscode.Uri, _content: Uint8Array, _options: { create: boolean; overwrite: boolean }): void {
+		throw new Error('not implemented');
+	}
+
+	rename(_oldUri: vscode.Uri, _newUri: vscode.Uri, _options: { overwrite: boolean }): void {
+		throw new Error('not implemented');
+	}
+
+	delete(_uri: vscode.Uri): void {
+		throw new Error('not implemented');
+	}
+
+	createDirectory(_uri: vscode.Uri): void {
+		throw new Error('not implemented');
+	}
+
+	private async ensurePackageContents(incomingUri: MappedUri): Promise<void> {
+		// console.log('ensurePackageContents', incomingUri.path);
+
+		// If we're not looking for something inside node_modules, bail early.
+		if (!incomingUri.path.includes('node_modules')) {
+			throw vscode.FileSystemError.FileNotFound();
+		}
+
+		// standard lib files aren't handled through here
+		if (incomingUri.path.includes('node_modules/@typescript') || incomingUri.path.includes('node_modules/@types/typescript__')) {
+			throw vscode.FileSystemError.FileNotFound();
+		}
+
+		const root = this.getProjectRoot(incomingUri.path);
+
+		const pkgPath = packagePath(incomingUri.path);
+		if (!root || this.projectCache.get(root)?.has(pkgPath)) {
+			return;
+		}
+
+		const proj = await (new PackageManager(this.fs)).resolveProject(root, await this.getInstallOpts(incomingUri.original, root));
+
+		const restore = proj.restorePackageAt(incomingUri.path);
+		try {
+			await restore;
+		} catch (e) {
+			console.error(`failed to restore package at ${incomingUri.path}: `, e);
+			throw e;
+		}
+		if (!this.projectCache.has(root)) {
+			this.projectCache.set(root, new Set());
+		}
+		this.projectCache.get(root)!.add(pkgPath);
+	}
+
+	private async getInstallOpts(originalUri: URI, root: string) {
+		const vsfs = vscode.workspace.fs;
+		let pkgJson;
+		try {
+			pkgJson = TEXT_DECODER.decode(await vsfs.readFile(originalUri.with({ path: join(root, 'package.json') })));
+		} catch (e) { }
+
+		let kdlLock;
+		try {
+			kdlLock = TEXT_DECODER.decode(await vsfs.readFile(originalUri.with({ path: join(root, 'package-lock.kdl') })));
+		} catch (e) { }
+
+		let npmLock;
+		try {
+			npmLock = TEXT_DECODER.decode(await vsfs.readFile(originalUri.with({ path: join(root, 'package-lock.json') })));
+		} catch (e) { }
+
+		return {
+			pkgJson,
+			kdlLock,
+			npmLock
+		};
+	}
+
+	private getProjectRoot(path: string): string | undefined {
+		const pkgPath = path.match(/(^.*)\/node_modules/);
+		return pkgPath?.[1];
+	}
+
+	// --- manage file events
+
+}
+
+class MappedUri {
+	readonly raw: vscode.Uri;
+	readonly original: vscode.Uri;
+	readonly mapped: vscode.Uri;
+	constructor(uri: vscode.Uri) {
+		this.raw = uri;
+
+		const parts = uri.path.match(/^\/([^\/]+)\/([^\/]*)(?:\/(.+))?$/);
+		if (!parts) {
+			throw new Error(`Invalid path: ${uri.path}`);
+		}
+
+		const scheme = parts[1];
+		const authority = parts[2] === 'ts-nul-authority' ? '' : parts[2];
+		const path = parts[3];
+		this.original = URI.from({ scheme, authority, path: (path ? '/' + path : path) });
+		this.mapped = this.original.with({ scheme: this.raw.scheme, authority: this.raw.authority });
+	}
+
+	get path() {
+		return this.mapped.path;
+	}
+	get scheme() {
+		return this.mapped.scheme;
+	}
+	get authority() {
+		return this.mapped.authority;
+	}
+	get flatPath() {
+		return join('/', this.scheme, this.authority, this.path);
+	}
+}

extensions/typescript-language-features/src/filesystems/memFs.ts

@@ -0,0 +1,198 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as vscode from 'vscode';
+import { basename, dirname } from 'path';
+
+export class MemFs implements vscode.FileSystemProvider {
+
+	private readonly root = new FsEntry(
+		new Map(),
+		0,
+		0,
+	);
+
+	stat(uri: vscode.Uri): vscode.FileStat {
+		// console.log('stat', uri.toString());
+		const entry = this.getEntry(uri);
+		if (!entry) {
+			throw vscode.FileSystemError.FileNotFound();
+		}
+
+		return entry;
+	}
+
+	readDirectory(uri: vscode.Uri): [string, vscode.FileType][] {
+		// console.log('readDirectory', uri.toString());
+
+		const entry = this.getEntry(uri);
+		if (!entry) {
+			throw vscode.FileSystemError.FileNotFound();
+		}
+
+		return [...entry.contents.entries()].map(([name, entry]) => [name, entry.type]);
+	}
+
+	readFile(uri: vscode.Uri): Uint8Array {
+		// console.log('readFile', uri.toString());
+
+		const entry = this.getEntry(uri);
+		if (!entry) {
+			throw vscode.FileSystemError.FileNotFound();
+		}
+
+		return entry.data;
+	}
+
+	writeFile(uri: vscode.Uri, content: Uint8Array, { create, overwrite }: { create: boolean; overwrite: boolean }): void {
+		// console.log('writeFile', uri.toString());
+
+		const dir = this.getParent(uri);
+
+		const fileName = basename(uri.path);
+		const dirContents = dir.contents;
+
+		const time = Date.now() / 1000;
+		const entry = dirContents.get(basename(uri.path));
+		if (!entry) {
+			if (create) {
+				dirContents.set(fileName, new FsEntry(content, time, time));
+				this._emitter.fire([{ type: vscode.FileChangeType.Created, uri }]);
+			} else {
+				throw vscode.FileSystemError.FileNotFound();
+			}
+		} else {
+			if (overwrite) {
+				entry.mtime = time;
+				entry.data = content;
+				this._emitter.fire([{ type: vscode.FileChangeType.Changed, uri }]);
+			} else {
+				throw vscode.FileSystemError.NoPermissions('overwrite option was not passed in');
+			}
+		}
+	}
+
+	rename(_oldUri: vscode.Uri, _newUri: vscode.Uri, _options: { overwrite: boolean }): void {
+		throw new Error('not implemented');
+	}
+
+	delete(uri: vscode.Uri): void {
+		try {
+			const dir = this.getParent(uri);
+			dir.contents.delete(basename(uri.path));
+			this._emitter.fire([{ type: vscode.FileChangeType.Deleted, uri }]);
+		} catch (e) { }
+	}
+
+	createDirectory(uri: vscode.Uri): void {
+		// console.log('createDirectory', uri.toString());
+		const dir = this.getParent(uri);
+		const now = Date.now() / 1000;
+		dir.contents.set(basename(uri.path), new FsEntry(new Map(), now, now));
+	}
+
+	private getEntry(uri: vscode.Uri): FsEntry | void {
+		// TODO: have this throw FileNotFound itself?
+		// TODO: support configuring case sensitivity
+		let node: FsEntry = this.root;
+		for (const component of uri.path.split('/')) {
+			if (!component) {
+				// Skip empty components (root, stuff between double slashes,
+				// trailing slashes)
+				continue;
+			}
+
+			if (node.type !== vscode.FileType.Directory) {
+				// We're looking at a File or such, so bail.
+				return;
+			}
+
+			const next = node.contents.get(component);
+
+			if (!next) {
+				// not found!
+				return;
+			}
+
+			node = next;
+		}
+		return node;
+	}
+
+	private getParent(uri: vscode.Uri) {
+		const dir = this.getEntry(uri.with({ path: dirname(uri.path) }));
+		if (!dir) {
+			throw vscode.FileSystemError.FileNotFound();
+		}
+		return dir;
+	}
+
+	// --- manage file events
+
+	private readonly _emitter = new vscode.EventEmitter<vscode.FileChangeEvent[]>();
+
+	readonly onDidChangeFile: vscode.Event<vscode.FileChangeEvent[]> = this._emitter.event;
+	private readonly watchers = new Map<string, Set<Symbol>>;
+
+	watch(resource: vscode.Uri): vscode.Disposable {
+		if (!this.watchers.has(resource.path)) {
+			this.watchers.set(resource.path, new Set());
+		}
+		const sy = Symbol(resource.path);
+		return new vscode.Disposable(() => {
+			const watcher = this.watchers.get(resource.path);
+			if (watcher) {
+				watcher.delete(sy);
+				if (!watcher.size) {
+					this.watchers.delete(resource.path);
+				}
+			}
+		});
+	}
+}
+
+class FsEntry {
+	get type(): vscode.FileType {
+		if (this._data instanceof Uint8Array) {
+			return vscode.FileType.File;
+		} else {
+			return vscode.FileType.Directory;
+		}
+	}
+
+	get size(): number {
+		if (this.type === vscode.FileType.Directory) {
+			return [...this.contents.values()].reduce((acc: number, entry: FsEntry) => acc + entry.size, 0);
+		} else {
+			return this.data.length;
+		}
+	}
+
+	constructor(
+		private _data: Uint8Array | Map<string, FsEntry>,
+		public ctime: number,
+		public mtime: number,
+	) { }
+
+	get data() {
+		if (this.type === vscode.FileType.Directory) {
+			throw vscode.FileSystemError.FileIsADirectory;
+		}
+		return <Uint8Array>this._data;
+	}
+	set data(val: Uint8Array) {
+		if (this.type === vscode.FileType.Directory) {
+			throw vscode.FileSystemError.FileIsADirectory;
+		}
+		this._data = val;
+	}
+
+	get contents() {
+		if (this.type !== vscode.FileType.Directory) {
+			throw vscode.FileSystemError.FileNotADirectory;
+		}
+		return <Map<string, FsEntry>>this._data;
+	}
+}

extensions/typescript-language-features/src/tsServer/serverProcess.browser.ts

@@ -43,13 +43,15 @@ export class WorkerServerProcessFactory implements TsServerProcessFactory {
 		tsServerLog: TsServerLog | undefined,
 	) {
 		const tsServerPath = version.tsServerPath;
-		return new WorkerServerProcess(kind, tsServerPath, this._extensionUri, [
+		const launchArgs = [
 			...args,
-
-			// Explicitly give TS Server its path so it can
-			// load local resources
+			// Explicitly give TS Server its path so it can load local resources
 			'--executingFilePath', tsServerPath,
-		], tsServerLog, this._logger);
+		];
+		if (_configuration.webExperimentalTypeAcquisition) {
+			launchArgs.push('--experimentalTypeAcquisition');
+		}
+		return new WorkerServerProcess(kind, tsServerPath, this._extensionUri, launchArgs, tsServerLog, this._logger);
 	}
 }