Use esbuild to pack the markdown extension for desktop and web (#294208)

* Try using esbuild to bundle our built-in extensions

Test switching to esbuild instead of webpack to bundle our buildin extensions. Setup so we can do this incrementally and starting with the markdown extension as a test

* Fix build ext media

* Fix .ts script name check

* Update comment

* Use ts for all scripts

build/gulpfile.extensions.ts

@@ -273,11 +273,33 @@ gulp.task(compileWebExtensionsTask);
 export const watchWebExtensionsTask = task.define('watch-web', () => buildWebExtensions(true));
 gulp.task(watchWebExtensionsTask);
 
-async function buildWebExtensions(isWatch: boolean) {
+async function buildWebExtensions(isWatch: boolean): Promise<void> {
 	const extensionsPath = path.join(root, 'extensions');
-	const webpackConfigLocations = await nodeUtil.promisify(glob)(
-		path.join(extensionsPath, '**', 'extension-browser.webpack.config.js'),
+
+	// Find all esbuild-browser.ts files
+	const esbuildConfigLocations = await nodeUtil.promisify(glob)(
+		path.join(extensionsPath, '**', 'esbuild-browser.ts'),
 		{ ignore: ['**/node_modules'] }
 	);
-	return ext.webpackExtensions('packaging web extension', isWatch, webpackConfigLocations.map(configPath => ({ configPath })));
+
+	// Find all webpack configs, excluding those that will be esbuilt
+	const esbuildExtensionDirs = new Set(esbuildConfigLocations.map(p => path.dirname(p)));
+	const webpackConfigLocations = (await nodeUtil.promisify(glob)(
+		path.join(extensionsPath, '**', 'extension-browser.webpack.config.js'),
+		{ ignore: ['**/node_modules'] }
+	)).filter(configPath => !esbuildExtensionDirs.has(path.dirname(configPath)));
+
+	const promises: Promise<unknown>[] = [];
+
+	// Esbuild for extensions
+	if (esbuildConfigLocations.length > 0) {
+		promises.push(ext.esbuildExtensions('packaging web extension (esbuild)', isWatch, esbuildConfigLocations.map(script => ({ script }))));
+	}
+
+	// Run webpack for remaining extensions
+	if (webpackConfigLocations.length > 0) {
+		promises.push(ext.webpackExtensions('packaging web extension', isWatch, webpackConfigLocations.map(configPath => ({ configPath }))));
+	}
+
+	await Promise.all(promises);
 }

build/lib/extensions.ts

@@ -66,16 +66,31 @@ function updateExtensionPackageJSON(input: Stream, update: (data: any) => any):
 
 function fromLocal(extensionPath: string, forWeb: boolean, disableMangle: boolean): Stream {
 
+	const esbuildConfigFileName = forWeb
+		? 'esbuild-browser.ts'
+		: 'esbuild.ts';
+
 	const webpackConfigFileName = forWeb
 		? `extension-browser.webpack.config.js`
 		: `extension.webpack.config.js`;
 
+	const hasEsbuild = fs.existsSync(path.join(extensionPath, esbuildConfigFileName));
 	const isWebPacked = fs.existsSync(path.join(extensionPath, webpackConfigFileName));
-	let input = isWebPacked
-		? fromLocalWebpack(extensionPath, webpackConfigFileName, disableMangle)
-		: fromLocalNormal(extensionPath);
 
-	if (isWebPacked) {
+	let input: Stream;
+	let isBundled = false;
+
+	if (hasEsbuild) {
+		input = fromLocalEsbuild(extensionPath, esbuildConfigFileName);
+		isBundled = true;
+	} else if (isWebPacked) {
+		input = fromLocalWebpack(extensionPath, webpackConfigFileName, disableMangle);
+		isBundled = true;
+	} else {
+		input = fromLocalNormal(extensionPath);
+	}
+
+	if (isBundled) {
 		input = updateExtensionPackageJSON(input, (data: any) => {
 			delete data.scripts;
 			delete data.dependencies;
@@ -240,6 +255,51 @@ function fromLocalNormal(extensionPath: string): Stream {
 	return result.pipe(createStatsStream(path.basename(extensionPath)));
 }
 
+function fromLocalEsbuild(extensionPath: string, esbuildConfigFileName: string): Stream {
+	const vsce = require('@vscode/vsce') as typeof import('@vscode/vsce');
+	const result = es.through();
+
+	const esbuildScript = path.join(extensionPath, esbuildConfigFileName);
+
+	// Run esbuild, then collect the files
+	new Promise<void>((resolve, reject) => {
+		const proc = cp.execFile(process.argv[0], [esbuildScript], {}, (error, _stdout, stderr) => {
+			if (error) {
+				return reject(error);
+			}
+			const matches = (stderr || '').match(/\> (.+): error: (.+)?/g);
+			fancyLog(`Bundled extension: ${ansiColors.yellow(path.join(path.basename(extensionPath), esbuildConfigFileName))} with ${matches ? matches.length : 0} errors.`);
+			for (const match of matches || []) {
+				fancyLog.error(match);
+			}
+			return resolve();
+		});
+
+		proc.stdout!.on('data', (data) => {
+			fancyLog(`${ansiColors.green('esbuilding')}: ${data.toString('utf8')}`);
+		});
+	}).then(() => {
+		// After esbuild completes, collect all files using vsce
+		return vsce.listFiles({ cwd: extensionPath, packageManager: vsce.PackageManager.None });
+	}).then(fileNames => {
+		const files = fileNames
+			.map(fileName => path.join(extensionPath, fileName))
+			.map(filePath => new File({
+				path: filePath,
+				stat: fs.statSync(filePath),
+				base: extensionPath,
+				contents: fs.createReadStream(filePath)
+			}));
+
+		es.readArray(files).pipe(result);
+	}).catch(err => {
+		console.error(extensionPath);
+		result.emit('error', err);
+	});
+
+	return result.pipe(createStatsStream(path.basename(extensionPath)));
+}
+
 const userAgent = 'VSCode Build';
 const baseHeaders = {
 	'X-Market-Client-Id': 'VSCode Build',
@@ -647,7 +707,7 @@ export async function webpackExtensions(taskName: string, isWatch: boolean, webp
 	});
 }
 
-async function esbuildExtensions(taskName: string, isWatch: boolean, scripts: { script: string; outputRoot?: string }[]) {
+export async function esbuildExtensions(taskName: string, isWatch: boolean, scripts: { script: string; outputRoot?: string }[]): Promise<void> {
 	function reporter(stdError: string, script: string) {
 		const matches = (stdError || '').match(/\> (.+): error: (.+)?/g);
 		fancyLog(`Finished ${ansiColors.green(taskName)} ${script} with ${matches ? matches.length : 0} errors.`);
@@ -678,10 +738,11 @@ async function esbuildExtensions(taskName: string, isWatch: boolean, scripts: {
 			});
 		});
 	});
-	return Promise.all(tasks);
+
+	await Promise.all(tasks);
 }
 
-export async function buildExtensionMedia(isWatch: boolean, outputRoot?: string) {
+export function buildExtensionMedia(isWatch: boolean, outputRoot?: string): Promise<void> {
 	return esbuildExtensions('esbuilding extension media', isWatch, esbuildMediaScripts.map(p => ({
 		script: path.join(extensionsPath, p),
 		outputRoot: outputRoot ? path.join(root, outputRoot, path.dirname(p)) : undefined

eslint.config.js

@@ -2064,7 +2064,9 @@ export default tseslint.config(
 	// Additional extension strictness rules
 	{
 		files: [
-			'extensions/markdown-language-features/**/*.ts',
+			'extensions/markdown-language-features/src/**/*.ts',
+			'extensions/markdown-language-features/notebook/**/*.ts',
+			'extensions/markdown-language-features/preview-src/**/*.ts',
 			'extensions/mermaid-chat-features/**/*.ts',
 			'extensions/media-preview/**/*.ts',
 			'extensions/simple-browser/**/*.ts',

extensions/esbuild-extension-common.ts

@@ -0,0 +1,77 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+/**
+ * @fileoverview Common build script for extensions.
+ */
+import path from 'node:path';
+import esbuild from 'esbuild';
+
+type BuildOptions = Partial<esbuild.BuildOptions> & {
+	outdir: string;
+};
+
+/**
+ * Build the source code once using esbuild.
+ */
+async function build(options: BuildOptions, didBuild?: (outDir: string) => unknown): Promise<void> {
+	await esbuild.build({
+		bundle: true,
+		minify: true,
+		sourcemap: false,
+		format: 'cjs',
+		platform: 'node',
+		target: ['es2024'],
+		external: ['vscode'],
+		...options,
+	});
+
+	await didBuild?.(options.outdir);
+}
+
+/**
+ * Build the source code once using esbuild, logging errors instead of throwing.
+ */
+async function tryBuild(options: BuildOptions, didBuild?: (outDir: string) => unknown): Promise<void> {
+	try {
+		await build(options, didBuild);
+	} catch (err) {
+		console.error(err);
+	}
+}
+
+interface RunConfig {
+	srcDir: string;
+	outdir: string;
+	entryPoints: string[] | Record<string, string> | { in: string; out: string }[];
+	additionalOptions?: Partial<esbuild.BuildOptions>;
+}
+
+export async function run(config: RunConfig, args: string[], didBuild?: (outDir: string) => unknown): Promise<void> {
+	let outdir = config.outdir;
+	const outputRootIndex = args.indexOf('--outputRoot');
+	if (outputRootIndex >= 0) {
+		const outputRoot = args[outputRootIndex + 1];
+		const outputDirName = path.basename(outdir);
+		outdir = path.join(outputRoot, outputDirName);
+	}
+
+	const resolvedOptions: BuildOptions = {
+		entryPoints: config.entryPoints,
+		outdir,
+		logOverride: {
+			'import-is-undefined': 'error',
+		},
+		...(config.additionalOptions || {}),
+	};
+
+	const isWatch = args.indexOf('--watch') >= 0;
+	if (isWatch) {
+		await tryBuild(resolvedOptions, didBuild);
+		const watcher = await import('@parcel/watcher');
+		watcher.subscribe(config.srcDir, () => tryBuild(resolvedOptions, didBuild));
+	} else {
+		return build(resolvedOptions, didBuild).catch(() => process.exit(1));
+	}
+}

extensions/markdown-language-features/esbuild-browser.ts

@@ -0,0 +1,40 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { run } from '../esbuild-extension-common.ts';
+
+const srcDir = path.join(import.meta.dirname, 'src');
+const outDir = path.join(import.meta.dirname, 'dist', 'browser');
+
+/**
+ * Copy the language server worker main file to the output directory.
+ */
+async function copyServerWorkerMain(outDir: string): Promise<void> {
+	const srcPath = path.join(import.meta.dirname, 'node_modules', 'vscode-markdown-languageserver', 'dist', 'browser', 'workerMain.js');
+	const destPath = path.join(outDir, 'serverWorkerMain.js');
+	await fs.promises.copyFile(srcPath, destPath);
+}
+
+run({
+	entryPoints: {
+		'extension': path.join(srcDir, 'extension.browser.ts'),
+	},
+	srcDir,
+	outdir: outDir,
+	additionalOptions: {
+		platform: 'browser',
+		format: 'cjs',
+		alias: {
+			'path': 'path-browserify',
+		},
+		define: {
+			'process.platform': JSON.stringify('web'),
+			'process.env': JSON.stringify({}),
+			'process.env.BROWSER_ENV': JSON.stringify('true'),
+		},
+		tsconfig: path.join(import.meta.dirname, 'tsconfig.browser.json'),
+	},
+}, process.argv, copyServerWorkerMain);

extensions/markdown-language-features/esbuild.ts

@@ -0,0 +1,27 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { run } from '../esbuild-extension-common.ts';
+
+const srcDir = path.join(import.meta.dirname, 'src');
+const outDir = path.join(import.meta.dirname, 'dist');
+
+/**
+ * Copy the language server worker main file to the output directory.
+ */
+async function copyServerWorkerMain(outDir: string): Promise<void> {
+	const srcPath = path.join(import.meta.dirname, 'node_modules', 'vscode-markdown-languageserver', 'dist', 'node', 'workerMain.js');
+	const destPath = path.join(outDir, 'serverWorkerMain.js');
+	await fs.promises.copyFile(srcPath, destPath);
+}
+
+run({
+	entryPoints: {
+		'extension': path.join(srcDir, 'extension.ts'),
+	},
+	srcDir,
+	outdir: outDir,
+}, process.argv, copyServerWorkerMain);

extensions/markdown-language-features/extension-browser.webpack.config.js

@@ -1,27 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-// @ts-check
-import CopyPlugin from 'copy-webpack-plugin';
-import { browser, browserPlugins } from '../shared.webpack.config.mjs';
-
-export default browser({
-	context: import.meta.dirname,
-	entry: {
-		extension: './src/extension.browser.ts'
-	},
-	plugins: [
-		...browserPlugins(import.meta.dirname), // add plugins, don't replace inherited
-		new CopyPlugin({
-			patterns: [
-				{
-					from: './node_modules/vscode-markdown-languageserver/dist/browser/workerMain.js',
-					to: 'serverWorkerMain.js',
-				}
-			],
-		}),
-	],
-}, {
-	configFile: 'tsconfig.browser.json'
-});

extensions/markdown-language-features/extension.webpack.config.js

@@ -1,28 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-// @ts-check
-import CopyPlugin from 'copy-webpack-plugin';
-import withDefaults, { nodePlugins } from '../shared.webpack.config.mjs';
-
-export default withDefaults({
-	context: import.meta.dirname,
-	resolve: {
-		mainFields: ['module', 'main']
-	},
-	entry: {
-		extension: './src/extension.ts',
-	},
-	plugins: [
-		...nodePlugins(import.meta.dirname), // add plugins, don't replace inherited
-		new CopyPlugin({
-			patterns: [
-				{
-					from: './node_modules/vscode-markdown-languageserver/dist/node/workerMain.js',
-					to: 'serverWorkerMain.js',
-				}
-			],
-		}),
-	],
-});