vscode/build/lib/compilation.ts

/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

import es from 'event-stream';
import fs from 'fs';
import { gulp, bom, sourcemaps } from './gulp/facade.ts';
import path from 'path';
import * as monacodts from './monaco-api.ts';
import * as nls from './nls.ts';
import { createReporter } from './reporter.ts';
import * as util from './util.ts';
import fancyLog from 'fancy-log';
import ansiColors from 'ansi-colors';
import os from 'os';
import File from 'vinyl';
import * as task from './gulp/task.ts';
import ts from 'typescript';
import watch from './watch/index.ts';
import * as tsb from './tsb/index.ts';
import { createTsgoStream, spawnTsgo } from './tsgo.ts';
import { apiProposalNamesSource, checkApiProposalNames, generateApiProposalNames } from './apiProposalNames.ts';


import { extractExtensionPointNamesFromFile } from './extractExtensionPoints.ts';


// --- gulp-tsb: compile and transpile --------------------------------

const reporter = createReporter();

function getTypeScriptCompilerOptions(src: string): ts.CompilerOptions {
	const rootDir = path.join(import.meta.dirname, `../../${src}`);
	const options: ts.CompilerOptions = {};
	options.verbose = false;
	options.sourceMap = true;
	if (process.env['VSCODE_NO_SOURCEMAP']) { // To be used by developers in a hurry
		options.sourceMap = false;
	}
	options.rootDir = rootDir;
	options.baseUrl = rootDir;
	options.sourceRoot = util.toFileUri(rootDir);
	options.newLine = /\r\n/.test(fs.readFileSync(import.meta.filename, 'utf8')) ? 0 : 1;
	return options;
}

interface ICompileTaskOptions {
	readonly build: boolean;
	readonly emitError: boolean;
	readonly transpileOnly: boolean | { esbuild: boolean };
	readonly preserveEnglish: boolean;
}

export function createCompile(src: string, { build, emitError, transpileOnly, preserveEnglish }: ICompileTaskOptions) {
	const projectPath = path.join(import.meta.dirname, '../../', src, 'tsconfig.json');
	const overrideOptions = { ...getTypeScriptCompilerOptions(src), inlineSources: Boolean(build) };
	if (!build) {
		overrideOptions.inlineSourceMap = true;
	}
	const compilation = tsb.create(projectPath, overrideOptions, {
		verbose: false,
		transpileOnly: Boolean(transpileOnly),
		transpileWithEsbuild: typeof transpileOnly !== 'boolean' && transpileOnly.esbuild
	}, err => reporter(err));

	function pipeline(token?: util.ICancellationToken) {

		const tsFilter = util.filter(data => /\.ts$/.test(data.path));
		const isUtf8Test = (f: File) => /(\/|\\)test(\/|\\).*utf8/.test(f.path);
		const isRuntimeJs = (f: File) => f.path.endsWith('.js') && !f.path.includes('fixtures');
		const noDeclarationsFilter = util.filter(data => !(/\.d\.ts$/.test(data.path)));

		const input = es.through();
		const output = input
			.pipe(util.$if(isUtf8Test, bom())) // this is required to preserve BOM in test files that loose it otherwise
			.pipe(util.$if(!build && isRuntimeJs, util.appendOwnPathSourceURL()))
			.pipe(tsFilter)
			.pipe(util.loadSourcemaps())
			.pipe(compilation(token))
			.pipe(noDeclarationsFilter)
			.pipe(util.$if(build, nls.nls({ preserveEnglish })))
			.pipe(noDeclarationsFilter.restore)
			.pipe(util.$if(!transpileOnly, sourcemaps.write('.', {
				addComment: false,
				includeContent: !!build,
				sourceRoot: overrideOptions.sourceRoot
			})))
			.pipe(tsFilter.restore)
			.pipe(reporter.end(!!emitError));

		return es.duplex(input, output);
	}
	pipeline.projectPath = projectPath;
	return pipeline;
}

export function transpileTask(src: string, out: string, esbuild?: boolean): task.StreamTask {

	const task = () => {

		const transpile = createCompile(src, { build: false, emitError: true, transpileOnly: { esbuild: !!esbuild }, preserveEnglish: false });
		const srcPipe = gulp.src(`${src}/**`, { base: `${src}` });

		return srcPipe
			.pipe(transpile())
			.pipe(gulp.dest(out));
	};

	task.taskName = `transpile-${path.basename(src)}`;
	return task;
}

export function compileTask(src: string, out: string): task.Task {

	const task = async () => {

		if (os.totalmem() < 4_000_000_000) {
			throw new Error('compilation requires 4GB of RAM');
		}

		const compile = createCompile(src, { build: false, emitError: true, transpileOnly: { esbuild: true }, preserveEnglish: false });
		const srcPipe = gulp.src(`${src}/**`, { base: `${src}` });
		const generator = new MonacoGenerator(false);
		if (src === 'src') {
			generator.execute();
		}

		const emit = util.streamToPromise(srcPipe
			.pipe(generator.stream)
			.pipe(compile())
			.pipe(gulp.dest(out)));

		const typecheck = spawnTsgo(compile.projectPath, { taskName: `compile-${path.basename(src)}`, noEmit: true });

		await Promise.all([emit, typecheck]);
	};

	task.taskName = `compile-${path.basename(src)}`;
	return task;
}

export function watchTypeCheckTask(src: string): task.Task {
	return task.define(`watch-typecheck-${path.basename(src)}`, () => {
		const projectPath = path.join(import.meta.dirname, '../../', src, 'tsconfig.json');
		const generator = new MonacoGenerator(true);
		generator.execute();
		const watchInput = watch(`${src}/**`, { base: src, readDelay: 200 });
		const tsgoStream = watchInput.pipe(generator.stream).pipe(util.debounce(() => {
			const stream = createTsgoStream(projectPath, { taskName: 'watch-client-noEmit', noEmit: true });
			const result = es.through();
			stream.on('end', () => {
				result.emit('end');
			});
			stream.on('error', err => {
				reporter(err);
				fancyLog.error(ansiColors.red('[tsgo] watch-client-noEmit failed'));
				result.emit('end');
			});
			return result.pipe(reporter.end(false));
		}));
		return tsgoStream;
	});
}

const REPO_SRC_FOLDER = path.join(import.meta.dirname, '../../src');

class MonacoGenerator {
	private readonly _isWatch: boolean;
	public readonly stream: NodeJS.ReadWriteStream;

	private readonly _watchedFiles: { [filePath: string]: boolean };
	private readonly _fsProvider: monacodts.FSProvider;
	private readonly _declarationResolver: monacodts.DeclarationResolver;

	constructor(isWatch: boolean) {
		this._isWatch = isWatch;
		this.stream = es.through();
		this._watchedFiles = {};
		const onWillReadFile = (moduleId: string, filePath: string) => {
			if (!this._isWatch) {
				return;
			}
			if (this._watchedFiles[filePath]) {
				return;
			}
			this._watchedFiles[filePath] = true;

			fs.watchFile(filePath, () => {
				this._declarationResolver.invalidateCache(moduleId);
				this._executeSoon();
			});
		};
		this._fsProvider = new class extends monacodts.FSProvider {
			public readFileSync(moduleId: string, filePath: string): Buffer {
				onWillReadFile(moduleId, filePath);
				return super.readFileSync(moduleId, filePath);
			}
		};
		this._declarationResolver = new monacodts.DeclarationResolver(this._fsProvider);

		if (this._isWatch) {
			fs.watchFile(monacodts.RECIPE_PATH, () => {
				this._executeSoon();
			});
		}
	}

	private _executeSoonTimer: NodeJS.Timeout | null = null;
	private _executeSoon(): void {
		if (this._executeSoonTimer !== null) {
			clearTimeout(this._executeSoonTimer);
			this._executeSoonTimer = null;
		}
		this._executeSoonTimer = setTimeout(() => {
			this._executeSoonTimer = null;
			this.execute();
		}, 20);
	}

	private _run(): monacodts.IMonacoDeclarationResult | null {
		const r = monacodts.run3(this._declarationResolver);
		if (!r && !this._isWatch) {
			// The build must always be able to generate the monaco.d.ts
			throw new Error(`monaco.d.ts generation error - Cannot continue`);
		}
		return r;
	}

	private _log(message: string, ...rest: unknown[]): void {
		fancyLog(ansiColors.cyan('[monaco.d.ts]'), message, ...rest);
	}

	public execute(): void {
		const startTime = Date.now();
		const result = this._run();
		if (!result) {
			// nothing really changed
			return;
		}
		if (result.isTheSame) {
			return;
		}

		fs.writeFileSync(result.filePath, result.content);
		fs.writeFileSync(path.join(REPO_SRC_FOLDER, 'vs/editor/common/standalone/standaloneEnums.ts'), result.enums);
		this._log(`monaco.d.ts is changed - total time took ${Date.now() - startTime} ms`);
		if (!this._isWatch) {
			this.stream.emit('error', 'monaco.d.ts is no longer up to date. Please run gulp watch and commit the new file.');
		}
	}
}

function createApiProposalNamesGenerator() {
	let eol: string;

	try {
		const src = fs.readFileSync('src/vs/platform/extensions/common/extensionsApiProposals.ts', 'utf-8');
		const match = /\r?\n/m.exec(src);
		eol = match ? match[0] : os.EOL;
	} catch {
		eol = os.EOL;
	}

	const proposalFiles: string[] = [];

	const input = es.through();
	const output = input
		.pipe(es.through((f: File) => {
			proposalFiles.push(f.path);
		}, function () {
			const contents = generateApiProposalNames(proposalFiles, eol);

			const filePath = 'vs/platform/extensions/common/extensionsApiProposals.ts';
			try {
				const existing = fs.readFileSync(path.join('src', filePath), 'utf-8');
				if (existing === contents) {
					this.emit('end');
					return;
				}
			} catch {
				// File doesn't exist yet, emit it
			}
			this.emit('data', new File({
				path: filePath,
				contents: Buffer.from(contents)
			}));
			this.emit('end');
		}));

	return es.duplex(input, output);
}

const apiProposalNamesReporter = createReporter('api-proposal-names');

export const checkApiProposalNamesTask = task.define('check-api-proposal-names', async () => {
	checkApiProposalNames(process.cwd());
});

export const compileApiProposalNamesTask = task.define('compile-api-proposal-names', () => {
	return gulp.src(apiProposalNamesSource)
		.pipe(createApiProposalNamesGenerator())
		.pipe(gulp.dest('src'))
		.pipe(apiProposalNamesReporter.end(true));
});

function generateExtensionPointNames() {
	const collectedNames: string[] = [];

	const input = es.through();
	const output = input
		.pipe(es.through(function (file: File) {
			const contents = file.contents?.toString('utf-8');
			if (contents && contents.includes('registerExtensionPoint')) {
				const sourceFile = ts.createSourceFile(file.path, contents, ts.ScriptTarget.Latest, true);
				collectedNames.push(...extractExtensionPointNamesFromFile(sourceFile));
			}
		}, function () {
			collectedNames.sort();
			const content = JSON.stringify(collectedNames, undefined, '\t') + '\n';
			const filePath = 'vs/workbench/services/extensions/common/extensionPoints.json';
			try {
				const existing = fs.readFileSync(path.join('src', filePath), 'utf-8');
				if (existing.replace(/\r\n/g, '\n') === content) {
					this.emit('end');
					return;
				}
			} catch {
				// File doesn't exist yet, emit it
			}
			this.emit('data', new File({
				path: filePath,
				contents: Buffer.from(content)
			}));
			this.emit('end');
		}));

	return es.duplex(input, output);
}

const extensionPointNamesReporter = createReporter('extension-point-names');

export const compileExtensionPointNamesTask = task.define('compile-extension-point-names', () => {
	return gulp.src('src/vs/workbench/**/*.ts')
		.pipe(generateExtensionPointNames())
		.pipe(gulp.dest('src'))
		.pipe(extensionPointNamesReporter.end(true));
});

export const watchExtensionPointNamesTask = task.define('watch-extension-point-names', () => {
	const task = () => gulp.src('src/vs/workbench/**/*.ts')
		.pipe(generateExtensionPointNames())
		.pipe(extensionPointNamesReporter.end(true));

	return watch('src/vs/workbench/**/*.ts', { readDelay: 200 })
		.pipe(util.debounce(task))
		.pipe(gulp.dest('src'));
});

export const watchApiProposalNamesTask = task.define('watch-api-proposal-names', () => {
	const task = () => gulp.src(apiProposalNamesSource)
		.pipe(createApiProposalNamesGenerator())
		.pipe(apiProposalNamesReporter.end(true));

	return watch(apiProposalNamesSource, { readDelay: 200 })
		.pipe(util.debounce(task))
		.pipe(gulp.dest('src'));
});

// Codicons
const root = path.dirname(path.dirname(import.meta.dirname));
const codiconSource = path.join(root, 'node_modules', '@vscode', 'codicons', 'dist', 'codicon.ttf');
const codiconDest = path.join(root, 'src', 'vs', 'base', 'browser', 'ui', 'codicons', 'codicon', 'codicon.ttf');

function copyCodiconsImpl() {
	try {
		if (fs.existsSync(codiconSource)) {
			fs.mkdirSync(path.dirname(codiconDest), { recursive: true });
			fs.copyFileSync(codiconSource, codiconDest);
		} else {
			fancyLog(ansiColors.red('[codicons]'), `codicon.ttf not found in node_modules. Please run 'npm install' to install dependencies.`);
		}
	} catch (e) {
		fancyLog(ansiColors.red('[codicons]'), `Error copying codicon.ttf: ${e}`);
	}
}

export const copyCodiconsTask = task.define('copy-codicons', () => {
	copyCodiconsImpl();
	return Promise.resolve();
});
task.task(copyCodiconsTask);

export const watchCodiconsTask = task.define('watch-codicons', () => {
	copyCodiconsImpl();
	return watch('node_modules/@vscode/codicons/dist/**', { readDelay: 200 })
		.on('data', () => copyCodiconsImpl());
});
task.task(watchCodiconsTask);