From 9e21aff42e30b602108dc104fd5c7c8edd2c5b82 Mon Sep 17 00:00:00 2001 From: Johannes Rieken Date: Wed, 15 Jun 2022 16:52:48 +0200 Subject: [PATCH] joh/ts transpile (#152199) transpile-only tasks for client and extensions * extract transpile into its own file * add transpile-client task, polish transpiler * add transpile-extensions, improve transpile logic * move declaration of "const enum" above it usage so that it can be used with const-enum-inlining * (ugly) make d.ts transpilation configurable because it is needed for extensions but a problem for client * hack my way around so that `getOwnEmitOutputFilePath` is reusable by our transpile * honor `noEmit` flag --- build/gulpfile.compile.js | 2 +- build/gulpfile.editor.js | 2 +- build/gulpfile.extensions.js | 20 +- build/gulpfile.js | 6 +- build/lib/compilation.js | 13 +- build/lib/compilation.ts | 13 +- build/lib/tsb/index.js | 49 ++-- build/lib/tsb/index.ts | 62 ++-- build/lib/tsb/transpiler.js | 210 +++++++++++++ build/lib/tsb/transpiler.ts | 276 ++++++++++++++++++ extensions/debug-auto-launch/src/extension.ts | 12 +- 11 files changed, 581 insertions(+), 84 deletions(-) create mode 100644 build/lib/tsb/transpiler.js create mode 100644 build/lib/tsb/transpiler.ts diff --git a/build/gulpfile.compile.js b/build/gulpfile.compile.js index c3049784fe4..6f4523eeae6 100644 --- a/build/gulpfile.compile.js +++ b/build/gulpfile.compile.js @@ -15,7 +15,7 @@ const compileBuildTask = task.define('compile-build', task.series( util.rimraf('out-build'), util.buildWebNodePaths('out-build'), - compilation.compileTask('src', 'out-build', true) + compilation.compileTask('src', 'out-build', true, false) ) ); gulp.task(compileBuildTask); diff --git a/build/gulpfile.editor.js b/build/gulpfile.editor.js index 3b3fe4942d8..45546aa9e64 100644 --- a/build/gulpfile.editor.js +++ b/build/gulpfile.editor.js @@ -78,7 +78,7 @@ const extractEditorSrcTask = task.define('extract-editor-src', () => { }); }); -const compileEditorAMDTask = task.define('compile-editor-amd', compilation.compileTask('out-editor-src', 'out-editor-build', true)); +const compileEditorAMDTask = task.define('compile-editor-amd', compilation.compileTask('out-editor-src', 'out-editor-build', true, false)); const optimizeEditorAMDTask = task.define('optimize-editor-amd', common.optimizeTask({ src: 'out-editor-build', diff --git a/build/gulpfile.extensions.js b/build/gulpfile.extensions.js index 3f9f19b4b94..71fa1ec425c 100644 --- a/build/gulpfile.extensions.js +++ b/build/gulpfile.extensions.js @@ -100,7 +100,7 @@ const tasks = compilations.map(function (tsconfigFile) { headerOut = relativeDirname.substr(index + 1) + '/out'; } - function createPipeline(build, emitError) { + function createPipeline(build, emitError, transpileOnly) { const nlsDev = require('vscode-nls-dev'); const tsb = require('./lib/tsb'); const sourcemaps = require('gulp-sourcemaps'); @@ -110,7 +110,7 @@ const tasks = compilations.map(function (tsconfigFile) { overrideOptions.inlineSources = Boolean(build); overrideOptions.base = path.dirname(absolutePath); - const compilation = tsb.create(absolutePath, overrideOptions, { verbose: false }, err => reporter(err.toString())); + const compilation = tsb.create(absolutePath, overrideOptions, { verbose: false, transpileOnly, transpileOnlyIncludesDts: transpileOnly }, err => reporter(err.toString())); const pipeline = function () { const input = es.through(); @@ -152,6 +152,16 @@ const tasks = compilations.map(function (tsconfigFile) { const cleanTask = task.define(`clean-extension-${name}`, util.rimraf(out)); + const transpileTask = task.define(`transpile-extension:${name}`, task.series(cleanTask, () => { + const pipeline = createPipeline(false, true, true); + const nonts = gulp.src(src, srcOpts).pipe(filter(['**', '!**/*.ts'])); + const input = es.merge(nonts, pipeline.tsProjectSrc()); + + return input + .pipe(pipeline()) + .pipe(gulp.dest(out)); + })); + const compileTask = task.define(`compile-extension:${name}`, task.series(cleanTask, () => { const pipeline = createPipeline(false, true); const nonts = gulp.src(src, srcOpts).pipe(filter(['**', '!**/*.ts'])); @@ -184,12 +194,16 @@ const tasks = compilations.map(function (tsconfigFile) { })); // Tasks + gulp.task(transpileTask); gulp.task(compileTask); gulp.task(watchTask); - return { compileTask, watchTask, compileBuildTask }; + return { transpileTask, compileTask, watchTask, compileBuildTask }; }); +const transpileExtensionsTask = task.define('transpile-extensions', task.parallel(...tasks.map(t => t.transpileTask))); +gulp.task(transpileExtensionsTask); + const compileExtensionsTask = task.define('compile-extensions', task.parallel(...tasks.map(t => t.compileTask))); gulp.task(compileExtensionsTask); exports.compileExtensionsTask = compileExtensionsTask; diff --git a/build/gulpfile.js b/build/gulpfile.js index 65dda505fa7..75d3755d8e5 100644 --- a/build/gulpfile.js +++ b/build/gulpfile.js @@ -19,8 +19,12 @@ const { compileExtensionsTask, watchExtensionsTask, compileExtensionMediaTask } gulp.task(compileApiProposalNamesTask); gulp.task(watchApiProposalNamesTask); +// Transpile only +const transpileClientTask = task.define('transpile-client', task.series(util.rimraf('out'), util.buildWebNodePaths('out'), compileTask('src', 'out', false, true))); +gulp.task(transpileClientTask); + // Fast compile for development time -const compileClientTask = task.define('compile-client', task.series(util.rimraf('out'), util.buildWebNodePaths('out'), compileApiProposalNamesTask, compileTask('src', 'out', false))); +const compileClientTask = task.define('compile-client', task.series(util.rimraf('out'), util.buildWebNodePaths('out'), compileApiProposalNamesTask, compileTask('src', 'out', false, false))); gulp.task(compileClientTask); const watchClientTask = task.define('watch-client', task.series(util.rimraf('out'), util.buildWebNodePaths('out'), task.parallel(watchTask('out', false), watchApiProposalNamesTask))); diff --git a/build/lib/compilation.js b/build/lib/compilation.js index 2c815b1ad9b..d21c5b87f36 100644 --- a/build/lib/compilation.js +++ b/build/lib/compilation.js @@ -18,6 +18,7 @@ const ansiColors = require("ansi-colors"); const os = require("os"); const File = require("vinyl"); const task = require("./task"); +const tsb = require("./tsb"); const watch = require('./watch'); const reporter = (0, reporter_1.createReporter)(); function getTypeScriptCompilerOptions(src) { @@ -34,15 +35,15 @@ function getTypeScriptCompilerOptions(src) { options.newLine = /\r\n/.test(fs.readFileSync(__filename, 'utf8')) ? 0 : 1; return options; } -function createCompile(src, build, emitError) { - const tsb = require('./tsb'); +function createCompile(src, build, emitError, transpileOnly) { + // const tsb = require('./tsb') as typeof import('./tsb'); const sourcemaps = require('gulp-sourcemaps'); const projectPath = path.join(__dirname, '../../', src, 'tsconfig.json'); const overrideOptions = { ...getTypeScriptCompilerOptions(src), inlineSources: Boolean(build) }; if (!build) { overrideOptions.inlineSourceMap = true; } - const compilation = tsb.create(projectPath, overrideOptions, { verbose: false }, err => reporter(err)); + const compilation = tsb.create(projectPath, overrideOptions, { verbose: false, transpileOnly }, err => reporter(err)); function pipeline(token) { const bom = require('gulp-bom'); const utf8Filter = util.filter(data => /(\/|\\)test(\/|\\).*utf8/.test(data.path)); @@ -73,12 +74,12 @@ function createCompile(src, build, emitError) { }; return pipeline; } -function compileTask(src, out, build) { +function compileTask(src, out, build, transpileOnly) { return function () { if (os.totalmem() < 4000000000) { throw new Error('compilation requires 4GB of RAM'); } - const compile = createCompile(src, build, true); + const compile = createCompile(src, build, true, transpileOnly); const srcPipe = gulp.src(`${src}/**`, { base: `${src}` }); const generator = new MonacoGenerator(false); if (src === 'src') { @@ -93,7 +94,7 @@ function compileTask(src, out, build) { exports.compileTask = compileTask; function watchTask(out, build) { return function () { - const compile = createCompile('src', build); + const compile = createCompile('src', build, false, false); const src = gulp.src('src/**', { base: 'src' }); const watchSrc = watch('src/**', { base: 'src', readDelay: 200 }); const generator = new MonacoGenerator(true); diff --git a/build/lib/compilation.ts b/build/lib/compilation.ts index 0cb12f4ad53..c72f7fdecda 100644 --- a/build/lib/compilation.ts +++ b/build/lib/compilation.ts @@ -19,6 +19,7 @@ import * as os from 'os'; import ts = require('typescript'); import * as File from 'vinyl'; import * as task from './task'; +import * as tsb from './tsb'; const watch = require('./watch'); @@ -39,8 +40,8 @@ function getTypeScriptCompilerOptions(src: string): ts.CompilerOptions { return options; } -function createCompile(src: string, build: boolean, emitError?: boolean) { - const tsb = require('./tsb') as typeof import('./tsb'); +function createCompile(src: string, build: boolean, emitError: boolean, transpileOnly: boolean) { + // const tsb = require('./tsb') as typeof import('./tsb'); const sourcemaps = require('gulp-sourcemaps') as typeof import('gulp-sourcemaps'); @@ -50,7 +51,7 @@ function createCompile(src: string, build: boolean, emitError?: boolean) { overrideOptions.inlineSourceMap = true; } - const compilation = tsb.create(projectPath, overrideOptions, { verbose: false }, err => reporter(err)); + const compilation = tsb.create(projectPath, overrideOptions, { verbose: false, transpileOnly }, err => reporter(err)); function pipeline(token?: util.ICancellationToken) { const bom = require('gulp-bom') as typeof import('gulp-bom'); @@ -86,7 +87,7 @@ function createCompile(src: string, build: boolean, emitError?: boolean) { return pipeline; } -export function compileTask(src: string, out: string, build: boolean): () => NodeJS.ReadWriteStream { +export function compileTask(src: string, out: string, build: boolean, transpileOnly: boolean): () => NodeJS.ReadWriteStream { return function () { @@ -94,7 +95,7 @@ export function compileTask(src: string, out: string, build: boolean): () => Nod throw new Error('compilation requires 4GB of RAM'); } - const compile = createCompile(src, build, true); + const compile = createCompile(src, build, true, transpileOnly); const srcPipe = gulp.src(`${src}/**`, { base: `${src}` }); const generator = new MonacoGenerator(false); if (src === 'src') { @@ -111,7 +112,7 @@ export function compileTask(src: string, out: string, build: boolean): () => Nod export function watchTask(out: string, build: boolean): () => NodeJS.ReadWriteStream { return function () { - const compile = createCompile('src', build); + const compile = createCompile('src', build, false, false); const src = gulp.src('src/**', { base: 'src' }); const watchSrc = watch('src/**', { base: 'src', readDelay: 200 }); diff --git a/build/lib/tsb/index.js b/build/lib/tsb/index.js index 44b9bc862dd..adab557fe87 100644 --- a/build/lib/tsb/index.js +++ b/build/lib/tsb/index.js @@ -15,6 +15,7 @@ const utils_1 = require("./utils"); const fs_1 = require("fs"); const log = require("fancy-log"); const colors = require("ansi-colors"); +const transpiler_1 = require("./transpiler"); class EmptyDuplex extends stream_1.Duplex { _write(_chunk, _encoding, callback) { callback(); } _read() { this.push(null); } @@ -51,25 +52,21 @@ function create(projectPath, existingOptions, config, onError = _defaultOnError) } } // FULL COMPILE stream doing transpile, syntax and semantic diagnostics - let _builder; - function createCompileStream(token) { - if (!_builder) { - _builder = builder.createTypeScriptBuilder({ logFn }, projectPath, cmdLine); - } + function createCompileStream(builder, token) { return through(function (file) { // give the file to the compiler if (file.isStream()) { this.emit('error', 'no support for streams'); return; } - _builder.file(file); + builder.file(file); }, function () { // start the compilation process - _builder.build(file => this.queue(file), printDiagnostic, token).catch(e => console.error(e)).then(() => this.queue(null)); + builder.build(file => this.queue(file), printDiagnostic, token).catch(e => console.error(e)).then(() => this.queue(null)); }); } // TRANSPILE ONLY stream doing just TS to JS conversion - function createTranspileStream() { + function createTranspileStream(transpiler) { return through(function (file) { // give the file to the compiler if (file.isStream()) { @@ -79,27 +76,29 @@ function create(projectPath, existingOptions, config, onError = _defaultOnError) if (!file.contents) { return; } - const out = ts.transpileModule(String(file.contents), { - compilerOptions: { ...cmdLine.options, declaration: false, sourceMap: false } - }); - if (out.diagnostics) { - out.diagnostics.forEach(printDiagnostic); + if (!config.transpileOnlyIncludesDts && file.path.endsWith('.d.ts')) { + return; } - const outFile = new Vinyl({ - path: file.path.replace(/\.ts$/, '.js'), - cwd: file.cwd, - base: file.base, - contents: Buffer.from(out.outputText), + if (!transpiler.onOutfile) { + transpiler.onOutfile = file => this.queue(file); + } + transpiler.transpile(file); + }, function () { + transpiler.join().then(() => { + this.queue(null); + transpiler.onOutfile = undefined; }); - this.push(outFile); - logFn('Transpiled', file.path); }); } - const result = (token) => { - return config.transplileOnly - ? createTranspileStream() - : createCompileStream(token); - }; + let result; + if (config.transpileOnly) { + const transpiler = new transpiler_1.Transpiler(logFn, printDiagnostic, cmdLine); + result = (() => createTranspileStream(transpiler)); + } + else { + const _builder = builder.createTypeScriptBuilder({ logFn }, projectPath, cmdLine); + result = ((token) => createCompileStream(_builder, token)); + } result.src = (opts) => { let _pos = 0; const _fileNames = cmdLine.fileNames.slice(0); diff --git a/build/lib/tsb/index.ts b/build/lib/tsb/index.ts index 548094bbf03..1164efc9d03 100644 --- a/build/lib/tsb/index.ts +++ b/build/lib/tsb/index.ts @@ -13,6 +13,7 @@ import { strings } from './utils'; import { readFileSync, statSync } from 'fs'; import * as log from 'fancy-log'; import colors = require('ansi-colors'); +import { Transpiler } from './transpiler'; export interface IncrementalCompiler { (token?: any): Readable & Writable; @@ -35,7 +36,7 @@ const _defaultOnError = (err: string) => console.log(JSON.stringify(err, null, 4 export function create( projectPath: string, existingOptions: Partial, - config: { verbose?: boolean; transplileOnly?: boolean }, + config: { verbose?: boolean; transpileOnly?: boolean; transpileOnlyIncludesDts?: boolean }, onError: (message: string) => void = _defaultOnError ): IncrementalCompiler { @@ -73,13 +74,7 @@ export function create( } // FULL COMPILE stream doing transpile, syntax and semantic diagnostics - - let _builder!: builder.ITypeScriptBuilder; - function createCompileStream(token?: builder.CancellationToken): Readable & Writable { - - if (!_builder) { - _builder = builder.createTypeScriptBuilder({ logFn }, projectPath, cmdLine); - } + function createCompileStream(builder: builder.ITypeScriptBuilder, token?: builder.CancellationToken): Readable & Writable { return through(function (this: through.ThroughStream, file: Vinyl) { // give the file to the compiler @@ -87,11 +82,11 @@ export function create( this.emit('error', 'no support for streams'); return; } - _builder.file(file); + builder.file(file); }, function (this: { queue(a: any): void }) { // start the compilation process - _builder.build( + builder.build( file => this.queue(file), printDiagnostic, token @@ -100,46 +95,43 @@ export function create( } // TRANSPILE ONLY stream doing just TS to JS conversion - function createTranspileStream(): Readable & Writable { - - return through(function (this: through.ThroughStream, file: Vinyl) { + function createTranspileStream(transpiler: Transpiler): Readable & Writable { + return through(function (this: through.ThroughStream & { queue(a: any): void }, file: Vinyl) { // give the file to the compiler if (file.isStream()) { this.emit('error', 'no support for streams'); return; } - if (!file.contents) { return; } - - const out = ts.transpileModule(String(file.contents), { - compilerOptions: { ...cmdLine.options, declaration: false, sourceMap: false } - }); - - if (out.diagnostics) { - out.diagnostics.forEach(printDiagnostic); + if (!config.transpileOnlyIncludesDts && file.path.endsWith('.d.ts')) { + return; } - const outFile = new Vinyl({ - path: file.path.replace(/\.ts$/, '.js'), - cwd: file.cwd, - base: file.base, - contents: Buffer.from(out.outputText), + if (!transpiler.onOutfile) { + transpiler.onOutfile = file => this.queue(file); + } + + transpiler.transpile(file); + + }, function (this: { queue(a: any): void }) { + transpiler.join().then(() => { + this.queue(null); + transpiler.onOutfile = undefined; }); - - this.push(outFile); - - logFn('Transpiled', file.path); }); } - const result = (token: builder.CancellationToken) => { - return config.transplileOnly - ? createTranspileStream() - : createCompileStream(token); - }; + let result: IncrementalCompiler; + if (config.transpileOnly) { + const transpiler = new Transpiler(logFn, printDiagnostic, cmdLine); + result = (() => createTranspileStream(transpiler)); + } else { + const _builder = builder.createTypeScriptBuilder({ logFn }, projectPath, cmdLine); + result = ((token: builder.CancellationToken) => createCompileStream(_builder, token)); + } result.src = (opts?: { cwd?: string; base?: string }) => { let _pos = 0; diff --git a/build/lib/tsb/transpiler.js b/build/lib/tsb/transpiler.js new file mode 100644 index 00000000000..406075bbf8e --- /dev/null +++ b/build/lib/tsb/transpiler.js @@ -0,0 +1,210 @@ +"use strict"; +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.Transpiler = void 0; +const ts = require("typescript"); +const threads = require("node:worker_threads"); +const Vinyl = require("vinyl"); +const node_os_1 = require("node:os"); +function transpile(tsSrc, options) { + const isAmd = /\n(import|export)/m.test(tsSrc); + if (!isAmd && options.compilerOptions?.module === ts.ModuleKind.AMD) { + // enforce NONE module-system for not-amd cases + options = { ...options, ...{ compilerOptions: { ...options.compilerOptions, module: ts.ModuleKind.None } } }; + } + const out = ts.transpileModule(tsSrc, options); + return { + jsSrc: out.outputText, + diag: out.diagnostics ?? [] + }; +} +if (!threads.isMainThread) { + // WORKER + threads.parentPort?.addListener('message', (req) => { + const res = { + jsSrcs: [], + diagnostics: [] + }; + for (const tsSrc of req.tsSrcs) { + const out = transpile(tsSrc, req.options); + res.jsSrcs.push(out.jsSrc); + res.diagnostics.push(out.diag); + } + threads.parentPort.postMessage(res); + }); +} +class TranspileWorker { + constructor(outFileFn) { + this.id = TranspileWorker.pool++; + this._worker = new threads.Worker(__filename); + this._durations = []; + this._worker.addListener('message', (res) => { + if (!this._pending) { + console.error('RECEIVING data WITHOUT request'); + return; + } + const [resolve, reject, files, options, t1] = this._pending; + const outFiles = []; + const diag = []; + for (let i = 0; i < res.jsSrcs.length; i++) { + // inputs and outputs are aligned across the arrays + const file = files[i]; + const jsSrc = res.jsSrcs[i]; + const diag = res.diagnostics[i]; + if (diag.length > 0) { + diag.push(...diag); + continue; + } + let SuffixTypes; + (function (SuffixTypes) { + SuffixTypes[SuffixTypes["Dts"] = 5] = "Dts"; + SuffixTypes[SuffixTypes["Ts"] = 3] = "Ts"; + SuffixTypes[SuffixTypes["Unknown"] = 0] = "Unknown"; + })(SuffixTypes || (SuffixTypes = {})); + const suffixLen = file.path.endsWith('.d.ts') ? 5 /* SuffixTypes.Dts */ + : file.path.endsWith('.ts') ? 3 /* SuffixTypes.Ts */ + : 0 /* SuffixTypes.Unknown */; + // check if output of a DTS-files isn't just "empty" and iff so + // skip this file + if (suffixLen === 5 /* SuffixTypes.Dts */ && _isDefaultEmpty(jsSrc)) { + continue; + } + const outBase = options.compilerOptions?.outDir ?? file.base; + const outPath = outFileFn(file.path); + outFiles.push(new Vinyl({ + path: outPath, + base: outBase, + contents: Buffer.from(jsSrc), + })); + } + this._pending = undefined; + this._durations.push(Date.now() - t1); + if (diag.length > 0) { + reject(diag); + } + else { + resolve(outFiles); + } + }); + } + terminate() { + // console.log(`Worker#${this.id} ENDS after ${this._durations.length} jobs (total: ${this._durations.reduce((p, c) => p + c, 0)}, avg: ${this._durations.reduce((p, c) => p + c, 0) / this._durations.length})`); + this._worker.terminate(); + } + get isBusy() { + return this._pending !== undefined; + } + next(files, options) { + if (this._pending !== undefined) { + throw new Error('BUSY'); + } + return new Promise((resolve, reject) => { + this._pending = [resolve, reject, files, options, Date.now()]; + const req = { + options, + tsSrcs: files.map(file => String(file.contents)) + }; + this._worker.postMessage(req); + }); + } +} +TranspileWorker.pool = 1; +class Transpiler { + constructor(logFn, _onError, _cmdLine) { + this._onError = _onError; + this._cmdLine = _cmdLine; + this._workerPool = []; + this._queue = []; + this._allJobs = []; + this._tsApiInternalOutfileName = new class { + constructor(parsedCmd) { + const host = ts.createCompilerHost(parsedCmd.options); + const program = ts.createProgram({ options: parsedCmd.options, rootNames: parsedCmd.fileNames, host }); + const emitHost = { + getCompilerOptions: () => parsedCmd.options, + getCurrentDirectory: () => host.getCurrentDirectory(), + getCanonicalFileName: file => host.getCanonicalFileName(file), + getCommonSourceDirectory: () => program.getCommonSourceDirectory() + }; + this.getForInfile = file => { + return ts.getOwnEmitOutputFilePath(file, emitHost, '.js'); + }; + } + }(this._cmdLine); + logFn('Transpile', `will use ${Transpiler.P} transpile worker`); + } + async join() { + // wait for all penindg jobs + this._consumeQueue(); + await Promise.allSettled(this._allJobs); + this._allJobs.length = 0; + // terminate all worker + this._workerPool.forEach(w => w.terminate()); + this._workerPool.length = 0; + } + transpile(file) { + if (this._cmdLine.options.noEmit) { + // not doing ANYTHING here + return; + } + const newLen = this._queue.push(file); + if (newLen > Transpiler.P ** 2) { + this._consumeQueue(); + } + } + _consumeQueue() { + if (this._queue.length === 0) { + // no work... + return; + } + // kinda LAZYily create workers + if (this._workerPool.length === 0) { + for (let i = 0; i < Transpiler.P; i++) { + this._workerPool.push(new TranspileWorker(file => this._tsApiInternalOutfileName.getForInfile(file))); + } + } + const freeWorker = this._workerPool.filter(w => !w.isBusy); + if (freeWorker.length === 0) { + // OK, they will pick up work themselves + return; + } + for (const worker of freeWorker) { + if (this._queue.length === 0) { + break; + } + const job = new Promise(resolve => { + const consume = () => { + const files = this._queue.splice(0, Transpiler.P); + if (files.length === 0) { + // DONE + resolve(undefined); + return; + } + // work on the NEXT file + // const [inFile, outFn] = req; + worker.next(files, { compilerOptions: this._cmdLine.options }).then(outFiles => { + if (this.onOutfile) { + outFiles.map(this.onOutfile, this); + } + consume(); + }).catch(err => { + this._onError(err); + }); + }; + consume(); + }); + this._allJobs.push(job); + } + } +} +exports.Transpiler = Transpiler; +Transpiler.P = Math.floor((0, node_os_1.cpus)().length * .5); +function _isDefaultEmpty(src) { + return src + .replace('"use strict";', '') + .replace(/\/\*[\s\S]*?\*\/|([^\\:]|^)\/\/.*$/gm, '$1') + .trim().length === 0; +} diff --git a/build/lib/tsb/transpiler.ts b/build/lib/tsb/transpiler.ts new file mode 100644 index 00000000000..ad515263f6c --- /dev/null +++ b/build/lib/tsb/transpiler.ts @@ -0,0 +1,276 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as ts from 'typescript'; +import * as threads from 'node:worker_threads'; +import * as Vinyl from 'vinyl'; +import { cpus } from 'node:os'; + +interface TranspileReq { + readonly tsSrcs: string[]; + readonly options: ts.TranspileOptions; +} + +interface TranspileRes { + readonly jsSrcs: string[]; + readonly diagnostics: ts.Diagnostic[][]; +} + +function transpile(tsSrc: string, options: ts.TranspileOptions): { jsSrc: string; diag: ts.Diagnostic[] } { + + const isAmd = /\n(import|export)/m.test(tsSrc); + if (!isAmd && options.compilerOptions?.module === ts.ModuleKind.AMD) { + // enforce NONE module-system for not-amd cases + options = { ...options, ...{ compilerOptions: { ...options.compilerOptions, module: ts.ModuleKind.None } } }; + } + const out = ts.transpileModule(tsSrc, options); + return { + jsSrc: out.outputText, + diag: out.diagnostics ?? [] + }; +} + +if (!threads.isMainThread) { + // WORKER + threads.parentPort?.addListener('message', (req: TranspileReq) => { + const res: TranspileRes = { + jsSrcs: [], + diagnostics: [] + }; + for (const tsSrc of req.tsSrcs) { + const out = transpile(tsSrc, req.options); + res.jsSrcs.push(out.jsSrc); + res.diagnostics.push(out.diag); + } + threads.parentPort!.postMessage(res); + }); +} + +class TranspileWorker { + + private static pool = 1; + + readonly id = TranspileWorker.pool++; + + private _worker = new threads.Worker(__filename); + private _pending?: [resolve: Function, reject: Function, file: Vinyl[], options: ts.TranspileOptions, t1: number]; + private _durations: number[] = []; + + constructor(outFileFn: (fileName: string) => string) { + + this._worker.addListener('message', (res: TranspileRes) => { + if (!this._pending) { + console.error('RECEIVING data WITHOUT request'); + return; + } + + const [resolve, reject, files, options, t1] = this._pending; + + const outFiles: Vinyl[] = []; + const diag: ts.Diagnostic[] = []; + + for (let i = 0; i < res.jsSrcs.length; i++) { + // inputs and outputs are aligned across the arrays + const file = files[i]; + const jsSrc = res.jsSrcs[i]; + const diag = res.diagnostics[i]; + + if (diag.length > 0) { + diag.push(...diag); + continue; + } + const enum SuffixTypes { + Dts = 5, + Ts = 3, + Unknown = 0 + } + const suffixLen = file.path.endsWith('.d.ts') ? SuffixTypes.Dts + : file.path.endsWith('.ts') ? SuffixTypes.Ts + : SuffixTypes.Unknown; + + // check if output of a DTS-files isn't just "empty" and iff so + // skip this file + if (suffixLen === SuffixTypes.Dts && _isDefaultEmpty(jsSrc)) { + continue; + } + + const outBase = options.compilerOptions?.outDir ?? file.base; + const outPath = outFileFn(file.path); + + outFiles.push(new Vinyl({ + path: outPath, + base: outBase, + contents: Buffer.from(jsSrc), + })); + } + + this._pending = undefined; + this._durations.push(Date.now() - t1); + + if (diag.length > 0) { + reject(diag); + } else { + resolve(outFiles); + } + }); + } + + terminate() { + // console.log(`Worker#${this.id} ENDS after ${this._durations.length} jobs (total: ${this._durations.reduce((p, c) => p + c, 0)}, avg: ${this._durations.reduce((p, c) => p + c, 0) / this._durations.length})`); + this._worker.terminate(); + } + + get isBusy() { + return this._pending !== undefined; + } + + next(files: Vinyl[], options: ts.TranspileOptions) { + if (this._pending !== undefined) { + throw new Error('BUSY'); + } + return new Promise((resolve, reject) => { + this._pending = [resolve, reject, files, options, Date.now()]; + const req: TranspileReq = { + options, + tsSrcs: files.map(file => String(file.contents)) + }; + this._worker.postMessage(req); + }); + } +} + + +export class Transpiler { + + static P = Math.floor(cpus().length * .5); + + public onOutfile?: (file: Vinyl) => void; + + private _workerPool: TranspileWorker[] = []; + private _queue: Vinyl[] = []; + private _allJobs: Promise[] = []; + + constructor( + logFn: (topic: string, message: string) => void, + private readonly _onError: (err: any) => void, + private readonly _cmdLine: ts.ParsedCommandLine + ) { + logFn('Transpile', `will use ${Transpiler.P} transpile worker`); + } + + async join() { + // wait for all penindg jobs + this._consumeQueue(); + await Promise.allSettled(this._allJobs); + this._allJobs.length = 0; + + // terminate all worker + this._workerPool.forEach(w => w.terminate()); + this._workerPool.length = 0; + } + + + transpile(file: Vinyl) { + + if (this._cmdLine.options.noEmit) { + // not doing ANYTHING here + return; + } + + const newLen = this._queue.push(file); + if (newLen > Transpiler.P ** 2) { + this._consumeQueue(); + } + } + + private _consumeQueue(): void { + + if (this._queue.length === 0) { + // no work... + return; + } + + // kinda LAZYily create workers + if (this._workerPool.length === 0) { + for (let i = 0; i < Transpiler.P; i++) { + this._workerPool.push(new TranspileWorker(file => this._tsApiInternalOutfileName.getForInfile(file))); + } + } + + const freeWorker = this._workerPool.filter(w => !w.isBusy); + if (freeWorker.length === 0) { + // OK, they will pick up work themselves + return; + } + + for (const worker of freeWorker) { + if (this._queue.length === 0) { + break; + } + + const job = new Promise(resolve => { + + const consume = () => { + const files = this._queue.splice(0, Transpiler.P); + if (files.length === 0) { + // DONE + resolve(undefined); + return; + } + // work on the NEXT file + // const [inFile, outFn] = req; + worker.next(files, { compilerOptions: this._cmdLine.options }).then(outFiles => { + if (this.onOutfile) { + outFiles.map(this.onOutfile, this); + } + consume(); + }).catch(err => { + this._onError(err); + }); + }; + + consume(); + }); + + this._allJobs.push(job); + } + } + + private _tsApiInternalOutfileName = new class { + + getForInfile: (file: string) => string; + + constructor(parsedCmd: ts.ParsedCommandLine) { + + type InternalTsHost = { + getCompilerOptions(): ts.CompilerOptions; + getCurrentDirectory(): string; + getCommonSourceDirectory(): string; + getCanonicalFileName(file: string): string; + }; + type InternalTsApi = { getOwnEmitOutputFilePath(fileName: string, host: InternalTsHost, extension: string): string } & typeof ts; + + const host = ts.createCompilerHost(parsedCmd.options); + const program = ts.createProgram({ options: parsedCmd.options, rootNames: parsedCmd.fileNames, host }); + const emitHost: InternalTsHost = { + getCompilerOptions: () => parsedCmd.options, + getCurrentDirectory: () => host.getCurrentDirectory(), + getCanonicalFileName: file => host.getCanonicalFileName(file), + getCommonSourceDirectory: () => (program).getCommonSourceDirectory() + }; + + this.getForInfile = file => { + return (ts).getOwnEmitOutputFilePath(file, emitHost, '.js'); + }; + } + }(this._cmdLine); +} + +function _isDefaultEmpty(src: string): boolean { + return src + .replace('"use strict";', '') + .replace(/\/\*[\s\S]*?\*\/|([^\\:]|^)\/\/.*$/gm, '$1') + .trim().length === 0; +} diff --git a/extensions/debug-auto-launch/src/extension.ts b/extensions/debug-auto-launch/src/extension.ts index 572c468d843..86de0ac6ea4 100644 --- a/extensions/debug-auto-launch/src/extension.ts +++ b/extensions/debug-auto-launch/src/extension.ts @@ -8,6 +8,12 @@ import { createServer, Server } from 'net'; import * as vscode from 'vscode'; import * as nls from 'vscode-nls'; +const enum State { + Disabled = 'disabled', + OnlyWithFlag = 'onlyWithFlag', + Smart = 'smart', + Always = 'always', +} const localize = nls.loadMessageBundle(); const TEXT_STATUSBAR_LABEL = { [State.Disabled]: localize('status.text.auto.attach.disabled', 'Auto Attach: Disabled'), @@ -62,12 +68,6 @@ const SETTINGS_CAUSE_REFRESH = new Set( ['autoAttachSmartPattern', SETTING_STATE].map(s => `${SETTING_SECTION}.${s}`), ); -const enum State { - Disabled = 'disabled', - OnlyWithFlag = 'onlyWithFlag', - Smart = 'smart', - Always = 'always', -} let currentState: Promise<{ context: vscode.ExtensionContext; state: State | null }>; let statusItem: vscode.StatusBarItem | undefined; // and there is no status bar item