diff --git a/build/filters.js b/build/filters.js index 1d19218f7ef..b0f23d3f96f 100644 --- a/build/filters.js +++ b/build/filters.js @@ -77,6 +77,7 @@ module.exports.indentationFilter = [ '!src/vs/base/common/semver/semver.js', '!src/vs/base/node/terminateProcess.sh', '!src/vs/base/node/cpuUsage.sh', + '!src/vs/editor/common/languages/highlights/*.scm', '!test/unit/assert.js', '!test/unit/assert-esm.js', '!resources/linux/snap/electron-launch', diff --git a/build/gulpfile.editor.js b/build/gulpfile.editor.js index fe29d4fe183..088748b8356 100644 --- a/build/gulpfile.editor.js +++ b/build/gulpfile.editor.js @@ -79,7 +79,9 @@ const extractEditorSrcTask = task.define('extract-editor-src', () => { shakeLevel: 2, // 0-Files, 1-InnerFile, 2-ClassMembers importIgnorePattern: /(^vs\/css!)/, destRoot: path.join(root, 'out-editor-src'), - redirects: [] + redirects: { + '@vscode/tree-sitter-wasm': '../node_modules/@vscode/tree-sitter-wasm/wasm/tree-sitter-web', + } }); }); @@ -133,7 +135,8 @@ const compileEditorESMTask = task.define('compile-editor-esm', () => { let result; if (process.platform === 'win32') { result = cp.spawnSync(`..\\node_modules\\.bin\\tsc.cmd`, { - cwd: path.join(__dirname, '../out-editor-esm') + cwd: path.join(__dirname, '../out-editor-esm'), + shell: true }); } else { result = cp.spawnSync(`node`, [`../node_modules/.bin/tsc`], { diff --git a/build/lib/treeshaking.js b/build/lib/treeshaking.js index c8e95511877..842f691af8f 100644 --- a/build/lib/treeshaking.js +++ b/build/lib/treeshaking.js @@ -100,24 +100,22 @@ function discoverAndReadFiles(ts, options) { options.entryPoints.forEach((entryPoint) => enqueue(entryPoint)); while (queue.length > 0) { const moduleId = queue.shift(); - const dts_filename = path.join(options.sourcesRoot, moduleId + '.d.ts'); + let redirectedModuleId = moduleId; + if (options.redirects[moduleId]) { + redirectedModuleId = options.redirects[moduleId]; + } + const dts_filename = path.join(options.sourcesRoot, redirectedModuleId + '.d.ts'); if (fs.existsSync(dts_filename)) { const dts_filecontents = fs.readFileSync(dts_filename).toString(); FILES[`${moduleId}.d.ts`] = dts_filecontents; continue; } - const js_filename = path.join(options.sourcesRoot, moduleId + '.js'); + const js_filename = path.join(options.sourcesRoot, redirectedModuleId + '.js'); if (fs.existsSync(js_filename)) { // This is an import for a .js file, so ignore it... continue; } - let ts_filename; - if (options.redirects[moduleId]) { - ts_filename = path.join(options.sourcesRoot, options.redirects[moduleId] + '.ts'); - } - else { - ts_filename = path.join(options.sourcesRoot, moduleId + '.ts'); - } + const ts_filename = path.join(options.sourcesRoot, redirectedModuleId + '.ts'); const ts_filecontents = fs.readFileSync(ts_filename).toString(); const info = ts.preProcessFile(ts_filecontents); for (let i = info.importedFiles.length - 1; i >= 0; i--) { diff --git a/build/lib/treeshaking.ts b/build/lib/treeshaking.ts index 020e567eb72..d01b34d264f 100644 --- a/build/lib/treeshaking.ts +++ b/build/lib/treeshaking.ts @@ -155,25 +155,27 @@ function discoverAndReadFiles(ts: typeof import('typescript'), options: ITreeSha while (queue.length > 0) { const moduleId = queue.shift()!; - const dts_filename = path.join(options.sourcesRoot, moduleId + '.d.ts'); + let redirectedModuleId: string = moduleId; + if (options.redirects[moduleId]) { + redirectedModuleId = options.redirects[moduleId]; + } + + const dts_filename = path.join(options.sourcesRoot, redirectedModuleId + '.d.ts'); if (fs.existsSync(dts_filename)) { const dts_filecontents = fs.readFileSync(dts_filename).toString(); FILES[`${moduleId}.d.ts`] = dts_filecontents; continue; } - const js_filename = path.join(options.sourcesRoot, moduleId + '.js'); + + const js_filename = path.join(options.sourcesRoot, redirectedModuleId + '.js'); if (fs.existsSync(js_filename)) { // This is an import for a .js file, so ignore it... continue; } - let ts_filename: string; - if (options.redirects[moduleId]) { - ts_filename = path.join(options.sourcesRoot, options.redirects[moduleId] + '.ts'); - } else { - ts_filename = path.join(options.sourcesRoot, moduleId + '.ts'); - } + const ts_filename = path.join(options.sourcesRoot, redirectedModuleId + '.ts'); + const ts_filecontents = fs.readFileSync(ts_filename).toString(); const info = ts.preProcessFile(ts_filecontents); for (let i = info.importedFiles.length - 1; i >= 0; i--) { diff --git a/package.json b/package.json index 340ab4ecaff..0639a1e79de 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,7 @@ "@vscode/spdlog": "^0.15.0", "@vscode/sqlite3": "5.1.6-vscode", "@vscode/sudo-prompt": "9.3.1", - "@vscode/tree-sitter-wasm": "^0.0.1", + "@vscode/tree-sitter-wasm": "^0.0.2", "@vscode/vscode-languagedetection": "1.0.21", "@vscode/windows-mutex": "^0.5.0", "@vscode/windows-process-tree": "^0.6.0", diff --git a/remote/package.json b/remote/package.json index 3105d90868f..f08ba87447a 100644 --- a/remote/package.json +++ b/remote/package.json @@ -11,7 +11,7 @@ "@vscode/proxy-agent": "^0.22.0", "@vscode/ripgrep": "^1.15.9", "@vscode/spdlog": "^0.15.0", - "@vscode/tree-sitter-wasm": "^0.0.1", + "@vscode/tree-sitter-wasm": "^0.0.2", "@vscode/vscode-languagedetection": "1.0.21", "@vscode/windows-process-tree": "^0.6.0", "@vscode/windows-registry": "^1.1.0", diff --git a/remote/web/package.json b/remote/web/package.json index 6d2877d9893..1c75df5eb02 100644 --- a/remote/web/package.json +++ b/remote/web/package.json @@ -6,7 +6,7 @@ "@microsoft/1ds-core-js": "^3.2.13", "@microsoft/1ds-post-js": "^3.2.13", "@vscode/iconv-lite-umd": "0.7.0", - "@vscode/tree-sitter-wasm": "^0.0.1", + "@vscode/tree-sitter-wasm": "^0.0.2", "@vscode/vscode-languagedetection": "1.0.21", "@xterm/addon-clipboard": "0.2.0-beta.35", "@xterm/addon-image": "0.9.0-beta.52", diff --git a/remote/web/yarn.lock b/remote/web/yarn.lock index cf46e56d26f..519b2838d46 100644 --- a/remote/web/yarn.lock +++ b/remote/web/yarn.lock @@ -43,10 +43,10 @@ resolved "https://registry.yarnpkg.com/@vscode/iconv-lite-umd/-/iconv-lite-umd-0.7.0.tgz#d2f1e0664ee6036408f9743fee264ea0699b0e48" integrity sha512-bRRFxLfg5dtAyl5XyiVWz/ZBPahpOpPrNYnnHpOpUZvam4tKH35wdhP4Kj6PbM0+KdliOsPzbGWpkxcdpNB/sg== -"@vscode/tree-sitter-wasm@^0.0.1": - version "0.0.1" - resolved "https://registry.yarnpkg.com/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.1.tgz#ffb2e295a416698f4c77cbffeca3b28567d6754b" - integrity sha512-m0GKnQ3BxWnVd+20KLGwr1+Qvt/RiiaJmKAqHNU35pNydDtduUzyBm7ETz/T0vOVKoeIAaiYsJOA1aKWs7Y1tA== +"@vscode/tree-sitter-wasm@^0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.2.tgz#da21541d343be69bb263e9380d165e3b164ec1f0" + integrity sha512-N57MR/kt4jR0H/TXeDsVYeJmvvUiK7avow0fjy+/EeKcyNBJcM2BFhj4XOAaaMbhGsOcIeSvJFouRWctXI7sKw== "@vscode/vscode-languagedetection@1.0.21": version "1.0.21" diff --git a/remote/yarn.lock b/remote/yarn.lock index a3af51c0e36..abaad9e1f29 100644 --- a/remote/yarn.lock +++ b/remote/yarn.lock @@ -98,10 +98,10 @@ mkdirp "^1.0.4" node-addon-api "7.1.0" -"@vscode/tree-sitter-wasm@^0.0.1": - version "0.0.1" - resolved "https://registry.yarnpkg.com/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.1.tgz#ffb2e295a416698f4c77cbffeca3b28567d6754b" - integrity sha512-m0GKnQ3BxWnVd+20KLGwr1+Qvt/RiiaJmKAqHNU35pNydDtduUzyBm7ETz/T0vOVKoeIAaiYsJOA1aKWs7Y1tA== +"@vscode/tree-sitter-wasm@^0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.2.tgz#da21541d343be69bb263e9380d165e3b164ec1f0" + integrity sha512-N57MR/kt4jR0H/TXeDsVYeJmvvUiK7avow0fjy+/EeKcyNBJcM2BFhj4XOAaaMbhGsOcIeSvJFouRWctXI7sKw== "@vscode/vscode-languagedetection@1.0.21": version "1.0.21" diff --git a/src/tsconfig.monaco.json b/src/tsconfig.monaco.json index bad9fb8cacc..bb4fd1ccc35 100644 --- a/src/tsconfig.monaco.json +++ b/src/tsconfig.monaco.json @@ -8,7 +8,7 @@ ], "paths": {}, "module": "amd", - "moduleResolution": "classic", + "moduleResolution": "node", "removeComments": false, "preserveConstEnums": true, "target": "ES2022", diff --git a/src/vs/base/common/event.ts b/src/vs/base/common/event.ts index 62154bd99df..95bb4678c83 100644 --- a/src/vs/base/common/event.ts +++ b/src/vs/base/common/event.ts @@ -111,6 +111,15 @@ export namespace Event { }; } + /** + * Given an event, returns another event which only fires once, and only when the condition is met. + * + * @param event The event source for the new event. + */ + export function onceIf(event: Event, condition: (e: T) => boolean): Event { + return Event.once(Event.filter(event, condition)); + } + /** * Maps an event of one type into an event of another type using a mapping function, similar to how * `Array.prototype.map` works. diff --git a/src/vs/editor/browser/services/treeSitter/treeSitterParserService.ts b/src/vs/editor/browser/services/treeSitter/treeSitterParserService.ts index 649aaa45bea..5433241400b 100644 --- a/src/vs/editor/browser/services/treeSitter/treeSitterParserService.ts +++ b/src/vs/editor/browser/services/treeSitter/treeSitterParserService.ts @@ -3,26 +3,26 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { TreeSitterTokenizationRegistry } from 'vs/editor/common/languages'; import type { Parser } from '@vscode/tree-sitter-wasm'; import { AppResourcePath, FileAccess, nodeModulesAsarUnpackedPath, nodeModulesPath } from 'vs/base/common/network'; -import { ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'; +import { EDITOR_EXPERIMENTAL_PREFER_TREESITTER, ITreeSitterParserService, ITreeSitterParseResult } from 'vs/editor/common/services/treeSitterParserService'; import { IModelService } from 'vs/editor/common/services/model'; -import { Disposable, DisposableMap, DisposableStore, IDisposable } from 'vs/base/common/lifecycle'; +import { Disposable, DisposableMap, DisposableStore, dispose, IDisposable } from 'vs/base/common/lifecycle'; import { ITextModel } from 'vs/editor/common/model'; import { IFileService } from 'vs/platform/files/common/files'; -import { IModelContentChangedEvent } from 'vs/editor/common/textModelEvents'; +import { IModelContentChange } from 'vs/editor/common/textModelEvents'; import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry'; import { ILogService } from 'vs/platform/log/common/log'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { setTimeout0 } from 'vs/base/common/platform'; import { importAMDNodeModule } from 'vs/amdX'; -import { Event } from 'vs/base/common/event'; -import { cancelOnDispose } from 'vs/base/common/cancellation'; +import { Emitter, Event } from 'vs/base/common/event'; +import { CancellationToken, cancelOnDispose } from 'vs/base/common/cancellation'; import { IEnvironmentService } from 'vs/platform/environment/common/environment'; import { canASAR } from 'vs/base/common/amd'; +import { CancellationError, isCancellationError } from 'vs/base/common/errors'; +import { PromiseResult } from 'vs/base/common/observableInternal/promise'; -const EDITOR_EXPERIMENTAL_PREFER_TREESITTER = 'editor.experimental.preferTreeSitter'; const EDITOR_TREESITTER_TELEMETRY = 'editor.experimental.treeSitterTelemetry'; const MODULE_LOCATION_SUBPATH = `@vscode/tree-sitter-wasm/wasm`; const FILENAME_TREESITTER_WASM = `tree-sitter.wasm`; @@ -33,13 +33,12 @@ function getModuleLocation(environmentService: IEnvironmentService): AppResource export class TextModelTreeSitter extends Disposable { - private _treeSitterTree: TreeSitterTree | undefined; + private _parseResult: TreeSitterParseResult | undefined; - // Not currently used since we just get telemetry, but later this will be needed. - get tree() { return this._treeSitterTree; } + get parseResult(): ITreeSitterParseResult | undefined { return this._parseResult; } constructor(readonly model: ITextModel, - private readonly _treeSitterParser: TreeSitterLanguages, + private readonly _treeSitterLanguages: TreeSitterLanguages, private readonly _treeSitterImporter: TreeSitterImporter, private readonly _logService: ILogService, private readonly _telemetryService: ITelemetryService @@ -54,12 +53,17 @@ export class TextModelTreeSitter extends Disposable { */ private async _onDidChangeLanguage(languageId: string) { this._languageSessionDisposables.clear(); - this._treeSitterTree = undefined; + this._parseResult = undefined; const token = cancelOnDispose(this._languageSessionDisposables); - const language = await this._treeSitterParser.getLanguage(languageId); - if (!language || token.isCancellationRequested) { - return; + let language: Parser.Language | undefined; + try { + language = await this._getLanguage(languageId, token); + } catch (e) { + if (isCancellationError(e)) { + return; + } + throw e; } const Parser = await this._treeSitterImporter.getParserClass(); @@ -67,18 +71,39 @@ export class TextModelTreeSitter extends Disposable { return; } - const treeSitterTree = this._languageSessionDisposables.add(new TreeSitterTree(new Parser(), language, this._logService, this._telemetryService)); - this._languageSessionDisposables.add(this.model.onDidChangeContent(e => this._onDidChangeContent(treeSitterTree, e))); - await this._onDidChangeContent(treeSitterTree); + const treeSitterTree = this._languageSessionDisposables.add(new TreeSitterParseResult(new Parser(), language, this._logService, this._telemetryService)); + this._languageSessionDisposables.add(this.model.onDidChangeContent(e => this._onDidChangeContent(treeSitterTree, e.changes))); + await this._onDidChangeContent(treeSitterTree, []); if (token.isCancellationRequested) { return; } - this._treeSitterTree = treeSitterTree; + this._parseResult = treeSitterTree; } - private async _onDidChangeContent(treeSitterTree: TreeSitterTree, e?: IModelContentChangedEvent) { - return treeSitterTree.onDidChangeContent(this.model, e); + private _getLanguage(languageId: string, token: CancellationToken): Promise { + const language = this._treeSitterLanguages.getOrInitLanguage(languageId); + if (language) { + return Promise.resolve(language); + } + const disposables: IDisposable[] = []; + + return new Promise((resolve, reject) => { + disposables.push(this._treeSitterLanguages.onDidAddLanguage(e => { + if (e.id === languageId) { + dispose(disposables); + resolve(e.language); + } + })); + token.onCancellationRequested(() => { + dispose(disposables); + reject(new CancellationError()); + }, undefined, disposables); + }); + } + + private async _onDidChangeContent(treeSitterTree: TreeSitterParseResult, changes: IModelContentChange[]) { + return treeSitterTree.onDidChangeContent(this.model, changes); } } @@ -87,7 +112,7 @@ const enum TelemetryParseType { Incremental = 'incrementalParse' } -export class TreeSitterTree implements IDisposable { +export class TreeSitterParseResult implements IDisposable, ITreeSitterParseResult { private _tree: Parser.Tree | undefined; private _isDisposed: boolean = false; constructor(public readonly parser: Parser, @@ -103,47 +128,48 @@ export class TreeSitterTree implements IDisposable { this.parser?.delete(); } get tree() { return this._tree; } - set tree(newTree: Parser.Tree | undefined) { + private set tree(newTree: Parser.Tree | undefined) { this._tree?.delete(); this._tree = newTree; } get isDisposed() { return this._isDisposed; } private _onDidChangeContentQueue: Promise = Promise.resolve(); - public async onDidChangeContent(model: ITextModel, e?: IModelContentChangedEvent) { + public async onDidChangeContent(model: ITextModel, changes: IModelContentChange[]) { + this._applyEdits(model, changes); this._onDidChangeContentQueue = this._onDidChangeContentQueue.then(() => { if (this.isDisposed) { // No need to continue the queue if we are disposed return; } - return this._onDidChangeContent(model, e); + return this._parseAndUpdateTree(model); }).catch((e) => { this._logService.error('Error parsing tree-sitter tree', e); }); return this._onDidChangeContentQueue; } - private async _onDidChangeContent(model: ITextModel, e?: IModelContentChangedEvent) { - if (e) { - for (const change of e.changes) { - const newEndOffset = change.rangeOffset + change.text.length; - const newEndPosition = model.getPositionAt(newEndOffset); + private _applyEdits(model: ITextModel, changes: IModelContentChange[]) { + for (const change of changes) { + const newEndOffset = change.rangeOffset + change.text.length; + const newEndPosition = model.getPositionAt(newEndOffset); - this.tree?.edit({ - startIndex: change.rangeOffset, - oldEndIndex: change.rangeOffset + change.rangeLength, - newEndIndex: change.rangeOffset + change.text.length, - startPosition: { row: change.range.startLineNumber - 1, column: change.range.startColumn - 1 }, - oldEndPosition: { row: change.range.endLineNumber - 1, column: change.range.endColumn - 1 }, - newEndPosition: { row: newEndPosition.lineNumber - 1, column: newEndPosition.column - 1 } - }); - } + this.tree?.edit({ + startIndex: change.rangeOffset, + oldEndIndex: change.rangeOffset + change.rangeLength, + newEndIndex: change.rangeOffset + change.text.length, + startPosition: { row: change.range.startLineNumber - 1, column: change.range.startColumn - 1 }, + oldEndPosition: { row: change.range.endLineNumber - 1, column: change.range.endColumn - 1 }, + newEndPosition: { row: newEndPosition.lineNumber - 1, column: newEndPosition.column - 1 } + }); } - - this.tree = await this.parse(model); } - private parse(model: ITextModel): Promise { + private async _parseAndUpdateTree(model: ITextModel) { + this.tree = await this._parse(model); + } + + private _parse(model: ITextModel): Promise { let parseType: TelemetryParseType = TelemetryParseType.Full; if (this.tree) { parseType = TelemetryParseType.Incremental; @@ -200,41 +226,58 @@ export class TreeSitterTree implements IDisposable { } export class TreeSitterLanguages extends Disposable { - private _languages: Map = new Map(); + private _languages: AsyncCache = new AsyncCache(); + public /*exposed for tests*/ readonly _onDidAddLanguage: Emitter<{ id: string; language: Parser.Language }> = this._register(new Emitter()); + /** + * If you're looking for a specific language, make sure to check if it already exists with `getLanguage` as it will kick off the process to add it if it doesn't exist. + */ + public readonly onDidAddLanguage: Event<{ id: string; language: Parser.Language }> = this._onDidAddLanguage.event; constructor(private readonly _treeSitterImporter: TreeSitterImporter, private readonly _fileService: IFileService, - private readonly _environmentService: IEnvironmentService + private readonly _environmentService: IEnvironmentService, + private readonly _registeredLanguages: Map, ) { super(); } - public async getLanguage(languageId: string): Promise { - let language = this._languages.get(languageId); + public getOrInitLanguage(languageId: string): Parser.Language | undefined { + if (this._languages.isCached(languageId)) { + return this._languages.getSyncIfCached(languageId); + } else { + // kick off adding the language, but don't wait + this._addLanguage(languageId); + return undefined; + } + } + + private async _addLanguage(languageId: string): Promise { + let language = this._languages.getSyncIfCached(languageId); if (!language) { - language = await this._fetchLanguage(languageId); + const fetchPromise = this._fetchLanguage(languageId); + this._languages.set(languageId, fetchPromise); + language = await fetchPromise; if (!language) { return undefined; } - this._languages.set(languageId, language); + this._onDidAddLanguage.fire({ id: languageId, language }); } - return language; } private async _fetchLanguage(languageId: string): Promise { - const grammarName = TreeSitterTokenizationRegistry.get(languageId); + const grammarName = this._registeredLanguages.get(languageId); const languageLocation = this._getLanguageLocation(languageId); if (!grammarName || !languageLocation) { return undefined; } - const wasmPath: AppResourcePath = `${languageLocation}/${grammarName.name}.wasm`; + const wasmPath: AppResourcePath = `${languageLocation}/${grammarName}.wasm`; const languageFile = await (this._fileService.readFile(FileAccess.asFileUri(wasmPath))); const Parser = await this._treeSitterImporter.getParserClass(); return Parser.Language.load(languageFile.value.buffer); } private _getLanguageLocation(languageId: string): AppResourcePath | undefined { - const grammarName = TreeSitterTokenizationRegistry.get(languageId); + const grammarName = this._registeredLanguages.get(languageId); if (!grammarName) { return undefined; } @@ -264,9 +307,11 @@ export class TreeSitterTextModelService extends Disposable implements ITreeSitte readonly _serviceBrand: undefined; private _init!: Promise; private _textModelTreeSitters: DisposableMap = this._register(new DisposableMap()); - private _registeredLanguages: DisposableMap = this._register(new DisposableMap()); + private readonly _registeredLanguages: Map = new Map(); private readonly _treeSitterImporter: TreeSitterImporter = new TreeSitterImporter(); - private readonly _treeSitterParser: TreeSitterLanguages; + private readonly _treeSitterLanguages: TreeSitterLanguages; + + public readonly onDidAddLanguage: Event<{ id: string; language: Parser.Language }>; constructor(@IModelService private readonly _modelService: IModelService, @IFileService fileService: IFileService, @@ -276,7 +321,8 @@ export class TreeSitterTextModelService extends Disposable implements ITreeSitte @IEnvironmentService private readonly _environmentService: IEnvironmentService ) { super(); - this._treeSitterParser = this._register(new TreeSitterLanguages(this._treeSitterImporter, fileService, this._environmentService)); + this._treeSitterLanguages = this._register(new TreeSitterLanguages(this._treeSitterImporter, fileService, this._environmentService, this._registeredLanguages)); + this.onDidAddLanguage = this._treeSitterLanguages.onDidAddLanguage; this._register(this._configurationService.onDidChangeConfiguration(e => { if (e.affectsConfiguration(EDITOR_EXPERIMENTAL_PREFER_TREESITTER)) { this._supportedLanguagesChanged(); @@ -285,6 +331,15 @@ export class TreeSitterTextModelService extends Disposable implements ITreeSitte this._supportedLanguagesChanged(); } + getOrInitLanguage(languageId: string): Parser.Language | undefined { + return this._treeSitterLanguages.getOrInitLanguage(languageId); + } + + getParseResult(textModel: ITextModel): ITreeSitterParseResult | undefined { + const textModelTreeSitter = this._textModelTreeSitters.get(textModel); + return textModelTreeSitter?.parseResult; + } + private async _doInitParser() { const Parser = await this._treeSitterImporter.getParserClass(); const environmentService = this._environmentService; @@ -356,19 +411,57 @@ export class TreeSitterTextModelService extends Disposable implements ITreeSitte } private _createTextModelTreeSitter(model: ITextModel) { - const textModelTreeSitter = new TextModelTreeSitter(model, this._treeSitterParser, this._treeSitterImporter, this._logService, this._telemetryService); + const textModelTreeSitter = new TextModelTreeSitter(model, this._treeSitterLanguages, this._treeSitterImporter, this._logService, this._telemetryService); this._textModelTreeSitters.set(model, textModelTreeSitter); } private _addGrammar(languageId: string, grammarName: string) { - if (!TreeSitterTokenizationRegistry.get(languageId)) { - this._registeredLanguages.set(languageId, TreeSitterTokenizationRegistry.register(languageId, { name: grammarName })); + if (!this._registeredLanguages.has(languageId)) { + this._registeredLanguages.set(languageId, grammarName); } } private _removeGrammar(languageId: string) { if (this._registeredLanguages.has(languageId)) { - this._registeredLanguages.deleteAndDispose('typescript'); + this._registeredLanguages.delete('typescript'); } } } + +class PromiseWithSyncAccess { + private _result: PromiseResult | undefined; + /** + * Returns undefined if the promise did not resolve yet. + */ + get result(): PromiseResult | undefined { + return this._result; + } + + constructor(public readonly promise: Promise) { + promise.then(result => { + this._result = new PromiseResult(result, undefined); + }).catch(e => { + this._result = new PromiseResult(undefined, e); + }); + } +} + +class AsyncCache { + private readonly _values = new Map>(); + + set(key: TKey, promise: Promise) { + this._values.set(key, new PromiseWithSyncAccess(promise)); + } + + get(key: TKey): Promise | undefined { + return this._values.get(key)?.promise; + } + + getSyncIfCached(key: TKey): T | undefined { + return this._values.get(key)?.result?.data; + } + + isCached(key: TKey): boolean { + return this._values.get(key)?.result !== undefined; + } +} diff --git a/src/vs/editor/common/languages.ts b/src/vs/editor/common/languages.ts index f1f65bfe99f..fbb24ec6185 100644 --- a/src/vs/editor/common/languages.ts +++ b/src/vs/editor/common/languages.ts @@ -87,7 +87,8 @@ export class EncodedTokenizationResult { * @internal */ export interface ITreeSitterTokenizationSupport { - name: string; + tokenizeEncoded(lineNumber: number, textModel: model.ITextModel): Uint32Array | undefined; + captureAtPosition(lineNumber: number, column: number, textModel: model.ITextModel): any; } /** @@ -2121,10 +2122,10 @@ export interface ILazyTokenizationSupport { /** * @internal */ -export class LazyTokenizationSupport implements IDisposable, ILazyTokenizationSupport { - private _tokenizationSupport: Promise | null = null; +export class LazyTokenizationSupport implements IDisposable, ILazyTokenizationSupport { + private _tokenizationSupport: Promise | null = null; - constructor(private readonly createSupport: () => Promise) { + constructor(private readonly createSupport: () => Promise) { } dispose(): void { @@ -2137,7 +2138,7 @@ export class LazyTokenizationSupport implements IDisposable, ILazyTokenizationSu } } - get tokenizationSupport(): Promise { + get tokenizationSupport(): Promise { if (!this._tokenizationSupport) { this._tokenizationSupport = this.createSupport(); } diff --git a/src/vs/editor/common/languages/highlights/typescript.scm b/src/vs/editor/common/languages/highlights/typescript.scm new file mode 100644 index 00000000000..e50bc9dec97 --- /dev/null +++ b/src/vs/editor/common/languages/highlights/typescript.scm @@ -0,0 +1,271 @@ +; Order matters! Place higher precedence first. +; Adapted from https://github.com/zed-industries/zed/blob/main/crates/languages/src/typescript/highlights.scm + +; Language constants + +[ + (true) + (false) + (null) + (undefined) +] @constant.language + +(namespace_import + "*" @constant.language) + +; Keywords + +[ + "delete" + "in" + "infer" + "instanceof" + "keyof" + "of" + "typeof" +] @keyword.operator.expression + +[ + "as" + "await" + "break" + "case" + "catch" + "continue" + "default" + "do" + "else" + "export" + "finally" + "for" + "from" + "if" + "import" + "require" + "return" + "satisfies" + "switch" + "throw" + "try" + "type" + "while" + "yield" +] @keyword.control + +[ + "abstract" + "async" + "declare" + "extends" + "implements" + "override" + "private" + "protected" + "public" + "readonly" + "static" +] @storage.modifier + +[ + "=>" + "class" + "const" + "enum" + "function" + "get" + "interface" + "let" + "namespace" + "set" + "var" +] @storage.type + +[ + "debugger" + "target" + "with" +] @keyword + +; TODO: works in the playground but not here +(regex_flags) @keyword + +[ + "void" +] @support.type + +[ + "new" +] @keyword.operator.new + +; Tokens + +[ + ";" + "?." + "." + "," + ":" + "?" +] @punctuation.delimiter + +[ + "-" + "--" + "-=" + "+" + "++" + "+=" + "*" + "*=" + "**" + "**=" + "/" + "/=" + "%" + "%=" + "<" + "<=" + "<<" + "<<=" + "=" + "==" + "===" + "!" + "!=" + "!==" + "=>" + ">" + ">=" + ">>" + ">>=" + ">>>" + ">>>=" + "~" + "^" + "&" + "|" + "^=" + "&=" + "|=" + "&&" + "||" + "??" + "&&=" + "||=" + "??=" +] @keyword.operator + +; Special identifiers + +(type_identifier) @entity.name.type +(predefined_type) @support.type + +(("const") + (variable_declarator + name: (identifier) @variable.other.constant)) + +([ + (identifier) + (shorthand_property_identifier) + (shorthand_property_identifier_pattern)] @variable.other.constant + (#match? @variable.other.constant "^[A-Z][A-Z_]+$")) + +(extends_clause + value: (identifier) @entity.other.inherited-class) + +; Function and method calls + +(call_expression + function: (identifier) @entity.name.function) + +(call_expression + function: (member_expression + property: (property_identifier) @entity.name.function)) + +(new_expression + constructor: (identifier) @entity.name.function) + +; Function and method definitions + +(function_expression + name: (identifier) @entity.name.function) +(function_declaration + name: (identifier) @entity.name.function) +(method_definition + name: (property_identifier) @storage.type + (#eq? @storage.type "constructor")) +(method_definition + name: (property_identifier) @entity.name.function) +(method_signature + name: (property_identifier) @entity.name.function) + +(pair + key: (property_identifier) @entity.name.function + value: [(function_expression) (arrow_function)]) + +(assignment_expression + left: (member_expression + property: (property_identifier) @entity.name.function) + right: [(function_expression) (arrow_function)]) + +(variable_declarator + name: (identifier) @entity.name.function + value: [(function_expression) (arrow_function)]) + +(assignment_expression + left: (identifier) @entity.name.function + right: [(function_expression) (arrow_function)]) + +; Properties + +(member_expression + object: (this) + property: (property_identifier) @variable) + +(member_expression + property: (property_identifier) @variable.other.constant + (#match? @variable.other.constant "^[A-Z][A-Z_]+$")) + +[ + (property_identifier) + (shorthand_property_identifier) + (shorthand_property_identifier_pattern)] @variable + +; Variables + +(identifier) @variable + +; Template TODO: These don't seem to be working + +(template_substitution + "${" @punctuation.definition.template-expression.begin + "}" @punctuation.definition.template-expression.end) + +(template_type + "${" @punctuation.definition.template-expression.begin + "}" @punctuation.definition.template-expression.end) + +(type_arguments + "<" @punctuation.bracket + ">" @punctuation.bracket) + +; Literals + +(this) @variable.language +(super) @variable.language + +(comment) @comment + +; TODO: This doesn't seem to be working +(escape_sequence) @constant.character.escape + +[ + (string) + (template_string) + (template_literal_type) +] @string + +; NOTE: the typescript grammar doesn't break regex into nice parts so as to capture parts of it separately +(regex) @string.regexp +(number) @constant.numeric + diff --git a/src/vs/editor/common/model/tokenizationTextModelPart.ts b/src/vs/editor/common/model/tokenizationTextModelPart.ts index a20d85f76ea..47a6656572d 100644 --- a/src/vs/editor/common/model/tokenizationTextModelPart.ts +++ b/src/vs/editor/common/model/tokenizationTextModelPart.ts @@ -6,14 +6,14 @@ import { CharCode } from 'vs/base/common/charCode'; import { BugIndicatingError, onUnexpectedError } from 'vs/base/common/errors'; import { Emitter, Event } from 'vs/base/common/event'; -import { DisposableMap, MutableDisposable } from 'vs/base/common/lifecycle'; +import { DisposableMap, DisposableStore, MutableDisposable } from 'vs/base/common/lifecycle'; import { countEOL } from 'vs/editor/common/core/eolCounter'; import { LineRange } from 'vs/editor/common/core/lineRange'; import { IPosition, Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; import { IWordAtPosition, getWordAtText } from 'vs/editor/common/core/wordHelper'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; -import { IBackgroundTokenizationStore, IBackgroundTokenizer, ILanguageIdCodec, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/languages'; +import { IBackgroundTokenizationStore, IBackgroundTokenizer, ILanguageIdCodec, IState, ITokenizationSupport, TokenizationRegistry, TreeSitterTokenizationRegistry } from 'vs/editor/common/languages'; import { ILanguageService } from 'vs/editor/common/languages/language'; import { ILanguageConfigurationService, LanguageConfigurationServiceChangeEvent, ResolvedLanguageConfiguration } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { IAttachedView } from 'vs/editor/common/model'; @@ -22,6 +22,8 @@ import { TextModel } from 'vs/editor/common/model/textModel'; import { TextModelPart } from 'vs/editor/common/model/textModelPart'; import { DefaultBackgroundTokenizer, TokenizerWithStateStoreAndTextModel, TrackingTokenizationStateStore } from 'vs/editor/common/model/textModelTokens'; import { AbstractTokens, AttachedViewHandler, AttachedViews } from 'vs/editor/common/model/tokens'; +import { TreeSitterTokens } from 'vs/editor/common/model/treeSitterTokens'; +import { ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'; import { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent } from 'vs/editor/common/textModelEvents'; import { BackgroundTokenizationState, ITokenizationTextModelPart } from 'vs/editor/common/tokenizationTextModelPart'; import { ContiguousMultilineTokens } from 'vs/editor/common/tokens/contiguousMultilineTokens'; @@ -43,7 +45,8 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz private readonly _onDidChangeTokens: Emitter = this._register(new Emitter()); public readonly onDidChangeTokens: Event = this._onDidChangeTokens.event; - private readonly tokens = this._register(new GrammarTokens(this._languageService.languageIdCodec, this._textModel, () => this._languageId, this._attachedViews)); + private _tokens!: AbstractTokens; + private readonly _tokensDisposables: DisposableStore = this._register(new DisposableStore()); constructor( private readonly _textModel: TextModel, @@ -52,16 +55,61 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz private readonly _attachedViews: AttachedViews, @ILanguageService private readonly _languageService: ILanguageService, @ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService, + @ITreeSitterParserService private readonly _treeSitterService: ITreeSitterParserService, ) { super(); - this._register(this.tokens.onDidChangeTokens(e => { + this._register(this._languageConfigurationService.onDidChange(e => { + if (e.affects(this._languageId)) { + this._onDidChangeLanguageConfiguration.fire({}); + } + })); + + // We just look at registry changes to determine whether to use tree sitter. + // This means that removing a language from the setting will not cause a switch to textmate and will require a reload. + // Adding a language to the setting will not need a reload, however. + this._register(Event.filter(TreeSitterTokenizationRegistry.onDidChange, (e) => e.changedLanguages.includes(this._languageId))(() => { + this.createPreferredTokenProvider(); + })); + this.createPreferredTokenProvider(); + } + + private createGrammarTokens() { + return this._register(new GrammarTokens(this._languageService.languageIdCodec, this._textModel, () => this._languageId, this._attachedViews)); + } + + private createTreeSitterTokens(): AbstractTokens { + return this._register(new TreeSitterTokens(this._treeSitterService, this._languageService.languageIdCodec, this._textModel, () => this._languageId)); + } + + private createTokens(useTreeSitter: boolean): void { + const needsReset = this._tokens !== undefined; + this._tokens?.dispose(); + this._tokens = useTreeSitter ? this.createTreeSitterTokens() : this.createGrammarTokens(); + this._tokensDisposables.clear(); + this._tokensDisposables.add(this._tokens.onDidChangeTokens(e => { this._emitModelTokensChangedEvent(e); })); - this._register(this.tokens.onDidChangeBackgroundTokenizationState(e => { + this._tokensDisposables.add(this._tokens.onDidChangeBackgroundTokenizationState(e => { this._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState(); })); + if (needsReset) { + // We need to reset the tokenization, as the new token provider otherwise won't have a chance to provide tokens until some action happens in the editor. + this._tokens.resetTokenization(); + } + } + + private createPreferredTokenProvider() { + if (TreeSitterTokenizationRegistry.get(this._languageId)) { + if (!(this._tokens instanceof TreeSitterTokens)) { + this.createTokens(true); + } + } else { + if (!(this._tokens instanceof GrammarTokens)) { + this.createTokens(false); + } + } } _hasListeners(): boolean { @@ -93,11 +141,11 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz } } - this.tokens.handleDidChangeContent(e); + this._tokens.handleDidChangeContent(e); } public handleDidChangeAttached(): void { - this.tokens.handleDidChangeAttached(); + this._tokens.handleDidChangeAttached(); } /** @@ -105,7 +153,7 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz */ public getLineTokens(lineNumber: number): LineTokens { this.validateLineNumber(lineNumber); - const syntacticTokens = this.tokens.getLineTokens(lineNumber); + const syntacticTokens = this._tokens.getLineTokens(lineNumber); return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens); } @@ -125,43 +173,43 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz } public get hasTokens(): boolean { - return this.tokens.hasTokens; + return this._tokens.hasTokens; } public resetTokenization() { - this.tokens.resetTokenization(); + this._tokens.resetTokenization(); } public get backgroundTokenizationState() { - return this.tokens.backgroundTokenizationState; + return this._tokens.backgroundTokenizationState; } public forceTokenization(lineNumber: number): void { this.validateLineNumber(lineNumber); - this.tokens.forceTokenization(lineNumber); + this._tokens.forceTokenization(lineNumber); } public hasAccurateTokensForLine(lineNumber: number): boolean { this.validateLineNumber(lineNumber); - return this.tokens.hasAccurateTokensForLine(lineNumber); + return this._tokens.hasAccurateTokensForLine(lineNumber); } public isCheapToTokenize(lineNumber: number): boolean { this.validateLineNumber(lineNumber); - return this.tokens.isCheapToTokenize(lineNumber); + return this._tokens.isCheapToTokenize(lineNumber); } public tokenizeIfCheap(lineNumber: number): void { this.validateLineNumber(lineNumber); - this.tokens.tokenizeIfCheap(lineNumber); + this._tokens.tokenizeIfCheap(lineNumber); } public getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType { - return this.tokens.getTokenTypeIfInsertingCharacter(lineNumber, column, character); + return this._tokens.getTokenTypeIfInsertingCharacter(lineNumber, column, character); } public tokenizeLineWithEdit(position: IPosition, length: number, newText: string): LineTokens | null { - return this.tokens.tokenizeLineWithEdit(position, length, newText); + return this._tokens.tokenizeLineWithEdit(position, length, newText); } // #endregion @@ -326,7 +374,8 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz this._languageId = languageId; this._bracketPairsTextModelPart.handleDidChangeLanguage(e); - this.tokens.resetTokenization(); + this._tokens.resetTokenization(); + this.createPreferredTokenProvider(); this._onDidChangeLanguage.fire(e); this._onDidChangeLanguageConfiguration.fire({}); } diff --git a/src/vs/editor/common/model/treeSitterTokens.ts b/src/vs/editor/common/model/treeSitterTokens.ts new file mode 100644 index 00000000000..2da4bdc84e0 --- /dev/null +++ b/src/vs/editor/common/model/treeSitterTokens.ts @@ -0,0 +1,100 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ILanguageIdCodec, ITreeSitterTokenizationSupport, TreeSitterTokenizationRegistry } from 'vs/editor/common/languages'; +import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { TextModel } from 'vs/editor/common/model/textModel'; +import { ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'; +import { IModelContentChangedEvent } from 'vs/editor/common/textModelEvents'; +import { AbstractTokens } from 'vs/editor/common/model/tokens'; +import { IPosition } from 'vs/editor/common/core/position'; + +export class TreeSitterTokens extends AbstractTokens { + private _tokenizationSupport: ITreeSitterTokenizationSupport | null = null; + private _lastLanguageId: string | undefined; + + constructor(private readonly _treeSitterService: ITreeSitterParserService, + languageIdCodec: ILanguageIdCodec, + textModel: TextModel, + languageId: () => string) { + super(languageIdCodec, textModel, languageId); + + this._initialize(); + } + + private _initialize() { + const newLanguage = this.getLanguageId(); + if (!this._tokenizationSupport || this._lastLanguageId !== newLanguage) { + this._lastLanguageId = newLanguage; + this._tokenizationSupport = TreeSitterTokenizationRegistry.get(newLanguage); + } + } + + public getLineTokens(lineNumber: number): LineTokens { + const content = this._textModel.getLineContent(lineNumber); + if (this._tokenizationSupport) { + const rawTokens = this._tokenizationSupport.tokenizeEncoded(lineNumber, this._textModel); + if (rawTokens) { + return new LineTokens(rawTokens, content, this._languageIdCodec); + } + } + return LineTokens.createEmpty(content, this._languageIdCodec); + } + + public resetTokenization(fireTokenChangeEvent: boolean = true): void { + if (fireTokenChangeEvent) { + this._onDidChangeTokens.fire({ + semanticTokensApplied: false, + ranges: [ + { + fromLineNumber: 1, + toLineNumber: this._textModel.getLineCount(), + }, + ], + }); + } + this._initialize(); + } + + public override handleDidChangeAttached(): void { + // TODO @alexr00 implement for background tokenization + } + + public override handleDidChangeContent(e: IModelContentChangedEvent): void { + if (e.isFlush) { + // Don't fire the event, as the view might not have got the text change event yet + this.resetTokenization(false); + } + } + + public override forceTokenization(lineNumber: number): void { + // TODO @alexr00 implement + } + + public override hasAccurateTokensForLine(lineNumber: number): boolean { + // TODO @alexr00 update for background tokenization + return true; + } + + public override isCheapToTokenize(lineNumber: number): boolean { + // TODO @alexr00 update for background tokenization + return true; + } + + public override getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType { + // TODO @alexr00 implement once we have custom parsing and don't just feed in the whole text model value + return StandardTokenType.Other; + } + public override tokenizeLineWithEdit(position: IPosition, length: number, newText: string): LineTokens | null { + // TODO @alexr00 understand what this is for and implement + return null; + } + public override get hasTokens(): boolean { + // TODO @alexr00 once we have a token store, implement properly + const hasTree = this._treeSitterService.getParseResult(this._textModel) !== undefined; + return hasTree; + } +} diff --git a/src/vs/editor/common/services/treeSitterParserService.ts b/src/vs/editor/common/services/treeSitterParserService.ts index e3e911efc49..aed18b37d4f 100644 --- a/src/vs/editor/common/services/treeSitterParserService.ts +++ b/src/vs/editor/common/services/treeSitterParserService.ts @@ -3,14 +3,23 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import type { Parser } from '@vscode/tree-sitter-wasm'; +import { Event } from 'vs/base/common/event'; +import { ITextModel } from 'vs/editor/common/model'; import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; +export const EDITOR_EXPERIMENTAL_PREFER_TREESITTER = 'editor.experimental.preferTreeSitter'; + export const ITreeSitterParserService = createDecorator('treeSitterParserService'); -/** - * Currently this service just logs telemetry about how long it takes to parse files. - * Actual API will come later as we add features like syntax highlighting. - */ export interface ITreeSitterParserService { readonly _serviceBrand: undefined; + onDidAddLanguage: Event<{ id: string; language: Parser.Language }>; + getOrInitLanguage(languageId: string): Parser.Language | undefined; + getParseResult(textModel: ITextModel): ITreeSitterParseResult | undefined; +} + +export interface ITreeSitterParseResult { + readonly tree: Parser.Tree | undefined; + readonly language: Parser.Language; } diff --git a/src/vs/editor/standalone/browser/standaloneServices.ts b/src/vs/editor/standalone/browser/standaloneServices.ts index ca773a2b41d..0656f65a308 100644 --- a/src/vs/editor/standalone/browser/standaloneServices.ts +++ b/src/vs/editor/standalone/browser/standaloneServices.ts @@ -96,6 +96,8 @@ import { onUnexpectedError } from 'vs/base/common/errors'; import { ExtensionKind, IEnvironmentService, IExtensionHostDebugParams } from 'vs/platform/environment/common/environment'; import { mainWindow } from 'vs/base/browser/window'; import { ResourceMap } from 'vs/base/common/map'; +import { ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'; +import { StandaloneTreeSitterParserService } from 'vs/editor/standalone/browser/standaloneTreeSitterService'; class SimpleModel implements IResolvedTextEditorModel { @@ -1158,6 +1160,7 @@ registerSingleton(IClipboardService, BrowserClipboardService, InstantiationType. registerSingleton(IContextMenuService, StandaloneContextMenuService, InstantiationType.Eager); registerSingleton(IMenuService, MenuService, InstantiationType.Eager); registerSingleton(IAccessibilitySignalService, StandaloneAccessbilitySignalService, InstantiationType.Eager); +registerSingleton(ITreeSitterParserService, StandaloneTreeSitterParserService, InstantiationType.Eager); /** * We don't want to eagerly instantiate services because embedders get a one time chance diff --git a/src/vs/editor/standalone/browser/standaloneTreeSitterService.ts b/src/vs/editor/standalone/browser/standaloneTreeSitterService.ts new file mode 100644 index 00000000000..153a9b887a0 --- /dev/null +++ b/src/vs/editor/standalone/browser/standaloneTreeSitterService.ts @@ -0,0 +1,26 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// eslint-disable-next-line local/code-import-patterns +import type { Parser } from '@vscode/tree-sitter-wasm'; +import { Event } from 'vs/base/common/event'; +import { ITextModel } from 'vs/editor/common/model'; +import { ITreeSitterParseResult, ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'; + +/** + * The monaco build doesn't like the dynamic import of tree sitter in the real service. + * We use a dummy sertive here to make the build happy. + */ +export class StandaloneTreeSitterParserService implements ITreeSitterParserService { + readonly _serviceBrand: undefined; + onDidAddLanguage: Event<{ id: string; language: Parser.Language }> = Event.None; + + getOrInitLanguage(_languageId: string): Parser.Language | undefined { + return undefined; + } + getParseResult(textModel: ITextModel): ITreeSitterParseResult | undefined { + return undefined; + } +} diff --git a/src/vs/editor/test/browser/services/treeSitterParserService.test.ts b/src/vs/editor/test/browser/services/treeSitterParserService.test.ts index 65598ee1873..ca9dcaac4e9 100644 --- a/src/vs/editor/test/browser/services/treeSitterParserService.test.ts +++ b/src/vs/editor/test/browser/services/treeSitterParserService.test.ts @@ -123,10 +123,17 @@ suite('TreeSitterParserService', function () { const store = ensureNoDisposablesAreLeakedInTestSuite(); test('TextModelTreeSitter race condition: first language is slow to load', async function () { - class MockTreeSitterParser extends TreeSitterLanguages { - public override async getLanguage(languageId: string): Promise { + class MockTreeSitterLanguages extends TreeSitterLanguages { + private async _fetchJavascript(): Promise { + await timeout(200); + const language = new MockLanguage(); + language.languageId = 'javascript'; + this._onDidAddLanguage.fire({ id: 'javascript', language }); + } + public override getOrInitLanguage(languageId: string): Parser.Language | undefined { if (languageId === 'javascript') { - await timeout(200); + this._fetchJavascript(); + return undefined; } const language = new MockLanguage(); language.languageId = languageId; @@ -134,11 +141,11 @@ suite('TreeSitterParserService', function () { } } - const treeSitterParser: TreeSitterLanguages = store.add(new MockTreeSitterParser(treeSitterImporter, {} as any, { isBuilt: false } as any)); + const treeSitterParser: TreeSitterLanguages = store.add(new MockTreeSitterLanguages(treeSitterImporter, {} as any, { isBuilt: false } as any, new Map())); const textModel = store.add(createTextModel('console.log("Hello, world!");', 'javascript')); const textModelTreeSitter = store.add(new TextModelTreeSitter(textModel, treeSitterParser, treeSitterImporter, logService, telemetryService)); textModel.setLanguage('typescript'); await timeout(300); - assert.strictEqual((textModelTreeSitter.tree?.language as MockLanguage).languageId, 'typescript'); + assert.strictEqual((textModelTreeSitter.parseResult?.language as MockLanguage).languageId, 'typescript'); }); }); diff --git a/src/vs/editor/test/browser/testCodeEditor.ts b/src/vs/editor/test/browser/testCodeEditor.ts index 5fcd0c1bd4c..5a79e483795 100644 --- a/src/vs/editor/test/browser/testCodeEditor.ts +++ b/src/vs/editor/test/browser/testCodeEditor.ts @@ -23,9 +23,11 @@ import { LanguageService } from 'vs/editor/common/services/languageService'; import { IModelService } from 'vs/editor/common/services/model'; import { ModelService } from 'vs/editor/common/services/modelService'; import { ITextResourcePropertiesService } from 'vs/editor/common/services/textResourceConfiguration'; +import { ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'; import { ViewModel } from 'vs/editor/common/viewModel/viewModelImpl'; import { TestConfiguration } from 'vs/editor/test/browser/config/testConfiguration'; import { TestCodeEditorService, TestCommandService } from 'vs/editor/test/browser/editorTestServices'; +import { TestTreeSitterParserService } from 'vs/editor/test/common/services/testTreeSitterService'; import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService'; import { TestEditorWorkerService } from 'vs/editor/test/common/services/testEditorWorkerService'; import { TestTextResourcePropertiesService } from 'vs/editor/test/common/services/testTextResourcePropertiesService'; @@ -218,6 +220,7 @@ export function createCodeEditorServices(disposables: DisposableStore, services: }); define(ILanguageFeatureDebounceService, LanguageFeatureDebounceService); define(ILanguageFeaturesService, LanguageFeaturesService); + define(ITreeSitterParserService, TestTreeSitterParserService); const instantiationService = disposables.add(new TestInstantiationService(services, true)); disposables.add(toDisposable(() => { diff --git a/src/vs/editor/test/browser/services/testTreeSitterService.ts b/src/vs/editor/test/common/services/testTreeSitterService.ts similarity index 54% rename from src/vs/editor/test/browser/services/testTreeSitterService.ts rename to src/vs/editor/test/common/services/testTreeSitterService.ts index f449962d6cd..e5adcca9f83 100644 --- a/src/vs/editor/test/browser/services/testTreeSitterService.ts +++ b/src/vs/editor/test/common/services/testTreeSitterService.ts @@ -3,25 +3,22 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { AppResourcePath } from 'vs/base/common/network'; import type { Parser } from '@vscode/tree-sitter-wasm'; +import { Event } from 'vs/base/common/event'; import { ITextModel } from 'vs/editor/common/model'; -import { ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'; +import { ITreeSitterParserService, ITreeSitterParseResult } from 'vs/editor/common/services/treeSitterParserService'; export class TestTreeSitterParserService implements ITreeSitterParserService { - getLanguage(model: ITextModel): Parser.Language | undefined { + onDidAddLanguage: Event<{ id: string; language: Parser.Language }> = Event.None; + _serviceBrand: undefined; + getOrInitLanguage(languageId: string): Parser.Language | undefined { throw new Error('Method not implemented.'); } - getLanguageLocation(languageId: string): AppResourcePath { + waitForLanguage(languageId: string): Promise { throw new Error('Method not implemented.'); } - readonly _serviceBrand: undefined; - - public initTreeSitter(): Promise { - return Promise.resolve(); + getParseResult(textModel: ITextModel): ITreeSitterParseResult | undefined { + throw new Error('Method not implemented.'); } - public getTree(_model: ITextModel): Parser.Tree | undefined { - return undefined; - } } diff --git a/src/vs/editor/test/common/testTextModel.ts b/src/vs/editor/test/common/testTextModel.ts index cdab452aa9c..029e58d9188 100644 --- a/src/vs/editor/test/common/testTextModel.ts +++ b/src/vs/editor/test/common/testTextModel.ts @@ -34,6 +34,8 @@ import { ILanguageFeaturesService } from 'vs/editor/common/services/languageFeat import { LanguageFeaturesService } from 'vs/editor/common/services/languageFeaturesService'; import { IEnvironmentService } from 'vs/platform/environment/common/environment'; import { mock } from 'vs/base/test/common/mock'; +import { ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'; +import { TestTreeSitterParserService } from 'vs/editor/test/common/services/testTreeSitterService'; class TestTextModel extends TextModel { public registerDisposable(disposable: IDisposable): void { @@ -105,5 +107,6 @@ export function createModelServices(disposables: DisposableStore, services: Serv [ILanguageFeatureDebounceService, LanguageFeatureDebounceService], [ILanguageFeaturesService, LanguageFeaturesService], [IModelService, ModelService], + [ITreeSitterParserService, TestTreeSitterParserService] ])); } diff --git a/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts b/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts index 8b2fa1d6632..535ccd778d8 100644 --- a/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts +++ b/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts @@ -16,7 +16,7 @@ import { Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; import { IEditorContribution } from 'vs/editor/common/editorCommon'; import { ITextModel } from 'vs/editor/common/model'; -import { SemanticTokensLegend, SemanticTokens } from 'vs/editor/common/languages'; +import { SemanticTokensLegend, SemanticTokens, TreeSitterTokenizationRegistry } from 'vs/editor/common/languages'; import { FontStyle, ColorId, StandardTokenType, TokenMetadata } from 'vs/editor/common/encodedTokenAttributes'; import { ILanguageService } from 'vs/editor/common/languages/language'; import { INotificationService } from 'vs/platform/notification/common/notification'; @@ -31,6 +31,9 @@ import { IConfigurationService } from 'vs/platform/configuration/common/configur import { SEMANTIC_HIGHLIGHTING_SETTING_ID, IEditorSemanticHighlightingOptions } from 'vs/editor/contrib/semanticTokens/common/semanticTokensConfig'; import { Schemas } from 'vs/base/common/network'; import { ILanguageFeaturesService } from 'vs/editor/common/services/languageFeatures'; +import { ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'; +// eslint-disable-next-line local/code-import-patterns +import type { Parser } from '@vscode/tree-sitter-wasm'; const $ = dom.$; @@ -44,6 +47,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib private _editor: ICodeEditor; private _textMateService: ITextMateTokenizationService; + private _treeSitterService: ITreeSitterParserService; private _themeService: IWorkbenchThemeService; private _languageService: ILanguageService; private _notificationService: INotificationService; @@ -54,6 +58,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib constructor( editor: ICodeEditor, @ITextMateTokenizationService textMateService: ITextMateTokenizationService, + @ITreeSitterParserService treeSitterService: ITreeSitterParserService, @ILanguageService languageService: ILanguageService, @IWorkbenchThemeService themeService: IWorkbenchThemeService, @INotificationService notificationService: INotificationService, @@ -63,6 +68,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib super(); this._editor = editor; this._textMateService = textMateService; + this._treeSitterService = treeSitterService; this._themeService = themeService; this._languageService = languageService; this._notificationService = notificationService; @@ -91,7 +97,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib // disable in notebooks return; } - this._widget = new InspectEditorTokensWidget(this._editor, this._textMateService, this._languageService, this._themeService, this._notificationService, this._configurationService, this._languageFeaturesService); + this._widget = new InspectEditorTokensWidget(this._editor, this._textMateService, this._treeSitterService, this._languageService, this._themeService, this._notificationService, this._configurationService, this._languageFeaturesService); } public stop(): void { @@ -188,6 +194,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { private readonly _languageService: ILanguageService; private readonly _themeService: IWorkbenchThemeService; private readonly _textMateService: ITextMateTokenizationService; + private readonly _treeSitterService: ITreeSitterParserService; private readonly _notificationService: INotificationService; private readonly _configurationService: IConfigurationService; private readonly _languageFeaturesService: ILanguageFeaturesService; @@ -198,6 +205,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { constructor( editor: IActiveCodeEditor, textMateService: ITextMateTokenizationService, + treeSitterService: ITreeSitterParserService, languageService: ILanguageService, themeService: IWorkbenchThemeService, notificationService: INotificationService, @@ -210,6 +218,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { this._languageService = languageService; this._themeService = themeService; this._textMateService = textMateService; + this._treeSitterService = treeSitterService; this._notificationService = notificationService; this._configurationService = configurationService; this._languageFeaturesService = languageFeaturesService; @@ -238,6 +247,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { private _beginCompute(position: Position): void { const grammar = this._textMateService.createTokenizer(this._model.getLanguageId()); const semanticTokens = this._computeSemanticTokens(position); + const tree = this._treeSitterService.getParseResult(this._model); dom.clearNode(this._domNode); this._domNode.appendChild(document.createTextNode(nls.localize('inspectTMScopesWidget.loading', "Loading..."))); @@ -246,7 +256,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { if (this._isDisposed) { return; } - this._compute(grammar, semanticTokens, position); + this._compute(grammar, semanticTokens, tree?.tree, position); this._domNode.style.maxWidth = `${Math.max(this._editor.getLayoutInfo().width * 0.66, 500)}px`; this._editor.layoutContentWidget(this); }, (err) => { @@ -267,10 +277,11 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { return this._themeService.getColorTheme().semanticHighlighting; } - private _compute(grammar: IGrammar | null, semanticTokens: SemanticTokensResult | null, position: Position) { + private _compute(grammar: IGrammar | null, semanticTokens: SemanticTokensResult | null, tree: Parser.Tree | undefined, position: Position) { const textMateTokenInfo = grammar && this._getTokensAtPosition(grammar, position); const semanticTokenInfo = semanticTokens && this._getSemanticTokenAtPosition(semanticTokens, position); - if (!textMateTokenInfo && !semanticTokenInfo) { + const treeSitterTokenInfo = tree && this._getTreeSitterTokenAtPosition(tree, position); + if (!textMateTokenInfo && !semanticTokenInfo && !treeSitterTokenInfo) { dom.reset(this._domNode, 'No grammar or semantic tokens available.'); return; } @@ -389,6 +400,40 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { )); } } + + if (treeSitterTokenInfo) { + dom.append(this._domNode, $('hr.tiw-metadata-separator')); + const table = dom.append(this._domNode, $('table.tiw-metadata-table')); + const tbody = dom.append(table, $('tbody')); + + dom.append(tbody, $('tr', undefined, + $('td.tiw-metadata-key', undefined, 'tree-sitter token' as string), + $('td.tiw-metadata-value', undefined, `${treeSitterTokenInfo.text}`) + )); + const scopes = new Array(); + let node = treeSitterTokenInfo; + while (node.parent) { + scopes.push(node.type); + node = node.parent; + if (node) { + scopes.push($('br')); + } + } + + dom.append(tbody, $('tr', undefined, + $('td.tiw-metadata-key', undefined, 'tree-sitter scopes' as string), + $('td.tiw-metadata-value.tiw-metadata-scopes', undefined, ...scopes), + )); + + const tokenizationSupport = TreeSitterTokenizationRegistry.get(this._model.getLanguageId()); + const captures = tokenizationSupport?.captureAtPosition(position.lineNumber, position.column, this._model); + if (captures && captures.length > 0) { + dom.append(tbody, $('tr', undefined, + $('td.tiw-metadata-key', undefined, 'foreground'), + $('td.tiw-metadata-value', undefined, captures[0].name), + )); + } + } } private _formatMetadata(semantic?: IDecodedMetadata, tm?: IDecodedMetadata): Array { @@ -603,6 +648,28 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { return null; } + private _walkTreeforPosition(cursor: Parser.TreeCursor, pos: Position): Parser.SyntaxNode | null { + const offset = this._model.getOffsetAt(pos); + cursor.gotoFirstChild(); + let goChild: boolean = false; + let lastGoodNode: Parser.SyntaxNode | null = null; + do { + if (cursor.currentNode.startIndex <= offset && offset < cursor.currentNode.endIndex) { + goChild = true; + lastGoodNode = cursor.currentNode; + } else { + goChild = false; + } + } while (goChild ? cursor.gotoFirstChild() : cursor.gotoNextSibling()); + return lastGoodNode; + } + + private _getTreeSitterTokenAtPosition(tree: Parser.Tree, pos: Position): Parser.SyntaxNode | null { + const cursor = tree.walk(); + + return this._walkTreeforPosition(cursor, pos); + } + private _renderTokenStyleDefinition(definition: TokenStyleDefinition | undefined, property: keyof TokenStyleData): Array { const elements = new Array(); if (definition === undefined) { diff --git a/src/vs/workbench/services/themes/common/colorThemeData.ts b/src/vs/workbench/services/themes/common/colorThemeData.ts index e74a4ab1e05..2019848fea7 100644 --- a/src/vs/workbench/services/themes/common/colorThemeData.ts +++ b/src/vs/workbench/services/themes/common/colorThemeData.ts @@ -245,7 +245,7 @@ export class ColorThemeData implements IWorkbenchColorTheme { return undefined; } - private getTokenColorIndex(): TokenColorIndex { + public getTokenColorIndex(): TokenColorIndex { // collect all colors that tokens can have if (!this.tokenColorIndex) { const index = new TokenColorIndex(); diff --git a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.ts b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.ts index 61ed97523f9..ee04816c6ac 100644 --- a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.ts +++ b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.ts @@ -7,6 +7,7 @@ import { registerSingleton, InstantiationType } from 'vs/platform/instantiation/ import { IWorkbenchContribution, WorkbenchPhase, registerWorkbenchContribution2 } from 'vs/workbench/common/contributions'; import { TreeSitterTextModelService } from 'vs/editor/browser/services/treeSitter/treeSitterParserService'; import { ITreeSitterParserService } from 'vs/editor/common/services/treeSitterParserService'; +import { ITreeSitterTokenizationFeature } from 'vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature'; /** * Makes sure the ITreeSitterTokenizationService is instantiated @@ -16,7 +17,8 @@ class TreeSitterTokenizationInstantiator implements IWorkbenchContribution { static readonly ID = 'workbench.contrib.treeSitterTokenizationInstantiator'; constructor( - @ITreeSitterParserService _treeSitterTokenizationService: ITreeSitterParserService + @ITreeSitterParserService _treeSitterTokenizationService: ITreeSitterParserService, + @ITreeSitterTokenizationFeature _treeSitterTokenizationFeature: ITreeSitterTokenizationFeature ) { } } diff --git a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts new file mode 100644 index 00000000000..efc71a942c8 --- /dev/null +++ b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts @@ -0,0 +1,241 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// eslint-disable-next-line local/code-import-patterns +import type { Parser } from '@vscode/tree-sitter-wasm'; +import { Emitter, Event } from 'vs/base/common/event'; +import { Disposable, DisposableMap, DisposableStore, IDisposable } from 'vs/base/common/lifecycle'; +import { AppResourcePath, FileAccess } from 'vs/base/common/network'; +import { FontStyle, MetadataConsts } from 'vs/editor/common/encodedTokenAttributes'; +import { ITreeSitterTokenizationSupport, LazyTokenizationSupport, TreeSitterTokenizationRegistry } from 'vs/editor/common/languages'; +import { ITextModel } from 'vs/editor/common/model'; +import { EDITOR_EXPERIMENTAL_PREFER_TREESITTER, ITreeSitterParserService, ITreeSitterParseResult } from 'vs/editor/common/services/treeSitterParserService'; +import { IModelTokensChangedEvent } from 'vs/editor/common/textModelEvents'; +import { ColumnRange } from 'vs/editor/contrib/inlineCompletions/browser/utils'; +import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; +import { IFileService } from 'vs/platform/files/common/files'; +import { InstantiationType, registerSingleton } from 'vs/platform/instantiation/common/extensions'; +import { createDecorator, IInstantiationService } from 'vs/platform/instantiation/common/instantiation'; +import { IThemeService } from 'vs/platform/theme/common/themeService'; +import { TokenStyle } from 'vs/platform/theme/common/tokenClassificationRegistry'; +import { ColorThemeData } from 'vs/workbench/services/themes/common/colorThemeData'; + +const ALLOWED_SUPPORT = ['typescript']; +type TreeSitterQueries = string; + +export const ITreeSitterTokenizationFeature = createDecorator('treeSitterTokenizationFeature'); + +export interface ITreeSitterTokenizationFeature { + _serviceBrand: undefined; +} + +class TreeSitterTokenizationFeature extends Disposable implements ITreeSitterTokenizationFeature { + public _serviceBrand: undefined; + private readonly _tokenizersRegistrations: DisposableMap = new DisposableMap(); + + constructor( + @IConfigurationService private readonly _configurationService: IConfigurationService, + @IInstantiationService private readonly _instantiationService: IInstantiationService, + @IFileService private readonly _fileService: IFileService + ) { + super(); + + this._handleGrammarsExtPoint(); + this._register(this._configurationService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(EDITOR_EXPERIMENTAL_PREFER_TREESITTER)) { + this._handleGrammarsExtPoint(); + } + })); + } + + private _getSetting(): string[] { + return this._configurationService.getValue(EDITOR_EXPERIMENTAL_PREFER_TREESITTER) || []; + } + + private _handleGrammarsExtPoint(): void { + const setting = this._getSetting(); + + // Eventually, this should actually use an extension point to add tree sitter grammars, but for now they are hard coded in core + for (const languageId of setting) { + if (ALLOWED_SUPPORT.includes(languageId) && !this._tokenizersRegistrations.has(languageId)) { + const lazyTokenizationSupport = new LazyTokenizationSupport(() => this._createTokenizationSupport(languageId)); + const disposableStore = new DisposableStore(); + disposableStore.add(lazyTokenizationSupport); + disposableStore.add(TreeSitterTokenizationRegistry.registerFactory(languageId, lazyTokenizationSupport)); + this._tokenizersRegistrations.set(languageId, disposableStore); + TreeSitterTokenizationRegistry.getOrCreate(languageId); + } + } + } + + private async _fetchQueries(newLanguage: string): Promise { + const languageLocation: AppResourcePath = `vs/editor/common/languages/highlights/${newLanguage}.scm`; + const query = await this._fileService.readFile(FileAccess.asFileUri(languageLocation)); + return query.value.toString(); + } + + private async _createTokenizationSupport(languageId: string): Promise { + const queries = await this._fetchQueries(languageId); + return this._instantiationService.createInstance(TreeSitterTokenizationSupport, queries, languageId); + } +} + +class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTokenizationSupport { + private _query: Parser.Query | undefined; + private readonly _onDidChangeTokens: Emitter = new Emitter(); + public readonly onDidChangeTokens: Event = this._onDidChangeTokens.event; + private _colorThemeData!: ColorThemeData; + private _languageAddedListener: IDisposable | undefined; + + constructor( + private readonly _queries: TreeSitterQueries, + private readonly _languageId: string, + @ITreeSitterParserService private readonly _treeSitterService: ITreeSitterParserService, + @IThemeService private readonly _themeService: IThemeService, + ) { + super(); + this._register(Event.runAndSubscribe(this._themeService.onDidColorThemeChange, () => this.reset())); + } + + private _getTree(textModel: ITextModel): ITreeSitterParseResult | undefined { + return this._treeSitterService.getParseResult(textModel); + } + + private _ensureQuery() { + if (!this._query) { + const language = this._treeSitterService.getOrInitLanguage(this._languageId); + if (!language) { + if (!this._languageAddedListener) { + this._languageAddedListener = this._register(Event.onceIf(this._treeSitterService.onDidAddLanguage, e => e.id === this._languageId)((e) => { + this._query = e.language.query(this._queries); + })); + } + return; + } + this._query = language.query(this._queries); + } + return this._query; + } + + private reset() { + this._colorThemeData = this._themeService.getColorTheme() as ColorThemeData; + } + + captureAtPosition(lineNumber: number, column: number, textModel: ITextModel): any { + const captures = this._captureAtRange(lineNumber, new ColumnRange(column, column), textModel); + return captures; + } + + private _captureAtRange(lineNumber: number, columnRange: ColumnRange, textModel: ITextModel): Parser.QueryCapture[] { + const tree = this._getTree(textModel); + const query = this._ensureQuery(); + if (!tree?.tree || !query) { + return []; + } + // Tree sitter row is 0 based, column is 0 based + return query.captures(tree.tree.rootNode, { startPosition: { row: lineNumber - 1, column: columnRange.startColumn - 1 }, endPosition: { row: lineNumber - 1, column: columnRange.endColumnExclusive } }); + } + + /** + * Gets the tokens for a given line. + * Each token takes 2 elements in the array. The first element is the offset of the end of the token *in the line, not in the document*, and the second element is the metadata. + * + * @param lineNumber + * @returns + */ + public tokenizeEncoded(lineNumber: number, textModel: ITextModel): Uint32Array | undefined { + const lineLength = textModel.getLineMaxColumn(lineNumber); + const captures = this._captureAtRange(lineNumber, new ColumnRange(1, lineLength), textModel); + + if (captures.length === 0) { + return undefined; + } + + let tokens: Uint32Array = new Uint32Array(captures.length * 2); + let tokenIndex = 0; + const lineStartOffset = textModel.getOffsetAt({ lineNumber: lineNumber, column: 1 }); + + for (let captureIndex = 0; captureIndex < captures.length; captureIndex++) { + const capture = captures[captureIndex]; + const metadata = this.findMetadata(capture.name); + const tokenEndIndex = capture.node.endIndex < lineStartOffset + lineLength ? capture.node.endIndex : lineStartOffset + lineLength; + const tokenStartIndex = capture.node.startIndex < lineStartOffset ? lineStartOffset : capture.node.startIndex; + + const lineRelativeOffset = tokenEndIndex - lineStartOffset; + // Not every character will get captured, so we need to make sure that our current capture doesn't bleed toward the start of the line and cover characters that it doesn't apply to. + // We do this by creating a new token in the array if the previous token ends before the current token starts. + let previousTokenEnd: number; + const currentTokenLength = tokenEndIndex - tokenStartIndex; + if (captureIndex > 0) { + previousTokenEnd = tokens[(tokenIndex - 1) * 2]; + } else { + previousTokenEnd = tokenStartIndex - lineStartOffset - 1; + } + const intermediateTokenOffset = lineRelativeOffset - currentTokenLength; + if (previousTokenEnd < intermediateTokenOffset) { + tokens[tokenIndex * 2] = intermediateTokenOffset; + tokens[tokenIndex * 2 + 1] = 0; + tokenIndex++; + const newTokens = new Uint32Array(tokens.length + 2); + newTokens.set(tokens); + tokens = newTokens; + } + + tokens[tokenIndex * 2] = lineRelativeOffset; + tokens[tokenIndex * 2 + 1] = metadata; + tokenIndex++; + } + + if (captures[captures.length - 1].node.endPosition.column + 1 < lineLength) { + const newTokens = new Uint32Array(tokens.length + 2); + newTokens.set(tokens); + tokens = newTokens; + tokens[tokenIndex * 2] = lineLength; + tokens[tokenIndex * 2 + 1] = 0; + } + return tokens; + } + + private findMetadata(captureName: string): number { + const tokenStyle: TokenStyle | undefined = this._colorThemeData.resolveScopes([[captureName]]); + if (!tokenStyle) { + return 0; + } + + let metadata = 0; + if (typeof tokenStyle.italic !== 'undefined') { + const italicBit = (tokenStyle.italic ? FontStyle.Italic : 0); + metadata |= italicBit | MetadataConsts.ITALIC_MASK; + } + if (typeof tokenStyle.bold !== 'undefined') { + const boldBit = (tokenStyle.bold ? FontStyle.Bold : 0); + metadata |= boldBit | MetadataConsts.BOLD_MASK; + } + if (typeof tokenStyle.underline !== 'undefined') { + const underlineBit = (tokenStyle.underline ? FontStyle.Underline : 0); + metadata |= underlineBit | MetadataConsts.UNDERLINE_MASK; + } + if (typeof tokenStyle.strikethrough !== 'undefined') { + const strikethroughBit = (tokenStyle.strikethrough ? FontStyle.Strikethrough : 0); + metadata |= strikethroughBit | MetadataConsts.STRIKETHROUGH_MASK; + } + if (tokenStyle.foreground) { + const tokenStyleForeground = this._colorThemeData.getTokenColorIndex().get(tokenStyle?.foreground); + const foregroundBits = tokenStyleForeground << MetadataConsts.FOREGROUND_OFFSET; + metadata |= foregroundBits; + } + + return metadata; + } + + override dispose() { + super.dispose(); + this._query?.delete(); + this._query = undefined; + } +} + +registerSingleton(ITreeSitterTokenizationFeature, TreeSitterTokenizationFeature, InstantiationType.Eager); + diff --git a/yarn.lock b/yarn.lock index 645261bb0f7..7f503486760 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1673,10 +1673,10 @@ tar-fs "^3.0.6" vscode-uri "^3.0.8" -"@vscode/tree-sitter-wasm@^0.0.1": - version "0.0.1" - resolved "https://registry.yarnpkg.com/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.1.tgz#ffb2e295a416698f4c77cbffeca3b28567d6754b" - integrity sha512-m0GKnQ3BxWnVd+20KLGwr1+Qvt/RiiaJmKAqHNU35pNydDtduUzyBm7ETz/T0vOVKoeIAaiYsJOA1aKWs7Y1tA== +"@vscode/tree-sitter-wasm@^0.0.2": + version "0.0.2" + resolved "https://registry.yarnpkg.com/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.2.tgz#da21541d343be69bb263e9380d165e3b164ec1f0" + integrity sha512-N57MR/kt4jR0H/TXeDsVYeJmvvUiK7avow0fjy+/EeKcyNBJcM2BFhj4XOAaaMbhGsOcIeSvJFouRWctXI7sKw== "@vscode/v8-heap-parser@^0.1.0": version "0.1.0"