diff --git a/build/gulpfile.editor.js b/build/gulpfile.editor.js index 739c37fc677..9f7ea57465b 100644 --- a/build/gulpfile.editor.js +++ b/build/gulpfile.editor.js @@ -80,7 +80,7 @@ const extractEditorSrcTask = task.define('extract-editor-src', () => { importIgnorePattern: /\.css$/, destRoot: path.join(root, 'out-editor-src'), redirects: { - '@vscode/tree-sitter-wasm': '../node_modules/@vscode/tree-sitter-wasm/wasm/tree-sitter-web', + '@vscode/tree-sitter-wasm': '../node_modules/@vscode/tree-sitter-wasm/wasm/web-tree-sitter', } }); }); diff --git a/package-lock.json b/package-lock.json index 220b5d9879b..d20dc2d0950 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "@vscode/spdlog": "^0.15.0", "@vscode/sqlite3": "5.1.8-vscode", "@vscode/sudo-prompt": "9.3.1", - "@vscode/tree-sitter-wasm": "^0.0.5", + "@vscode/tree-sitter-wasm": "^0.1.1", "@vscode/vscode-languagedetection": "1.0.21", "@vscode/windows-mutex": "^0.5.0", "@vscode/windows-process-tree": "^0.6.0", @@ -3157,9 +3157,9 @@ } }, "node_modules/@vscode/tree-sitter-wasm": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.5.tgz", - "integrity": "sha512-qA+BkB2UgkfXMQVGsqPeG3vR3pXv0inP6WQ/dq6BALy7dIX9KQvGXvDCiqehdFvZZO4tDFt4qb5DdSsvwR4Y9Q==", + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.1.1.tgz", + "integrity": "sha512-2KHGbX2krHP/LyfpDB6QnSAqyqIL7N2Md7KTMuoqq8+GhUzrgXwIClhpm7lqL/isNTYVzsEubR7YI6f2YfAqqQ==", "license": "MIT" }, "node_modules/@vscode/v8-heap-parser": { diff --git a/package.json b/package.json index 3f97b4a3d11..2cd4edb0bbc 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "@vscode/spdlog": "^0.15.0", "@vscode/sqlite3": "5.1.8-vscode", "@vscode/sudo-prompt": "9.3.1", - "@vscode/tree-sitter-wasm": "^0.0.5", + "@vscode/tree-sitter-wasm": "^0.1.1", "@vscode/vscode-languagedetection": "1.0.21", "@vscode/windows-mutex": "^0.5.0", "@vscode/windows-process-tree": "^0.6.0", diff --git a/remote/package-lock.json b/remote/package-lock.json index 787c3c9ba96..a53da887592 100644 --- a/remote/package-lock.json +++ b/remote/package-lock.json @@ -16,7 +16,7 @@ "@vscode/proxy-agent": "^0.31.0", "@vscode/ripgrep": "^1.15.10", "@vscode/spdlog": "^0.15.0", - "@vscode/tree-sitter-wasm": "^0.0.5", + "@vscode/tree-sitter-wasm": "^0.1.1", "@vscode/vscode-languagedetection": "1.0.21", "@vscode/windows-process-tree": "^0.6.0", "@vscode/windows-registry": "^1.1.0", @@ -470,9 +470,9 @@ } }, "node_modules/@vscode/tree-sitter-wasm": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.5.tgz", - "integrity": "sha512-qA+BkB2UgkfXMQVGsqPeG3vR3pXv0inP6WQ/dq6BALy7dIX9KQvGXvDCiqehdFvZZO4tDFt4qb5DdSsvwR4Y9Q==", + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.1.1.tgz", + "integrity": "sha512-2KHGbX2krHP/LyfpDB6QnSAqyqIL7N2Md7KTMuoqq8+GhUzrgXwIClhpm7lqL/isNTYVzsEubR7YI6f2YfAqqQ==", "license": "MIT" }, "node_modules/@vscode/vscode-languagedetection": { diff --git a/remote/package.json b/remote/package.json index 5cc1ff19468..8dc72f471db 100644 --- a/remote/package.json +++ b/remote/package.json @@ -11,7 +11,7 @@ "@vscode/proxy-agent": "^0.31.0", "@vscode/ripgrep": "^1.15.10", "@vscode/spdlog": "^0.15.0", - "@vscode/tree-sitter-wasm": "^0.0.5", + "@vscode/tree-sitter-wasm": "^0.1.1", "@vscode/vscode-languagedetection": "1.0.21", "@vscode/windows-process-tree": "^0.6.0", "@vscode/windows-registry": "^1.1.0", diff --git a/remote/web/package-lock.json b/remote/web/package-lock.json index 45f7724805d..547912df017 100644 --- a/remote/web/package-lock.json +++ b/remote/web/package-lock.json @@ -11,7 +11,7 @@ "@microsoft/1ds-core-js": "^3.2.13", "@microsoft/1ds-post-js": "^3.2.13", "@vscode/iconv-lite-umd": "0.7.0", - "@vscode/tree-sitter-wasm": "^0.0.5", + "@vscode/tree-sitter-wasm": "^0.1.1", "@vscode/vscode-languagedetection": "1.0.21", "@xterm/addon-clipboard": "^0.2.0-beta.79", "@xterm/addon-image": "^0.9.0-beta.96", @@ -76,9 +76,9 @@ "integrity": "sha512-bRRFxLfg5dtAyl5XyiVWz/ZBPahpOpPrNYnnHpOpUZvam4tKH35wdhP4Kj6PbM0+KdliOsPzbGWpkxcdpNB/sg==" }, "node_modules/@vscode/tree-sitter-wasm": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.0.5.tgz", - "integrity": "sha512-qA+BkB2UgkfXMQVGsqPeG3vR3pXv0inP6WQ/dq6BALy7dIX9KQvGXvDCiqehdFvZZO4tDFt4qb5DdSsvwR4Y9Q==", + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/@vscode/tree-sitter-wasm/-/tree-sitter-wasm-0.1.1.tgz", + "integrity": "sha512-2KHGbX2krHP/LyfpDB6QnSAqyqIL7N2Md7KTMuoqq8+GhUzrgXwIClhpm7lqL/isNTYVzsEubR7YI6f2YfAqqQ==", "license": "MIT" }, "node_modules/@vscode/vscode-languagedetection": { diff --git a/remote/web/package.json b/remote/web/package.json index f1f5ce1ecb9..47350e2deaf 100644 --- a/remote/web/package.json +++ b/remote/web/package.json @@ -6,7 +6,7 @@ "@microsoft/1ds-core-js": "^3.2.13", "@microsoft/1ds-post-js": "^3.2.13", "@vscode/iconv-lite-umd": "0.7.0", - "@vscode/tree-sitter-wasm": "^0.0.5", + "@vscode/tree-sitter-wasm": "^0.1.1", "@vscode/vscode-languagedetection": "1.0.21", "@xterm/addon-clipboard": "^0.2.0-beta.79", "@xterm/addon-image": "^0.9.0-beta.96", diff --git a/src/vs/editor/common/languages.ts b/src/vs/editor/common/languages.ts index 76daa762217..a8f1af273f4 100644 --- a/src/vs/editor/common/languages.ts +++ b/src/vs/editor/common/languages.ts @@ -27,7 +27,7 @@ import { localize } from '../../nls.js'; import { ExtensionIdentifier } from '../../platform/extensions/common/extensions.js'; import { IMarkerData } from '../../platform/markers/common/markers.js'; import { IModelTokensChangedEvent } from './textModelEvents.js'; -import type { Parser } from '@vscode/tree-sitter-wasm'; +import type * as Parser from '@vscode/tree-sitter-wasm'; import { ITextModel } from './model.js'; import { TokenUpdate } from './model/tokenStore.js'; diff --git a/src/vs/editor/common/services/treeSitter/treeSitterParserService.ts b/src/vs/editor/common/services/treeSitter/treeSitterParserService.ts index 4ca0e3396de..fb8c0c912c4 100644 --- a/src/vs/editor/common/services/treeSitter/treeSitterParserService.ts +++ b/src/vs/editor/common/services/treeSitter/treeSitterParserService.ts @@ -3,9 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import type { Parser } from '@vscode/tree-sitter-wasm'; +import type * as Parser from '@vscode/tree-sitter-wasm'; import { AppResourcePath, FileAccess, nodeModulesAsarUnpackedPath, nodeModulesPath } from '../../../../base/common/network.js'; -import { EDITOR_EXPERIMENTAL_PREFER_TREESITTER, ITreeSitterParserService, ITreeSitterParseResult, ITextModelTreeSitter, RangeChange, TreeUpdateEvent, TreeParseUpdateEvent } from '../treeSitterParserService.js'; +import { EDITOR_EXPERIMENTAL_PREFER_TREESITTER, ITreeSitterParserService, ITreeSitterParseResult, ITextModelTreeSitter, RangeChange, TreeUpdateEvent, TreeParseUpdateEvent, ITreeSitterImporter } from '../treeSitterParserService.js'; import { IModelService } from '../model.js'; import { Disposable, DisposableMap, DisposableStore, dispose, IDisposable } from '../../../../base/common/lifecycle.js'; import { ITextModel } from '../../model.js'; @@ -15,7 +15,7 @@ import { ITelemetryService } from '../../../../platform/telemetry/common/telemet import { ILogService } from '../../../../platform/log/common/log.js'; import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js'; import { setTimeout0 } from '../../../../base/common/platform.js'; -import { canASAR, importAMDNodeModule } from '../../../../amdX.js'; +import { canASAR } from '../../../../amdX.js'; import { Emitter, Event } from '../../../../base/common/event.js'; import { CancellationToken, cancelOnDispose } from '../../../../base/common/cancellation.js'; import { IEnvironmentService } from '../../../../platform/environment/common/environment.js'; @@ -44,7 +44,7 @@ export class TextModelTreeSitter extends Disposable implements ITextModelTreeSit constructor(readonly model: ITextModel, private readonly _treeSitterLanguages: TreeSitterLanguages, - private readonly _treeSitterImporter: TreeSitterImporter, + private readonly _treeSitterImporter: ITreeSitterImporter, private readonly _logService: ILogService, private readonly _telemetryService: ITelemetryService, parseImmediately: boolean = true @@ -155,11 +155,10 @@ export class TreeSitterParseResult implements IDisposable, ITreeSitterParseResul return this._versionId; } private _isDisposed: boolean = false; - constructor(public readonly parser: Parser, + constructor(public readonly parser: Parser.Parser, public /** exposed for tests **/ readonly language: Parser.Language, private readonly _logService: ILogService, private readonly _telemetryService: ITelemetryService) { - this.parser.setTimeoutMicros(50 * 1000); // 50 ms this.parser.setLanguage(language); } dispose(): void { @@ -228,7 +227,7 @@ export class TreeSitterParseResult implements IDisposable, ITreeSitterParseResul return false; }; - const getClosestPreviousNodes = (): { old: Parser.SyntaxNode; new: Parser.SyntaxNode } | undefined => { + const getClosestPreviousNodes = (): { old: Parser.Node; new: Parser.Node } | undefined => { // Go up parents until the end of the parent is before the start of the current. const newFindPrev = newTree.walk(); newFindPrev.resetTo(newCursor); @@ -267,10 +266,10 @@ export class TreeSitterParseResult implements IDisposable, ITreeSitterParseResul const newChildren = newCursor.currentNode.children; const indexChangedChildren: number[] = []; const changedChildren = newChildren.filter((c, index) => { - if (c.hasChanges) { + if (c?.hasChanges) { indexChangedChildren.push(index); } - return c.hasChanges; + return c?.hasChanges; }); if (changedChildren.length >= 1) { next = gotoNthChild(indexChangedChildren[0]); @@ -412,12 +411,13 @@ export class TreeSitterParseResult implements IDisposable, ITreeSitterParseResul let time: number = 0; let passes: number = 0; const inProgressVersion = this._editVersion; - let newTree: Parser.Tree | undefined; + let newTree: Parser.Tree | null | undefined; + this._lastYieldTime = performance.now(); do { const timer = performance.now(); try { - newTree = this.parser.parse((index: number, position?: Parser.Point) => this._parseCallback(model, index), this._tree); + newTree = this.parser.parse((index: number, position?: Parser.Point) => this._parseCallback(model, index), this._tree, { progressCallback: this._parseProgressCallback.bind(this) }); } catch (e) { // parsing can fail when the timeout is reached, will resume upon next loop } finally { @@ -433,13 +433,23 @@ export class TreeSitterParseResult implements IDisposable, ITreeSitterParseResul return (newTree && (inProgressVersion === model.getVersionId())) ? newTree : undefined; } - private _parseCallback(textModel: ITextModel, index: number): string | null { + private _lastYieldTime: number = 0; + private _parseProgressCallback(state: Parser.ParseState) { + const now = performance.now(); + if (now - this._lastYieldTime > 50) { + this._lastYieldTime = now; + return true; + } + return false; + } + + private _parseCallback(textModel: ITextModel, index: number): string | undefined { try { return textModel.getTextBuffer().getNearestChunk(index); } catch (e) { this._logService.debug('Error getting chunk for tree-sitter parsing', e); } - return null; + return undefined; } private sendParseTimeTelemetry(parseType: TelemetryParseType, languageId: string, time: number, passes: number): void { @@ -467,7 +477,7 @@ export class TreeSitterLanguages extends Disposable { */ public readonly onDidAddLanguage: Event<{ id: string; language: Parser.Language }> = this._onDidAddLanguage.event; - constructor(private readonly _treeSitterImporter: TreeSitterImporter, + constructor(private readonly _treeSitterImporter: ITreeSitterImporter, private readonly _fileService: IFileService, private readonly _environmentService: IEnvironmentService, private readonly _registeredLanguages: Map, @@ -514,8 +524,8 @@ export class TreeSitterLanguages extends Disposable { } const wasmPath: AppResourcePath = `${languageLocation}/${grammarName}.wasm`; const languageFile = await (this._fileService.readFile(FileAccess.asFileUri(wasmPath))); - const Parser = await this._treeSitterImporter.getParserClass(); - return Parser.Language.load(languageFile.value.buffer); + const Language = await this._treeSitterImporter.getLanguageClass(); + return Language.load(languageFile.value.buffer); } private _getLanguageLocation(languageId: string): AppResourcePath | undefined { @@ -527,24 +537,6 @@ export class TreeSitterLanguages extends Disposable { } } -export class TreeSitterImporter { - private _treeSitterImport: typeof import('@vscode/tree-sitter-wasm') | undefined; - private async _getTreeSitterImport() { - if (!this._treeSitterImport) { - this._treeSitterImport = await importAMDNodeModule('@vscode/tree-sitter-wasm', 'wasm/tree-sitter.js'); - } - return this._treeSitterImport; - } - - private _parserClass: typeof Parser | undefined; - public async getParserClass() { - if (!this._parserClass) { - this._parserClass = (await this._getTreeSitterImport()).Parser; - } - return this._parserClass; - } -} - interface TextModelTreeSitterItem { dispose(): void; textModelTreeSitter: TextModelTreeSitter; @@ -556,7 +548,6 @@ export class TreeSitterTextModelService extends Disposable implements ITreeSitte private _init!: Promise; private _textModelTreeSitters: DisposableMap = this._register(new DisposableMap()); private readonly _registeredLanguages: Map = new Map(); - private readonly _treeSitterImporter: TreeSitterImporter = new TreeSitterImporter(); private readonly _treeSitterLanguages: TreeSitterLanguages; public readonly onDidAddLanguage: Event<{ id: string; language: Parser.Language }>; @@ -570,7 +561,8 @@ export class TreeSitterTextModelService extends Disposable implements ITreeSitte @ITelemetryService private readonly _telemetryService: ITelemetryService, @ILogService private readonly _logService: ILogService, @IConfigurationService private readonly _configurationService: IConfigurationService, - @IEnvironmentService private readonly _environmentService: IEnvironmentService + @IEnvironmentService private readonly _environmentService: IEnvironmentService, + @ITreeSitterImporter private readonly _treeSitterImporter: ITreeSitterImporter ) { super(); this._treeSitterLanguages = this._register(new TreeSitterLanguages(this._treeSitterImporter, fileService, this._environmentService, this._registeredLanguages)); @@ -601,7 +593,7 @@ export class TreeSitterTextModelService extends Disposable implements ITreeSitte if (language) { const parser = new Parser(); parser.setLanguage(language); - return parser.parse(content); + return parser.parse(content) ?? undefined; } return undefined; } diff --git a/src/vs/editor/common/services/treeSitterParserService.ts b/src/vs/editor/common/services/treeSitterParserService.ts index 4d0ddb3f971..43d480443b2 100644 --- a/src/vs/editor/common/services/treeSitterParserService.ts +++ b/src/vs/editor/common/services/treeSitterParserService.ts @@ -3,11 +3,12 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import type { Parser } from '@vscode/tree-sitter-wasm'; +import type * as Parser from '@vscode/tree-sitter-wasm'; import { Event } from '../../../base/common/event.js'; import { ITextModel } from '../model.js'; import { createDecorator } from '../../../platform/instantiation/common/instantiation.js'; import { Range } from '../core/range.js'; +import { importAMDNodeModule } from '../../../amdX.js'; export const EDITOR_EXPERIMENTAL_PREFER_TREESITTER = 'editor.experimental.preferTreeSitter'; @@ -57,3 +58,50 @@ export interface ITextModelTreeSitter { parse(languageId?: string): Promise; dispose(): void; } + +export const ITreeSitterImporter = createDecorator('treeSitterImporter'); + +export interface ITreeSitterImporter { + readonly _serviceBrand: undefined; + getParserClass(): Promise; + getLanguageClass(): Promise; + getQueryClass(): Promise; +} + +export class TreeSitterImporter implements ITreeSitterImporter { + readonly _serviceBrand: undefined; + private _treeSitterImport: typeof import('@vscode/tree-sitter-wasm') | undefined; + + constructor() { } + + private async _getTreeSitterImport() { + if (!this._treeSitterImport) { + this._treeSitterImport = await importAMDNodeModule('@vscode/tree-sitter-wasm', 'wasm/tree-sitter.js'); + } + return this._treeSitterImport; + } + + private _parserClass: typeof Parser.Parser | undefined; + public async getParserClass() { + if (!this._parserClass) { + this._parserClass = (await this._getTreeSitterImport()).Parser; + } + return this._parserClass; + } + + private _languageClass: typeof Parser.Language | undefined; + public async getLanguageClass() { + if (!this._languageClass) { + this._languageClass = (await this._getTreeSitterImport()).Language; + } + return this._languageClass; + } + + private _queryClass: typeof Parser.Query | undefined; + public async getQueryClass() { + if (!this._queryClass) { + this._queryClass = (await this._getTreeSitterImport()).Query; + } + return this._queryClass; + } +} diff --git a/src/vs/editor/standalone/browser/standaloneTreeSitterService.ts b/src/vs/editor/standalone/browser/standaloneTreeSitterService.ts index 20a2f082f3c..c0f0b1354d9 100644 --- a/src/vs/editor/standalone/browser/standaloneTreeSitterService.ts +++ b/src/vs/editor/standalone/browser/standaloneTreeSitterService.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import type { Parser } from '@vscode/tree-sitter-wasm'; +import type * as Parser from '@vscode/tree-sitter-wasm'; import { Event } from '../../../base/common/event.js'; import { ITextModel } from '../../common/model.js'; import { ITextModelTreeSitter, ITreeSitterParseResult, ITreeSitterParserService, TreeUpdateEvent } from '../../common/services/treeSitterParserService.js'; diff --git a/src/vs/editor/test/browser/services/treeSitterParserService.test.ts b/src/vs/editor/test/browser/services/treeSitterParserService.test.ts index 5475195b7db..f234cc6973e 100644 --- a/src/vs/editor/test/browser/services/treeSitterParserService.test.ts +++ b/src/vs/editor/test/browser/services/treeSitterParserService.test.ts @@ -4,48 +4,57 @@ *--------------------------------------------------------------------------------------------*/ import assert from 'assert'; import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; -import { TextModelTreeSitter, TreeSitterImporter, TreeSitterLanguages } from '../../../common/services/treeSitter/treeSitterParserService.js'; -import type { Parser } from '@vscode/tree-sitter-wasm'; +import { TextModelTreeSitter, TreeSitterLanguages } from '../../../common/services/treeSitter/treeSitterParserService.js'; +import type * as Parser from '@vscode/tree-sitter-wasm'; import { createTextModel } from '../../common/testTextModel.js'; import { timeout } from '../../../../base/common/async.js'; import { ConsoleMainLogger, ILogService } from '../../../../platform/log/common/log.js'; import { ITelemetryService } from '../../../../platform/telemetry/common/telemetry.js'; import { LogService } from '../../../../platform/log/common/logService.js'; import { mock } from '../../../../base/test/common/mock.js'; +import { ITreeSitterImporter } from '../../../common/services/treeSitterParserService.js'; -class MockParser implements Parser { - static async init(): Promise { } +class MockParser implements Parser.Parser { + language: Parser.Language | null = null; delete(): void { } - parse(input: string | Parser.Input, oldTree?: Parser.Tree, options?: Parser.Options): Parser.Tree { + setLanguage(language: Parser.Language | null) { return this; } + parse(callback: string | Parser.ParseCallback, oldTree?: Parser.Tree | null, options?: Parser.ParseOptions): Parser.Tree | null { return new MockTree(); } + reset(): void { } getIncludedRanges(): Parser.Range[] { return []; } getTimeoutMicros(): number { return 0; } setTimeoutMicros(timeout: number): void { } - reset(): void { } - getLanguage(): Parser.Language { return {} as any; } - setLanguage(): void { } - getLogger(): Parser.Logger { + setLogger(callback: Parser.LogCallback | boolean | null): this { throw new Error('Method not implemented.'); } - setLogger(logFunc?: Parser.Logger | false | null): void { + getLogger(): Parser.LogCallback | null { throw new Error('Method not implemented.'); } } -class MockTreeSitterImporter extends TreeSitterImporter { - public override async getParserClass(): Promise { +class MockTreeSitterImporter implements ITreeSitterImporter { + _serviceBrand: undefined; + getParserClass(): Promise { return MockParser as any; } + getLanguageClass(): Promise { + return MockLanguage as any; + } + getQueryClass(): Promise { + throw new Error('Method not implemented.'); + } + } class MockTree implements Parser.Tree { + language: Parser.Language = new MockLanguage(); editorLanguage: string = ''; editorContents: string = ''; - rootNode: Parser.SyntaxNode = {} as any; - rootNodeWithOffset(offsetBytes: number, offsetExtent: Parser.Point): Parser.SyntaxNode { + rootNode: Parser.Node = {} as any; + rootNodeWithOffset(offsetBytes: number, offsetExtent: Parser.Point): Parser.Node { throw new Error('Method not implemented.'); } copy(): Parser.Tree { @@ -73,6 +82,23 @@ class MockTree implements Parser.Tree { } class MockLanguage implements Parser.Language { + types: string[] = []; + fields: (string | null)[] = []; + get name(): string | null { + throw new Error('Method not implemented.'); + } + get abiVersion(): number { + throw new Error('Method not implemented.'); + } + get metadata(): Parser.LanguageMetadata | null { + throw new Error('Method not implemented.'); + } + get supertypes(): number[] { + throw new Error('Method not implemented.'); + } + subtypes(supertype: number): number[] { + throw new Error('Method not implemented.'); + } version: number = 0; fieldCount: number = 0; stateCount: number = 0; @@ -101,14 +127,14 @@ class MockLanguage implements Parser.Language { query(source: string): Parser.Query { throw new Error('Method not implemented.'); } - lookaheadIterator(stateId: number): Parser.LookaheadIterable | null { + lookaheadIterator(stateId: number): Parser.LookaheadIterator | null { throw new Error('Method not implemented.'); } languageId: string = ''; } suite('TreeSitterParserService', function () { - const treeSitterImporter: TreeSitterImporter = new MockTreeSitterImporter(); + const treeSitterImporter: ITreeSitterImporter = new MockTreeSitterImporter(); let logService: ILogService; let telemetryService: ITelemetryService; setup(function () { diff --git a/src/vs/editor/test/common/services/testTreeSitterService.ts b/src/vs/editor/test/common/services/testTreeSitterService.ts index e2ca50462ec..a41369c53e0 100644 --- a/src/vs/editor/test/common/services/testTreeSitterService.ts +++ b/src/vs/editor/test/common/services/testTreeSitterService.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import type { Parser } from '@vscode/tree-sitter-wasm'; +import type * as Parser from '@vscode/tree-sitter-wasm'; import { Event } from '../../../../base/common/event.js'; import { ITextModel } from '../../../common/model.js'; import { ITreeSitterParserService, ITreeSitterParseResult, ITextModelTreeSitter, TreeUpdateEvent } from '../../../common/services/treeSitterParserService.js'; diff --git a/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts b/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts index 4da76c31144..ff2710dcd2b 100644 --- a/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts +++ b/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts @@ -32,7 +32,7 @@ import { SEMANTIC_HIGHLIGHTING_SETTING_ID, IEditorSemanticHighlightingOptions } import { Schemas } from '../../../../../base/common/network.js'; import { ILanguageFeaturesService } from '../../../../../editor/common/services/languageFeatures.js'; import { ITreeSitterParserService } from '../../../../../editor/common/services/treeSitterParserService.js'; -import type { Parser } from '@vscode/tree-sitter-wasm'; +import type * as Parser from '@vscode/tree-sitter-wasm'; const $ = dom.$; @@ -646,11 +646,11 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { return null; } - private _walkTreeforPosition(cursor: Parser.TreeCursor, pos: Position): Parser.SyntaxNode | null { + private _walkTreeforPosition(cursor: Parser.TreeCursor, pos: Position): Parser.Node | null { const offset = this._model.getOffsetAt(pos); cursor.gotoFirstChild(); let goChild: boolean = false; - let lastGoodNode: Parser.SyntaxNode | null = null; + let lastGoodNode: Parser.Node | null = null; do { if (cursor.currentNode.startIndex <= offset && offset < cursor.currentNode.endIndex) { goChild = true; @@ -662,7 +662,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { return lastGoodNode; } - private _getTreeSitterTokenAtPosition(tree: Parser.Tree, pos: Position): Parser.SyntaxNode | null { + private _getTreeSitterTokenAtPosition(tree: Parser.Tree, pos: Position): Parser.Node | null { const cursor = tree.walk(); return this._walkTreeforPosition(cursor, pos); diff --git a/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts b/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts index 57e6c944465..496922ef391 100644 --- a/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts +++ b/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { URI } from '../../../../base/common/uri.js'; -import type { Parser } from '@vscode/tree-sitter-wasm'; +import type * as Parser from '@vscode/tree-sitter-wasm'; import { ILanguageService } from '../../../../editor/common/languages/language.js'; import { CommandsRegistry } from '../../../../platform/commands/common/commands.js'; import { IInstantiationService, ServicesAccessor } from '../../../../platform/instantiation/common/instantiation.js'; diff --git a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.ts b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.ts index 21701ec617c..6c06c1b189c 100644 --- a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.ts +++ b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.contribution.ts @@ -6,7 +6,7 @@ import { registerSingleton, InstantiationType } from '../../../../platform/instantiation/common/extensions.js'; import { IWorkbenchContribution, WorkbenchPhase, registerWorkbenchContribution2 } from '../../../common/contributions.js'; import { TreeSitterTextModelService } from '../../../../editor/common/services/treeSitter/treeSitterParserService.js'; -import { ITreeSitterParserService } from '../../../../editor/common/services/treeSitterParserService.js'; +import { ITreeSitterImporter, ITreeSitterParserService, TreeSitterImporter } from '../../../../editor/common/services/treeSitterParserService.js'; import { ITreeSitterTokenizationFeature } from './treeSitterTokenizationFeature.js'; import { ServicesAccessor } from '../../../../platform/instantiation/common/instantiation.js'; import { CommandsRegistry } from '../../../../platform/commands/common/commands.js'; @@ -28,6 +28,7 @@ class TreeSitterTokenizationInstantiator implements IWorkbenchContribution { ) { } } +registerSingleton(ITreeSitterImporter, TreeSitterImporter, InstantiationType.Eager); registerSingleton(ITreeSitterParserService, TreeSitterTextModelService, InstantiationType.Eager); registerWorkbenchContribution2(TreeSitterTokenizationInstantiator.ID, TreeSitterTokenizationInstantiator, WorkbenchPhase.BlockRestore); diff --git a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts index 34bcb6b7536..139cd64d5c3 100644 --- a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts +++ b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts @@ -3,13 +3,13 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import type { Parser } from '@vscode/tree-sitter-wasm'; +import type * as Parser from '@vscode/tree-sitter-wasm'; import { Emitter, Event } from '../../../../base/common/event.js'; import { Disposable, DisposableMap, DisposableStore, IDisposable } from '../../../../base/common/lifecycle.js'; import { AppResourcePath, FileAccess } from '../../../../base/common/network.js'; import { ILanguageIdCodec, ITreeSitterTokenizationSupport, LazyTokenizationSupport, QueryCapture, TreeSitterTokenizationRegistry } from '../../../../editor/common/languages.js'; import { ITextModel } from '../../../../editor/common/model.js'; -import { EDITOR_EXPERIMENTAL_PREFER_TREESITTER, ITreeSitterParserService, ITreeSitterParseResult, TreeUpdateEvent, RangeChange } from '../../../../editor/common/services/treeSitterParserService.js'; +import { EDITOR_EXPERIMENTAL_PREFER_TREESITTER, ITreeSitterParserService, ITreeSitterParseResult, TreeUpdateEvent, RangeChange, ITreeSitterImporter } from '../../../../editor/common/services/treeSitterParserService.js'; import { IModelTokensChangedEvent } from '../../../../editor/common/textModelEvents.js'; import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js'; import { IFileService } from '../../../../platform/files/common/files.js'; @@ -45,6 +45,7 @@ export class TreeSitterTokenizationFeature extends Disposable implements ITreeSi private readonly _tokenizersRegistrations: DisposableMap = this._register(new DisposableMap()); constructor( + @ITreeSitterImporter private readonly _treeSitterImporter: ITreeSitterImporter, @ILanguageService private readonly _languageService: ILanguageService, @IConfigurationService private readonly _configurationService: IConfigurationService, @IInstantiationService private readonly _instantiationService: IInstantiationService, @@ -92,7 +93,8 @@ export class TreeSitterTokenizationFeature extends Disposable implements ITreeSi private async _createTokenizationSupport(languageId: string): Promise { const queries = await this._fetchQueries(languageId); - return this._instantiationService.createInstance(TreeSitterTokenizationSupport, queries, languageId, this._languageService.languageIdCodec); + const Query = await this._treeSitterImporter.getQueryClass(); + return this._instantiationService.createInstance(TreeSitterTokenizationSupport, queries, Query, languageId, this._languageService.languageIdCodec); } } @@ -105,6 +107,7 @@ export class TreeSitterTokenizationSupport extends Disposable implements ITreeSi constructor( private readonly _queries: TreeSitterQueries, + private readonly Query: typeof Parser.Query, private readonly _languageId: string, private readonly _languageIdCodec: ILanguageIdCodec, @ITreeSitterParserService private readonly _treeSitterService: ITreeSitterParserService, @@ -314,12 +317,12 @@ export class TreeSitterTokenizationSupport extends Disposable implements ITreeSi if (!language) { if (!this._languageAddedListener) { this._languageAddedListener = this._register(Event.onceIf(this._treeSitterService.onDidAddLanguage, e => e.id === this._languageId)((e) => { - this._query = e.language.query(this._queries); + this._query = new this.Query(e.language, this._queries); })); } return; } - this._query = language.query(this._queries); + this._query = new this.Query(language, this._queries); } return this._query; } diff --git a/src/vs/workbench/test/electron-main/treeSitterTokenizationFeature.test.ts b/src/vs/workbench/test/electron-main/treeSitterTokenizationFeature.test.ts index 1eab214ca26..c246b1e8568 100644 --- a/src/vs/workbench/test/electron-main/treeSitterTokenizationFeature.test.ts +++ b/src/vs/workbench/test/electron-main/treeSitterTokenizationFeature.test.ts @@ -20,7 +20,7 @@ import { IEnvironmentService } from '../../../platform/environment/common/enviro import { ModelService } from '../../../editor/common/services/modelService.js'; // eslint-disable-next-line local/code-layering, local/code-import-patterns import { TreeSitterTokenizationFeature } from '../../services/treeSitter/browser/treeSitterTokenizationFeature.js'; -import { ITreeSitterParserService, TreeUpdateEvent } from '../../../editor/common/services/treeSitterParserService.js'; +import { ITreeSitterImporter, ITreeSitterParserService, TreeSitterImporter, TreeUpdateEvent } from '../../../editor/common/services/treeSitterParserService.js'; import { ITreeSitterTokenizationSupport, TreeSitterTokenizationRegistry } from '../../../editor/common/languages.js'; import { FileService } from '../../../platform/files/common/fileService.js'; import { Schemas } from '../../../base/common/network.js'; @@ -131,6 +131,7 @@ suite('Tree Sitter TokenizationFeature', function () { instantiationService.set(ITextResourcePropertiesService, textResourcePropertiesService); languageConfigurationService = disposables.add(instantiationService.createInstance(TestLanguageConfigurationService)); instantiationService.set(ILanguageConfigurationService, languageConfigurationService); + instantiationService.set(ITreeSitterImporter, instantiationService.createInstance(TreeSitterImporter)); fileService = disposables.add(instantiationService.createInstance(FileService)); const diskFileSystemProvider = disposables.add(new DiskFileSystemProvider(logService));