diff --git a/src/vs/editor/common/model/textModelTokens.ts b/src/vs/editor/common/model/textModelTokens.ts index 5b84ecd6322..c6591d3fd64 100644 --- a/src/vs/editor/common/model/textModelTokens.ts +++ b/src/vs/editor/common/model/textModelTokens.ts @@ -28,26 +28,15 @@ export class TokenizationStateStore { private _len: number; private _invalidLineStartIndex: number; - constructor() { + constructor( + public readonly tokenizationSupport: ITokenizationSupport, + public readonly initialState: IState + ) { this._beginState = []; this._valid = []; this._len = 0; this._invalidLineStartIndex = 0; - } - - private _reset(initialState: IState | null): void { - this._beginState = []; - this._valid = []; - this._len = 0; - this._invalidLineStartIndex = 0; - - if (initialState) { - this._setBeginState(0, initialState); - } - } - - public flush(initialState: IState | null): void { - this._reset(initialState); + this._setBeginState(0, this.initialState); } public get invalidLineStartIndex() { @@ -195,9 +184,8 @@ export class TokenizationStateStore { export class TextModelTokenization extends Disposable { - private readonly _tokenizationStateStore: TokenizationStateStore; + private _tokenizationStateStore: TokenizationStateStore | null; private _isDisposed: boolean; - private _tokenizationSupport: ITokenizationSupport | null; constructor( private readonly _textModel: TextModel, @@ -205,8 +193,7 @@ export class TextModelTokenization extends Disposable { ) { super(); this._isDisposed = false; - this._tokenizationStateStore = new TokenizationStateStore(); - this._tokenizationSupport = null; + this._tokenizationStateStore = null; this._register(TokenizationRegistry.onDidChange((e) => { const languageId = this._textModel.getLanguageId(); @@ -223,10 +210,12 @@ export class TextModelTokenization extends Disposable { this._resetTokenizationState(); return; } - for (let i = 0, len = e.changes.length; i < len; i++) { - const change = e.changes[i]; - const [eolCount] = countEOL(change.text); - this._tokenizationStateStore.applyEdits(change.range, eolCount); + if (this._tokenizationStateStore) { + for (let i = 0, len = e.changes.length; i < len; i++) { + const change = e.changes[i]; + const [eolCount] = countEOL(change.text); + this._tokenizationStateStore.applyEdits(change.range, eolCount); + } } this._beginBackgroundTokenization(); @@ -251,8 +240,11 @@ export class TextModelTokenization extends Disposable { private _resetTokenizationState(): void { const [tokenizationSupport, initialState] = initializeTokenization(this._textModel); - this._tokenizationSupport = tokenizationSupport; - this._tokenizationStateStore.flush(initialState); + if (tokenizationSupport && initialState) { + this._tokenizationStateStore = new TokenizationStateStore(tokenizationSupport, initialState); + } else { + this._tokenizationStateStore = null; + } this._beginBackgroundTokenization(); } @@ -342,7 +334,7 @@ export class TextModelTokenization extends Disposable { } public getTokenTypeIfInsertingCharacter(position: Position, character: string): StandardTokenType { - if (!this._tokenizationSupport) { + if (!this._tokenizationStateStore) { return StandardTokenType.Other; } @@ -362,7 +354,7 @@ export class TextModelTokenization extends Disposable { + lineContent.substring(position.column - 1) ); - const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationSupport, text, true, lineStartState); + const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationStateStore.tokenizationSupport, text, true, lineStartState); const lineTokens = new LineTokens(r.tokens, text, this._languageIdCodec); if (lineTokens.getCount() === 0) { return StandardTokenType.Other; @@ -376,7 +368,7 @@ export class TextModelTokenization extends Disposable { const lineNumber = position.lineNumber; const column = position.column; - if (!this._tokenizationSupport) { + if (!this._tokenizationStateStore) { return null; } @@ -394,7 +386,7 @@ export class TextModelTokenization extends Disposable { const result = safeTokenize( this._languageIdCodec, languageId, - this._tokenizationSupport, + this._tokenizationStateStore.tokenizationSupport, newLineContent, true, lineStartState @@ -405,7 +397,7 @@ export class TextModelTokenization extends Disposable { } public isCheapToTokenize(lineNumber: number): boolean { - if (!this._tokenizationSupport) { + if (!this._tokenizationStateStore) { return true; } @@ -426,14 +418,14 @@ export class TextModelTokenization extends Disposable { } private _hasLinesToTokenize(): boolean { - if (!this._tokenizationSupport) { + if (!this._tokenizationStateStore) { return false; } return (this._tokenizationStateStore.invalidLineStartIndex < this._textModel.getLineCount()); } private _tokenizeOneInvalidLine(builder: ContiguousMultilineTokensBuilder): number { - if (!this._hasLinesToTokenize()) { + if (!this._tokenizationStateStore || !this._hasLinesToTokenize()) { return this._textModel.getLineCount() + 1; } const lineNumber = this._tokenizationStateStore.invalidLineStartIndex + 1; @@ -442,7 +434,7 @@ export class TextModelTokenization extends Disposable { } private _updateTokensUntilLine(builder: ContiguousMultilineTokensBuilder, lineNumber: number): void { - if (!this._tokenizationSupport) { + if (!this._tokenizationStateStore) { return; } const languageId = this._textModel.getLanguageId(); @@ -454,7 +446,7 @@ export class TextModelTokenization extends Disposable { const text = this._textModel.getLineContent(lineIndex + 1); const lineStartState = this._tokenizationStateStore.getBeginState(lineIndex); - const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationSupport, text, true, lineStartState!); + const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationStateStore.tokenizationSupport, text, true, lineStartState!); builder.add(lineIndex + 1, r.tokens); this._tokenizationStateStore.setEndState(linesLength, lineIndex, r.endState); lineIndex = this._tokenizationStateStore.invalidLineStartIndex - 1; // -1 because the outer loop increments it @@ -462,7 +454,7 @@ export class TextModelTokenization extends Disposable { } private _tokenizeViewport(builder: ContiguousMultilineTokensBuilder, startLineNumber: number, endLineNumber: number): void { - if (!this._tokenizationSupport) { + if (!this._tokenizationStateStore) { // nothing to do return; } @@ -499,19 +491,19 @@ export class TextModelTokenization extends Disposable { } if (!initialState) { - initialState = this._tokenizationSupport.getInitialState(); + initialState = this._tokenizationStateStore.initialState; } const languageId = this._textModel.getLanguageId(); let state = initialState; for (let i = fakeLines.length - 1; i >= 0; i--) { - const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationSupport, fakeLines[i], false, state); + const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationStateStore.tokenizationSupport, fakeLines[i], false, state); state = r.endState; } for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) { const text = this._textModel.getLineContent(lineNumber); - const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationSupport, text, true, state); + const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationStateStore.tokenizationSupport, text, true, state); builder.add(lineNumber, r.tokens); this._tokenizationStateStore.setFakeTokens(lineNumber - 1); state = r.endState; @@ -519,21 +511,20 @@ export class TextModelTokenization extends Disposable { } } -function initializeTokenization(textModel: TextModel): [ITokenizationSupport | null, IState | null] { - const languageId = textModel.getLanguageId(); - let tokenizationSupport = ( - textModel.isTooLargeForTokenization() - ? null - : TokenizationRegistry.get(languageId) - ); - let initialState: IState | null = null; - if (tokenizationSupport) { - try { - initialState = tokenizationSupport.getInitialState(); - } catch (e) { - onUnexpectedError(e); - tokenizationSupport = null; - } +function initializeTokenization(textModel: TextModel): [ITokenizationSupport, IState] | [null, null] { + if (textModel.isTooLargeForTokenization()) { + return [null, null]; + } + const tokenizationSupport = TokenizationRegistry.get(textModel.getLanguageId()); + if (!tokenizationSupport) { + return [null, null]; + } + let initialState: IState; + try { + initialState = tokenizationSupport.getInitialState(); + } catch (e) { + onUnexpectedError(e); + return [null, null]; } return [tokenizationSupport, initialState]; } diff --git a/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts b/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts index c48fc691dab..25eecc009ad 100644 --- a/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts +++ b/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts @@ -12,7 +12,7 @@ import * as resources from 'vs/base/common/resources'; import * as types from 'vs/base/common/types'; import { equals as equalArray } from 'vs/base/common/arrays'; import { URI } from 'vs/base/common/uri'; -import { IState, ITokenizationSupport, LanguageId, TokenMetadata, TokenizationRegistry, StandardTokenType, ITokenizationSupportFactory, TokenizationResult, EncodedTokenizationResult } from 'vs/editor/common/languages'; +import { IState, ITokenizationSupport, LanguageId, TokenizationRegistry, StandardTokenType, ITokenizationSupportFactory, TokenizationResult, EncodedTokenizationResult } from 'vs/editor/common/languages'; import { nullTokenizeEncoded } from 'vs/editor/common/languages/nullMode'; import { generateTokensCSSForColorMap } from 'vs/editor/common/languages/supports/tokenization'; import { ILanguageService } from 'vs/editor/common/services/language'; @@ -29,6 +29,7 @@ import { IValidGrammarDefinition, IValidEmbeddedLanguagesMap, IValidTokenTypeMap import { TMGrammarFactory } from 'vs/workbench/services/textMate/common/TMGrammarFactory'; import { IExtensionResourceLoaderService } from 'vs/workbench/services/extensionResourceLoader/common/extensionResourceLoader'; import { IProgressService, ProgressLocation } from 'vs/platform/progress/common/progress'; +import { TMTokenization } from 'vs/workbench/services/textMate/common/TMTokenization'; export abstract class AbstractTextMateService extends Disposable implements ITextMateService { public _serviceBrand: undefined; @@ -263,7 +264,7 @@ export abstract class AbstractTextMateService extends Disposable implements ITex this._onDidEncounterLanguage.fire(languageId); } }); - return new TMTokenizationSupport(languageId, encodedLanguageId, tokenization, this._configurationService); + return new TMTokenizationSupportWithLineLimit(languageId, encodedLanguageId, tokenization, this._configurationService); } catch (err) { onUnexpectedError(err); return null; @@ -396,7 +397,7 @@ export abstract class AbstractTextMateService extends Disposable implements ITex protected abstract _loadVSCodeOnigurumWASM(): Promise; } -class TMTokenizationSupport implements ITokenizationSupport { +class TMTokenizationSupportWithLineLimit implements ITokenizationSupport { private readonly _languageId: string; private readonly _encodedLanguageId: LanguageId; private readonly _actual: TMTokenization; @@ -437,66 +438,6 @@ class TMTokenizationSupport implements ITokenizationSupport { return nullTokenizeEncoded(this._encodedLanguageId, state); } - return this._actual.tokenizeEncoded(line, state); - } -} - -class TMTokenization extends Disposable { - - private readonly _grammar: IGrammar; - private readonly _containsEmbeddedLanguages: boolean; - private readonly _seenLanguages: boolean[]; - private readonly _initialState: StackElement; - - private readonly _onDidEncounterLanguage: Emitter = this._register(new Emitter()); - public readonly onDidEncounterLanguage: Event = this._onDidEncounterLanguage.event; - - constructor(grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean) { - super(); - this._grammar = grammar; - this._initialState = initialState; - this._containsEmbeddedLanguages = containsEmbeddedLanguages; - this._seenLanguages = []; - } - - public getInitialState(): IState { - return this._initialState; - } - - public tokenizeEncoded(line: string, state: StackElement): EncodedTokenizationResult { - const textMateResult = this._grammar.tokenizeLine2(line, state, 500); - - if (textMateResult.stoppedEarly) { - console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`); - // return the state at the beginning of the line - return new EncodedTokenizationResult(textMateResult.tokens, state); - } - - if (this._containsEmbeddedLanguages) { - let seenLanguages = this._seenLanguages; - let tokens = textMateResult.tokens; - - // Must check if any of the embedded languages was hit - for (let i = 0, len = (tokens.length >>> 1); i < len; i++) { - let metadata = tokens[(i << 1) + 1]; - let languageId = TokenMetadata.getLanguageId(metadata); - - if (!seenLanguages[languageId]) { - seenLanguages[languageId] = true; - this._onDidEncounterLanguage.fire(languageId); - } - } - } - - let endState: StackElement; - // try to save an object if possible - if (state.equals(textMateResult.ruleStack)) { - endState = state; - } else { - endState = textMateResult.ruleStack; - - } - - return new EncodedTokenizationResult(textMateResult.tokens, endState); + return this._actual.tokenizeEncoded(line, hasEOL, state); } } diff --git a/src/vs/workbench/services/textMate/browser/textMateWorker.ts b/src/vs/workbench/services/textMate/browser/textMateWorker.ts index 35672fbb115..6c88a038203 100644 --- a/src/vs/workbench/services/textMate/browser/textMateWorker.ts +++ b/src/vs/workbench/services/textMate/browser/textMateWorker.ts @@ -11,11 +11,12 @@ import { TMGrammarFactory, ICreateGrammarResult } from 'vs/workbench/services/te import { IModelChangedEvent, MirrorTextModel } from 'vs/editor/common/model/mirrorTextModel'; import { TextMateWorkerHost } from 'vs/workbench/services/textMate/browser/nativeTextMateService'; import { TokenizationStateStore } from 'vs/editor/common/model/textModelTokens'; -import type { IGrammar, StackElement, IRawTheme, IOnigLib } from 'vscode-textmate'; +import type { IRawTheme, IOnigLib } from 'vscode-textmate'; import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/tokens/contiguousMultilineTokensBuilder'; import { countEOL } from 'vs/editor/common/core/eolCounter'; import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { FileAccess } from 'vs/base/common/network'; +import { TMTokenization } from 'vs/workbench/services/textMate/common/TMTokenization'; export interface IValidGrammarDefinitionDTO { location: UriComponents; @@ -41,21 +42,19 @@ export interface IRawModelData { class TextMateWorkerModel extends MirrorTextModel { - private readonly _tokenizationStateStore: TokenizationStateStore; + private _tokenizationStateStore: TokenizationStateStore | null; private readonly _worker: TextMateWorker; private _languageId: string; private _encodedLanguageId: LanguageId; - private _grammar: IGrammar | null; private _isDisposed: boolean; constructor(uri: URI, lines: string[], eol: string, versionId: number, worker: TextMateWorker, languageId: string, encodedLanguageId: LanguageId) { super(uri, lines, eol, versionId); - this._tokenizationStateStore = new TokenizationStateStore(); + this._tokenizationStateStore = null; this._worker = worker; this._languageId = languageId; this._encodedLanguageId = encodedLanguageId; this._isDisposed = false; - this._grammar = null; this._resetTokenization(); } @@ -72,17 +71,18 @@ class TextMateWorkerModel extends MirrorTextModel { override onEvents(e: IModelChangedEvent): void { super.onEvents(e); - for (let i = 0; i < e.changes.length; i++) { - const change = e.changes[i]; - const [eolCount] = countEOL(change.text); - this._tokenizationStateStore.applyEdits(change.range, eolCount); + if (this._tokenizationStateStore) { + for (let i = 0; i < e.changes.length; i++) { + const change = e.changes[i]; + const [eolCount] = countEOL(change.text); + this._tokenizationStateStore.applyEdits(change.range, eolCount); + } } this._ensureTokens(); } private _resetTokenization(): void { - this._grammar = null; - this._tokenizationStateStore.flush(null); + this._tokenizationStateStore = null; const languageId = this._languageId; const encodedLanguageId = this._encodedLanguageId; @@ -91,14 +91,18 @@ class TextMateWorkerModel extends MirrorTextModel { return; } - this._grammar = r.grammar; - this._tokenizationStateStore.flush(r.initialState); + if (r.grammar) { + const tokenizationSupport = new TMTokenization(r.grammar, r.initialState, false); + this._tokenizationStateStore = new TokenizationStateStore(tokenizationSupport, tokenizationSupport.getInitialState()); + } else { + this._tokenizationStateStore = null; + } this._ensureTokens(); }); } private _ensureTokens(): void { - if (!this._grammar) { + if (!this._tokenizationStateStore) { return; } const builder = new ContiguousMultilineTokensBuilder(); @@ -109,10 +113,10 @@ class TextMateWorkerModel extends MirrorTextModel { const text = this._lines[lineIndex]; const lineStartState = this._tokenizationStateStore.getBeginState(lineIndex); - const r = this._grammar.tokenizeLine2(text, lineStartState!); + const r = this._tokenizationStateStore.tokenizationSupport.tokenizeEncoded(text, true, lineStartState!); LineTokens.convertToEndOffset(r.tokens, text.length); builder.add(lineIndex + 1, r.tokens); - this._tokenizationStateStore.setEndState(lineCount, lineIndex, r.ruleStack); + this._tokenizationStateStore.setEndState(lineCount, lineIndex, r.endState); lineIndex = this._tokenizationStateStore.invalidLineStartIndex - 1; // -1 because the outer loop increments it } diff --git a/src/vs/workbench/services/textMate/common/TMTokenization.ts b/src/vs/workbench/services/textMate/common/TMTokenization.ts new file mode 100644 index 00000000000..64816e5d912 --- /dev/null +++ b/src/vs/workbench/services/textMate/common/TMTokenization.ts @@ -0,0 +1,73 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Emitter, Event } from 'vs/base/common/event'; +import { IState, ITokenizationSupport, LanguageId, TokenMetadata, TokenizationResult, EncodedTokenizationResult } from 'vs/editor/common/languages'; +import type { IGrammar, StackElement } from 'vscode-textmate'; +import { Disposable } from 'vs/base/common/lifecycle'; + +export class TMTokenization extends Disposable implements ITokenizationSupport { + + private readonly _grammar: IGrammar; + private readonly _containsEmbeddedLanguages: boolean; + private readonly _seenLanguages: boolean[]; + private readonly _initialState: StackElement; + + private readonly _onDidEncounterLanguage: Emitter = this._register(new Emitter()); + public readonly onDidEncounterLanguage: Event = this._onDidEncounterLanguage.event; + + constructor(grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean) { + super(); + this._grammar = grammar; + this._initialState = initialState; + this._containsEmbeddedLanguages = containsEmbeddedLanguages; + this._seenLanguages = []; + } + + public getInitialState(): IState { + return this._initialState; + } + + public tokenize(line: string, hasEOL: boolean, state: IState): TokenizationResult { + throw new Error('Not supported!'); + } + + public tokenizeEncoded(line: string, hasEOL: boolean, state: StackElement): EncodedTokenizationResult { + const textMateResult = this._grammar.tokenizeLine2(line, state, 500); + + if (textMateResult.stoppedEarly) { + console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`); + // return the state at the beginning of the line + return new EncodedTokenizationResult(textMateResult.tokens, state); + } + + if (this._containsEmbeddedLanguages) { + let seenLanguages = this._seenLanguages; + let tokens = textMateResult.tokens; + + // Must check if any of the embedded languages was hit + for (let i = 0, len = (tokens.length >>> 1); i < len; i++) { + let metadata = tokens[(i << 1) + 1]; + let languageId = TokenMetadata.getLanguageId(metadata); + + if (!seenLanguages[languageId]) { + seenLanguages[languageId] = true; + this._onDidEncounterLanguage.fire(languageId); + } + } + } + + let endState: StackElement; + // try to save an object if possible + if (state.equals(textMateResult.ruleStack)) { + endState = state; + } else { + endState = textMateResult.ruleStack; + + } + + return new EncodedTokenizationResult(textMateResult.tokens, endState); + } +}