From 72b1ad9f24f854d4b87af56e603b9035a3040b5d Mon Sep 17 00:00:00 2001 From: Henning Dieterichs Date: Tue, 14 Mar 2023 18:07:57 +0100 Subject: [PATCH] Fixes #175125 --- src/vs/base/common/observableImpl/utils.ts | 8 +- .../textMateTokenizationFeatureImpl.ts | 22 ++++- .../tokenizationSupportWithLineLimit.ts | 21 +---- .../browser/worker/textMate.worker.ts | 7 +- .../browser/worker/textMateWorkerModel.ts | 84 ++++++++++++++++--- .../browser/workerHost/textMateWorkerHost.ts | 5 +- .../textMateWorkerTokenizerController.ts | 9 +- 7 files changed, 120 insertions(+), 36 deletions(-) diff --git a/src/vs/base/common/observableImpl/utils.ts b/src/vs/base/common/observableImpl/utils.ts index b021df0fc11..de5e6c73f14 100644 --- a/src/vs/base/common/observableImpl/utils.ts +++ b/src/vs/base/common/observableImpl/utils.ts @@ -276,8 +276,12 @@ export function wasEventTriggeredRecently(event: Event, timeoutMs: number, } /** - * This ensures the observable is kept up-to-date. - * This is useful when the observables `get` method is used. + * This ensures the observable cache is kept up-to-date, even if there are no subscribers. + * This is useful when the observables `get` method is used, but not its `read` method. + * + * (Usually, when no one is actually observing the observable, getting its value will + * compute it from scratch, as the cache cannot be trusted: + * Because no one is actually observing its value, keeping the cache up-to-date would be too expensive) */ export function keepAlive(observable: IObservable): IDisposable { const o = new KeepAliveObserver(); diff --git a/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts b/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts index 5d4cbe6a671..78479a85dad 100644 --- a/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts +++ b/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts @@ -9,6 +9,7 @@ import { Color } from 'vs/base/common/color'; import { onUnexpectedError } from 'vs/base/common/errors'; import { Disposable, DisposableStore, IDisposable } from 'vs/base/common/lifecycle'; import { FileAccess, nodeModulesAsarUnpackedPath, nodeModulesPath } from 'vs/base/common/network'; +import { IObservable, observableFromEvent } from 'vs/base/common/observable'; import { isWeb } from 'vs/base/common/platform'; import * as resources from 'vs/base/common/resources'; import * as types from 'vs/base/common/types'; @@ -270,11 +271,17 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate if (!r.grammar) { return null; } + const maxTokenizationLineLength = observableConfigValue( + 'editor.maxTokenizationLineLength', + languageId, + -1, + this._configurationService + ); const tokenization = new TextMateTokenizationSupport( r.grammar, r.initialState, r.containsEmbeddedLanguages, - (textModel, tokenStore) => this._workerHost.createBackgroundTokenizer(textModel, tokenStore), + (textModel, tokenStore) => this._workerHost.createBackgroundTokenizer(textModel, tokenStore, maxTokenizationLineLength), ); tokenization.onDidEncounterLanguage((encodedLanguageId) => { if (!this._encounteredLanguages[encodedLanguageId]) { @@ -283,7 +290,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate this._languageService.requestBasicLanguageFeatures(languageId); } }); - return new TokenizationSupportWithLineLimit(languageId, encodedLanguageId, tokenization, this._configurationService); + return new TokenizationSupportWithLineLimit(encodedLanguageId, tokenization, maxTokenizationLineLength); } catch (err) { if (err.message && err.message === missingTMGrammarErrorMessage) { // Don't log this error message @@ -423,3 +430,14 @@ function validateGrammarExtensionPoint(extensionLocation: URI, syntax: ITMSyntax } return true; } + +function observableConfigValue(key: string, languageId: string, defaultValue: T, configurationService: IConfigurationService): IObservable { + return observableFromEvent( + (handleChange) => configurationService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(key, { overrideIdentifier: languageId })) { + handleChange(e); + } + }), + () => configurationService.getValue(key, { overrideIdentifier: languageId }) ?? defaultValue, + ); +} diff --git a/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.ts b/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.ts index 39f1c34a1eb..51d661040ac 100644 --- a/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.ts +++ b/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.ts @@ -7,31 +7,18 @@ import { LanguageId } from 'vs/editor/common/encodedTokenAttributes'; import { EncodedTokenizationResult, IBackgroundTokenizationStore, IBackgroundTokenizer, IState, ITokenizationSupport, TokenizationResult } from 'vs/editor/common/languages'; import { nullTokenizeEncoded } from 'vs/editor/common/languages/nullTokenize'; import { ITextModel } from 'vs/editor/common/model'; -import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { Disposable } from 'vs/base/common/lifecycle'; +import { IObservable, keepAlive } from 'vs/base/common/observable'; export class TokenizationSupportWithLineLimit extends Disposable implements ITokenizationSupport { - private _maxTokenizationLineLength: number; - constructor( - private readonly _languageId: string, private readonly _encodedLanguageId: LanguageId, private readonly _actual: ITokenizationSupport, - @IConfigurationService private readonly _configurationService: IConfigurationService, + private readonly _maxTokenizationLineLength: IObservable, ) { super(); - this._maxTokenizationLineLength = this._configurationService.getValue('editor.maxTokenizationLineLength', { - overrideIdentifier: this._languageId - }); - - this._register(this._configurationService.onDidChangeConfiguration(e => { - if (e.affectsConfiguration('editor.maxTokenizationLineLength')) { - this._maxTokenizationLineLength = this._configurationService.getValue('editor.maxTokenizationLineLength', { - overrideIdentifier: this._languageId - }); - } - })); + this._register(keepAlive(this._maxTokenizationLineLength)); } getInitialState(): IState { @@ -44,7 +31,7 @@ export class TokenizationSupportWithLineLimit extends Disposable implements ITok tokenizeEncoded(line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult { // Do not attempt to tokenize if a line is too long - if (line.length >= this._maxTokenizationLineLength) { + if (line.length >= this._maxTokenizationLineLength.get()) { return nullTokenizeEncoded(this._encodedLanguageId, state); } diff --git a/src/vs/workbench/services/textMate/browser/worker/textMate.worker.ts b/src/vs/workbench/services/textMate/browser/worker/textMate.worker.ts index d339e2e99a2..67484e6ef40 100644 --- a/src/vs/workbench/services/textMate/browser/worker/textMate.worker.ts +++ b/src/vs/workbench/services/textMate/browser/worker/textMate.worker.ts @@ -84,7 +84,7 @@ export class TextMateTokenizationWorker { public acceptNewModel(data: IRawModelData): void { const uri = URI.revive(data.uri); const key = uri.toString(); - this._models[key] = new TextMateWorkerModel(uri, data.lines, data.EOL, data.versionId, this, data.languageId, data.encodedLanguageId); + this._models[key] = new TextMateWorkerModel(uri, data.lines, data.EOL, data.versionId, this, data.languageId, data.encodedLanguageId, data.maxTokenizationLineLength); } public acceptModelChanged(strURL: string, e: IModelChangedEvent): void { @@ -111,6 +111,10 @@ export class TextMateTokenizationWorker { grammarFactory?.setTheme(theme, colorMap); } + public acceptMaxTokenizationLineLength(strURL: string, value: number): void { + this._models[strURL].acceptMaxTokenizationLineLength(value); + } + // #endregion // #region called by worker model @@ -140,6 +144,7 @@ export interface IRawModelData { EOL: string; languageId: string; encodedLanguageId: LanguageId; + maxTokenizationLineLength: number; } export function create(ctx: IWorkerContext, createData: ICreateData): TextMateTokenizationWorker { diff --git a/src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts b/src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts index b4b1a6ca147..23704cbde0d 100644 --- a/src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts +++ b/src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts @@ -15,6 +15,8 @@ import { TextMateTokenizationSupport } from 'vs/workbench/services/textMate/brow import { StateDeltas } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost'; import { RunOnceScheduler } from 'vs/base/common/async'; import { TextMateTokenizationWorker } from './textMate.worker'; +import { observableValue } from 'vs/base/common/observable'; +import { TokenizationSupportWithLineLimit } from 'vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit'; export class TextMateWorkerModel extends MirrorTextModel { private _tokenizationStateStore: TokenizationStateStore | null; @@ -22,14 +24,28 @@ export class TextMateWorkerModel extends MirrorTextModel { private _languageId: string; private _encodedLanguageId: LanguageId; private _isDisposed: boolean; + private readonly _maxTokenizationLineLength = observableValue( + '_maxTokenizationLineLength', + -1 + ); - constructor(uri: URI, lines: string[], eol: string, versionId: number, worker: TextMateTokenizationWorker, languageId: string, encodedLanguageId: LanguageId) { + constructor( + uri: URI, + lines: string[], + eol: string, + versionId: number, + worker: TextMateTokenizationWorker, + languageId: string, + encodedLanguageId: LanguageId, + maxTokenizationLineLength: number, + ) { super(uri, lines, eol, versionId); this._tokenizationStateStore = null; this._worker = worker; this._languageId = languageId; this._encodedLanguageId = encodedLanguageId; this._isDisposed = false; + this._maxTokenizationLineLength.set(maxTokenizationLineLength, undefined); this._resetTokenization(); } @@ -44,7 +60,10 @@ export class TextMateWorkerModel extends MirrorTextModel { this._resetTokenization(); } - private readonly tokenizeDebouncer = new RunOnceScheduler(() => this._tokenize(), 10); + private readonly tokenizeDebouncer = new RunOnceScheduler( + () => this._tokenize(), + 10 + ); override onEvents(e: IModelChangedEvent): void { super.onEvents(e); @@ -59,9 +78,19 @@ export class TextMateWorkerModel extends MirrorTextModel { this.tokenizeDebouncer.schedule(); } + public acceptMaxTokenizationLineLength( + maxTokenizationLineLength: number + ): void { + this._maxTokenizationLineLength.set(maxTokenizationLineLength, undefined); + } + public retokenize(startLineNumber: number, endLineNumberExclusive: number) { if (this._tokenizationStateStore) { - for (let lineNumber = startLineNumber; lineNumber < endLineNumberExclusive; lineNumber++) { + for ( + let lineNumber = startLineNumber; + lineNumber < endLineNumberExclusive; + lineNumber++ + ) { this._tokenizationStateStore.markMustBeTokenized(lineNumber - 1); } this.tokenizeDebouncer.schedule(); @@ -74,13 +103,25 @@ export class TextMateWorkerModel extends MirrorTextModel { const languageId = this._languageId; const encodedLanguageId = this._encodedLanguageId; this._worker.getOrCreateGrammar(languageId, encodedLanguageId).then((r) => { - if (this._isDisposed || languageId !== this._languageId || encodedLanguageId !== this._encodedLanguageId || !r) { + if ( + this._isDisposed || + languageId !== this._languageId || + encodedLanguageId !== this._encodedLanguageId || + !r + ) { return; } if (r.grammar) { - const tokenizationSupport = new TextMateTokenizationSupport(r.grammar, r.initialState, false); - this._tokenizationStateStore = new TokenizationStateStore(tokenizationSupport, tokenizationSupport.getInitialState()); + const tokenizationSupport = new TokenizationSupportWithLineLimit( + this._encodedLanguageId, + new TextMateTokenizationSupport(r.grammar, r.initialState, false), + this._maxTokenizationLineLength + ); + this._tokenizationStateStore = new TokenizationStateStore( + tokenizationSupport, + tokenizationSupport.getInitialState() + ); } else { this._tokenizationStateStore = null; } @@ -115,10 +156,26 @@ export class TextMateWorkerModel extends MirrorTextModel { const text = this._lines[lineIndex]; - const lineStartState = this._tokenizationStateStore.getBeginState(lineIndex) as StateStack; - const tokenizeResult = this._tokenizationStateStore.tokenizationSupport.tokenizeEncoded(text, true, lineStartState); - if (this._tokenizationStateStore.setEndState(lineCount, lineIndex, tokenizeResult.endState)) { - const delta = diffStateStacksRefEq(lineStartState, tokenizeResult.endState as StateStack); + const lineStartState = this._tokenizationStateStore.getBeginState( + lineIndex + ) as StateStack; + const tokenizeResult = + this._tokenizationStateStore.tokenizationSupport.tokenizeEncoded( + text, + true, + lineStartState + ); + if ( + this._tokenizationStateStore.setEndState( + lineCount, + lineIndex, + tokenizeResult.endState + ) + ) { + const delta = diffStateStacksRefEq( + lineStartState, + tokenizeResult.endState as StateStack + ); stateDeltaBuilder.setState(lineIndex + 1, delta); } @@ -137,7 +194,12 @@ export class TextMateWorkerModel extends MirrorTextModel { } const stateDeltas = stateDeltaBuilder.getStateDeltas(); - this._worker.setTokensAndStates(this._uri, this._versionId, builder.serialize(), stateDeltas); + this._worker.setTokensAndStates( + this._uri, + this._versionId, + builder.serialize(), + stateDeltas + ); const deltaMs = new Date().getTime() - startTime; if (deltaMs > 20) { diff --git a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts index 712514353dc..32258ba540c 100644 --- a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts +++ b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts @@ -6,6 +6,7 @@ import { BugIndicatingError } from 'vs/base/common/errors'; import { DisposableStore, IDisposable, toDisposable } from 'vs/base/common/lifecycle'; import { AppResourcePath, FileAccess, nodeModulesAsarPath, nodeModulesPath } from 'vs/base/common/network'; +import { IObservable } from 'vs/base/common/observable'; import { isWeb } from 'vs/base/common/platform'; import { URI, UriComponents } from 'vs/base/common/uri'; import { createWebWorker, MonacoWebWorker } from 'vs/editor/browser/services/webWorker'; @@ -122,7 +123,7 @@ export class TextMateWorkerHost implements IDisposable { } // Will be recreated when worker is killed (because tokenizer is re-registered when languages change) - public createBackgroundTokenizer(textModel: ITextModel, tokenStore: IBackgroundTokenizationStore): IBackgroundTokenizer | undefined { + public createBackgroundTokenizer(textModel: ITextModel, tokenStore: IBackgroundTokenizationStore, maxTokenizationLineLength: IObservable): IBackgroundTokenizer | undefined { if (this._workerTokenizerControllers.has(textModel.uri.toString())) { throw new BugIndicatingError(); } @@ -144,7 +145,7 @@ export class TextMateWorkerHost implements IDisposable { } store.add(keepAliveWhenAttached(textModel, () => { - const controller = new TextMateWorkerTokenizerController(textModel, workerProxy, this._languageService.languageIdCodec, tokenStore, INITIAL, this._configurationService); + const controller = new TextMateWorkerTokenizerController(textModel, workerProxy, this._languageService.languageIdCodec, tokenStore, INITIAL, this._configurationService, maxTokenizationLineLength); this._workerTokenizerControllers.set(textModel.uri.toString(), controller); return toDisposable(() => { diff --git a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts index 9732969fd90..b7eae3dd6d1 100644 --- a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts +++ b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { Disposable } from 'vs/base/common/lifecycle'; -import { IObservable, keepAlive, observableFromEvent } from 'vs/base/common/observable'; +import { IObservable, autorun, keepAlive, observableFromEvent } from 'vs/base/common/observable'; import { countEOL } from 'vs/editor/common/core/eolCounter'; import { LineRange } from 'vs/editor/common/core/lineRange'; import { Range } from 'vs/editor/common/core/range'; @@ -37,6 +37,7 @@ export class TextMateWorkerTokenizerController extends Disposable { private readonly _backgroundTokenizationStore: IBackgroundTokenizationStore, private readonly _initialState: StateStack, private readonly _configurationService: IConfigurationService, + private readonly _maxTokenizationLineLength: IObservable, ) { super(); @@ -73,7 +74,13 @@ export class TextMateWorkerTokenizerController extends Disposable { EOL: this._model.getEOL(), languageId, encodedLanguageId, + maxTokenizationLineLength: this._maxTokenizationLineLength.get(), }); + + this._register(autorun('update maxTokenizationLineLength', reader => { + const maxTokenizationLineLength = this._maxTokenizationLineLength.read(reader); + this._worker.acceptMaxTokenizationLineLength(this._model.uri.toString(), maxTokenizationLineLength); + })); } get shouldLog() {