diff --git a/build/monaco/monaco.d.ts.recipe b/build/monaco/monaco.d.ts.recipe index 2e36edaec68..56dd80e19ad 100644 --- a/build/monaco/monaco.d.ts.recipe +++ b/build/monaco/monaco.d.ts.recipe @@ -74,7 +74,8 @@ export interface ICommandHandler { #includeAll(vs/editor/common/model): IScrollEvent #include(vs/editor/common/diff/smartLinesDiffComputer): IChange, ICharChange, ILineChange #include(vs/editor/common/diff/documentDiffProvider): IDocumentDiffProvider, IDocumentDiffProviderOptions, IDocumentDiff -#include(vs/editor/common/diff/linesDiffComputer): LineRangeMapping, LineRange, RangeMapping +#include(vs/editor/common/core/lineRange): LineRange +#include(vs/editor/common/diff/linesDiffComputer): LineRangeMapping, RangeMapping #include(vs/editor/common/core/dimension): IDimension #includeAll(vs/editor/common/editorCommon): IScrollEvent #includeAll(vs/editor/common/textModelEvents): diff --git a/src/vs/editor/browser/services/editorWorkerService.ts b/src/vs/editor/browser/services/editorWorkerService.ts index 398bdbebcc1..467ead7130c 100644 --- a/src/vs/editor/browser/services/editorWorkerService.ts +++ b/src/vs/editor/browser/services/editorWorkerService.ts @@ -27,7 +27,8 @@ import { IEditorWorkerHost } from 'vs/editor/common/services/editorWorkerHost'; import { ILanguageFeaturesService } from 'vs/editor/common/services/languageFeatures'; import { IChange } from 'vs/editor/common/diff/smartLinesDiffComputer'; import { IDocumentDiff, IDocumentDiffProviderOptions } from 'vs/editor/common/diff/documentDiffProvider'; -import { LineRangeMapping, LineRange, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer'; +import { LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer'; +import { LineRange } from 'vs/editor/common/core/lineRange'; /** * Stop syncing a model to the worker if it was not needed for 1 min. diff --git a/src/vs/editor/common/config/editorConfigurationSchema.ts b/src/vs/editor/common/config/editorConfigurationSchema.ts index a2c3d886017..a400b4015f5 100644 --- a/src/vs/editor/common/config/editorConfigurationSchema.ts +++ b/src/vs/editor/common/config/editorConfigurationSchema.ts @@ -101,6 +101,11 @@ const editorConfiguration: IConfigurationNode = { description: nls.localize('editor.experimental.asyncTokenization', "Controls whether the tokenization should happen asynchronously on a web worker."), tags: ['experimental'], }, + 'editor.experimental.asyncTokenizationLogging': { + type: 'boolean', + default: false, + description: nls.localize('editor.experimental.asyncTokenizationLogging', "Controls whether async tokenization should be logged. For debugging only."), + }, 'editor.language.brackets': { type: ['array', 'null'], default: null, // We want to distinguish the empty array from not configured. diff --git a/src/vs/editor/common/core/lineRange.ts b/src/vs/editor/common/core/lineRange.ts new file mode 100644 index 00000000000..45d12eedb0c --- /dev/null +++ b/src/vs/editor/common/core/lineRange.ts @@ -0,0 +1,62 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * A range of lines (1-based). + */ +export class LineRange { + /** + * The start line number. + */ + public readonly startLineNumber: number; + + /** + * The end line number (exclusive). + */ + public readonly endLineNumberExclusive: number; + + constructor( + startLineNumber: number, + endLineNumberExclusive: number, + ) { + this.startLineNumber = startLineNumber; + this.endLineNumberExclusive = endLineNumberExclusive; + } + + /** + * Indicates if this line range is empty. + */ + get isEmpty(): boolean { + return this.startLineNumber === this.endLineNumberExclusive; + } + + /** + * Moves this line range by the given offset of line numbers. + */ + public delta(offset: number): LineRange { + return new LineRange(this.startLineNumber + offset, this.endLineNumberExclusive + offset); + } + + /** + * The number of lines this line range spans. + */ + public get length(): number { + return this.endLineNumberExclusive - this.startLineNumber; + } + + /** + * Creates a line range that combines this and the given line range. + */ + public join(other: LineRange): LineRange { + return new LineRange( + Math.min(this.startLineNumber, other.startLineNumber), + Math.max(this.endLineNumberExclusive, other.endLineNumberExclusive) + ); + } + + public toString(): string { + return `[${this.startLineNumber},${this.endLineNumberExclusive})`; + } +} diff --git a/src/vs/editor/common/diff/linesDiffComputer.ts b/src/vs/editor/common/diff/linesDiffComputer.ts index 9a9961f77cd..32f3ceb7b8a 100644 --- a/src/vs/editor/common/diff/linesDiffComputer.ts +++ b/src/vs/editor/common/diff/linesDiffComputer.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { LineRange } from 'vs/editor/common/core/lineRange'; import { Range } from 'vs/editor/common/core/range'; export interface ILinesDiffComputer { @@ -83,61 +84,3 @@ export class RangeMapping { return `{${this.originalRange.toString()}->${this.modifiedRange.toString()}}`; } } - -/** - * A range of lines (1-based). - */ -export class LineRange { - /** - * The start line number. - */ - public readonly startLineNumber: number; - - /** - * The end line number (exclusive). - */ - public readonly endLineNumberExclusive: number; - - constructor( - startLineNumber: number, - endLineNumberExclusive: number, - ) { - this.startLineNumber = startLineNumber; - this.endLineNumberExclusive = endLineNumberExclusive; - } - - /** - * Indicates if this line range is empty. - */ - get isEmpty(): boolean { - return this.startLineNumber === this.endLineNumberExclusive; - } - - /** - * Moves this line range by the given offset of line numbers. - */ - public delta(offset: number): LineRange { - return new LineRange(this.startLineNumber + offset, this.endLineNumberExclusive + offset); - } - - /** - * The number of lines this line range spans. - */ - public get length(): number { - return this.endLineNumberExclusive - this.startLineNumber; - } - - /** - * Creates a line range that combines this and the given line range. - */ - public join(other: LineRange): LineRange { - return new LineRange( - Math.min(this.startLineNumber, other.startLineNumber), - Math.max(this.endLineNumberExclusive, other.endLineNumberExclusive) - ); - } - - public toString(): string { - return `[${this.startLineNumber},${this.endLineNumberExclusive})`; - } -} diff --git a/src/vs/editor/common/diff/smartLinesDiffComputer.ts b/src/vs/editor/common/diff/smartLinesDiffComputer.ts index 9c514f6a2d6..d6f6412c63a 100644 --- a/src/vs/editor/common/diff/smartLinesDiffComputer.ts +++ b/src/vs/editor/common/diff/smartLinesDiffComputer.ts @@ -5,10 +5,11 @@ import { CharCode } from 'vs/base/common/charCode'; import { IDiffChange, ISequence, LcsDiff, IDiffResult } from 'vs/base/common/diff/diff'; -import { ILinesDiffComputer, ILinesDiff, ILinesDiffComputerOptions, LineRange, RangeMapping, LineRangeMapping } from 'vs/editor/common/diff/linesDiffComputer'; +import { ILinesDiffComputer, ILinesDiff, ILinesDiffComputerOptions, RangeMapping, LineRangeMapping } from 'vs/editor/common/diff/linesDiffComputer'; import * as strings from 'vs/base/common/strings'; import { Range } from 'vs/editor/common/core/range'; import { assertFn, checkAdjacentItems } from 'vs/base/common/assert'; +import { LineRange } from 'vs/editor/common/core/lineRange'; const MINIMUM_MATCHING_CHARACTER_LENGTH = 3; diff --git a/src/vs/editor/common/diff/standardLinesDiffComputer.ts b/src/vs/editor/common/diff/standardLinesDiffComputer.ts index de289902022..1de7e1270ec 100644 --- a/src/vs/editor/common/diff/standardLinesDiffComputer.ts +++ b/src/vs/editor/common/diff/standardLinesDiffComputer.ts @@ -5,13 +5,14 @@ import { assertFn, checkAdjacentItems } from 'vs/base/common/assert'; import { CharCode } from 'vs/base/common/charCode'; +import { LineRange } from 'vs/editor/common/core/lineRange'; import { Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; import { OffsetRange, SequenceDiff, ISequence } from 'vs/editor/common/diff/algorithms/diffAlgorithm'; import { DynamicProgrammingDiffing } from 'vs/editor/common/diff/algorithms/dynamicProgrammingDiffing'; import { optimizeSequenceDiffs, smoothenSequenceDiffs } from 'vs/editor/common/diff/algorithms/joinSequenceDiffs'; import { MyersDiffAlgorithm } from 'vs/editor/common/diff/algorithms/myersDiffAlgorithm'; -import { ILinesDiff, ILinesDiffComputer, ILinesDiffComputerOptions, LineRange, LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer'; +import { ILinesDiff, ILinesDiffComputer, ILinesDiffComputerOptions, LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer'; export class StandardLinesDiffComputer implements ILinesDiffComputer { private readonly dynamicProgrammingDiffing = new DynamicProgrammingDiffing(); diff --git a/src/vs/editor/common/tokens/contiguousMultilineTokens.ts b/src/vs/editor/common/tokens/contiguousMultilineTokens.ts index a8860866aef..f4d267fba0a 100644 --- a/src/vs/editor/common/tokens/contiguousMultilineTokens.ts +++ b/src/vs/editor/common/tokens/contiguousMultilineTokens.ts @@ -9,12 +9,12 @@ import { Position } from 'vs/editor/common/core/position'; import { IRange } from 'vs/editor/common/core/range'; import { countEOL } from 'vs/editor/common/core/eolCounter'; import { ContiguousTokensEditing } from 'vs/editor/common/tokens/contiguousTokensEditing'; +import { LineRange } from 'vs/editor/common/core/lineRange'; /** * Represents contiguous tokens over a contiguous range of lines. */ export class ContiguousMultilineTokens { - public static deserialize(buff: Uint8Array, offset: number, result: ContiguousMultilineTokens[]): number { const view32 = new Uint32Array(buff.buffer); const startLineNumber = readUInt32BE(buff, offset); offset += 4; @@ -64,6 +64,10 @@ export class ContiguousMultilineTokens { this._tokens = tokens; } + getLineRange(): LineRange { + return new LineRange(this._startLineNumber, this._startLineNumber + this._tokens.length); + } + /** * @see {@link _tokens} */ diff --git a/src/vs/editor/standalone/browser/standaloneEditor.ts b/src/vs/editor/standalone/browser/standaloneEditor.ts index 6027a2ac7a5..f6e258e70f3 100644 --- a/src/vs/editor/standalone/browser/standaloneEditor.ts +++ b/src/vs/editor/standalone/browser/standaloneEditor.ts @@ -34,7 +34,8 @@ import { EditorCommand, ServicesAccessor } from 'vs/editor/browser/editorExtensi import { IMenuItem, MenuRegistry, MenuId } from 'vs/platform/actions/common/actions'; import { ContextKeyExpr } from 'vs/platform/contextkey/common/contextkey'; import { PLAINTEXT_LANGUAGE_ID } from 'vs/editor/common/languages/modesRegistry'; -import { LineRange, LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer'; +import { LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer'; +import { LineRange } from 'vs/editor/common/core/lineRange'; /** * Create a new editor under `domElement`. diff --git a/src/vs/monaco.d.ts b/src/vs/monaco.d.ts index ee659b50e67..b33c014eea1 100644 --- a/src/vs/monaco.d.ts +++ b/src/vs/monaco.d.ts @@ -2294,30 +2294,6 @@ declare namespace monaco.editor { */ readonly changes: LineRangeMapping[]; } - - /** - * Maps a line range in the original text model to a line range in the modified text model. - */ - export class LineRangeMapping { - /** - * The line range in the original text model. - */ - readonly originalRange: LineRange; - /** - * The line range in the modified text model. - */ - readonly modifiedRange: LineRange; - /** - * If inner changes have not been computed, this is set to undefined. - * Otherwise, it represents the character-level diff in this line range. - * The original range of each range mapping should be contained in the original line range (same for modified). - * Must not be an empty array. - */ - readonly innerChanges: RangeMapping[] | undefined; - constructor(originalRange: LineRange, modifiedRange: LineRange, innerChanges: RangeMapping[] | undefined); - toString(): string; - } - /** * A range of lines (1-based). */ @@ -2350,6 +2326,29 @@ declare namespace monaco.editor { toString(): string; } + /** + * Maps a line range in the original text model to a line range in the modified text model. + */ + export class LineRangeMapping { + /** + * The line range in the original text model. + */ + readonly originalRange: LineRange; + /** + * The line range in the modified text model. + */ + readonly modifiedRange: LineRange; + /** + * If inner changes have not been computed, this is set to undefined. + * Otherwise, it represents the character-level diff in this line range. + * The original range of each range mapping should be contained in the original line range (same for modified). + * Must not be an empty array. + */ + readonly innerChanges: RangeMapping[] | undefined; + constructor(originalRange: LineRange, modifiedRange: LineRange, innerChanges: RangeMapping[] | undefined); + toString(): string; + } + /** * Maps a range in the original text model to a range in the modified text model. */ diff --git a/src/vs/workbench/contrib/mergeEditor/browser/model/diffComputer.ts b/src/vs/workbench/contrib/mergeEditor/browser/model/diffComputer.ts index 59adb965242..7673c2a5b5b 100644 --- a/src/vs/workbench/contrib/mergeEditor/browser/model/diffComputer.ts +++ b/src/vs/workbench/contrib/mergeEditor/browser/model/diffComputer.ts @@ -5,13 +5,14 @@ import { assertFn, checkAdjacentItems } from 'vs/base/common/assert'; import { IReader } from 'vs/base/common/observable'; -import { LineRange as DiffLineRange, RangeMapping as DiffRangeMapping } from 'vs/editor/common/diff/linesDiffComputer'; +import { RangeMapping as DiffRangeMapping } from 'vs/editor/common/diff/linesDiffComputer'; import { ITextModel } from 'vs/editor/common/model'; import { IEditorWorkerService } from 'vs/editor/common/services/editorWorker'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { LineRange } from 'vs/workbench/contrib/mergeEditor/browser/model/lineRange'; import { DetailedLineRangeMapping, RangeMapping } from 'vs/workbench/contrib/mergeEditor/browser/model/mapping'; import { observableConfigValue } from 'vs/workbench/contrib/mergeEditor/browser/utils'; +import { LineRange as DiffLineRange } from 'vs/editor/common/core/lineRange'; export interface IMergeDiffComputer { computeDiff(textModel1: ITextModel, textModel2: ITextModel, reader: IReader): Promise; diff --git a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts index 16547b5314f..712514353dc 100644 --- a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts +++ b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts @@ -144,7 +144,7 @@ export class TextMateWorkerHost implements IDisposable { } store.add(keepAliveWhenAttached(textModel, () => { - const controller = new TextMateWorkerTokenizerController(textModel, workerProxy, this._languageService.languageIdCodec, tokenStore, INITIAL); + const controller = new TextMateWorkerTokenizerController(textModel, workerProxy, this._languageService.languageIdCodec, tokenStore, INITIAL, this._configurationService); this._workerTokenizerControllers.set(textModel.uri.toString(), controller); return toDisposable(() => { diff --git a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts index 2cbd515f969..9732969fd90 100644 --- a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts +++ b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts @@ -4,12 +4,16 @@ *--------------------------------------------------------------------------------------------*/ import { Disposable } from 'vs/base/common/lifecycle'; +import { IObservable, keepAlive, observableFromEvent } from 'vs/base/common/observable'; import { countEOL } from 'vs/editor/common/core/eolCounter'; +import { LineRange } from 'vs/editor/common/core/lineRange'; +import { Range } from 'vs/editor/common/core/range'; import { IBackgroundTokenizationStore, ILanguageIdCodec } from 'vs/editor/common/languages'; import { ITextModel } from 'vs/editor/common/model'; import { ContiguousGrowingArray } from 'vs/editor/common/model/textModelTokens'; import { IModelContentChange, IModelContentChangedEvent } from 'vs/editor/common/textModelEvents'; import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/tokens/contiguousMultilineTokensBuilder'; +import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { ArrayEdit, MonotonousIndexTransformer, SingleArrayEdit } from 'vs/workbench/services/textMate/browser/arrayOperation'; import { TextMateTokenizationWorker } from 'vs/workbench/services/textMate/browser/worker/textMate.worker'; import type { StateDeltas } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost'; @@ -24,16 +28,27 @@ export class TextMateWorkerTokenizerController extends Disposable { */ private readonly _states = new ContiguousGrowingArray(null); + private readonly _loggingEnabled = observableConfigValue('editor.experimental.asyncTokenizationLogging', false, this._configurationService); + constructor( private readonly _model: ITextModel, private readonly _worker: TextMateTokenizationWorker, private readonly _languageIdCodec: ILanguageIdCodec, private readonly _backgroundTokenizationStore: IBackgroundTokenizationStore, private readonly _initialState: StateStack, + private readonly _configurationService: IConfigurationService, ) { super(); + this._register(keepAlive(this._loggingEnabled)); + this._register(this._model.onDidChangeContent((e) => { + if (this.shouldLog) { + console.log('model change', { + fileName: this._model.uri.fsPath.split('\\').pop(), + changes: changesToString(e.changes), + }); + } this._worker.acceptModelChanged(this._model.uri.toString(), e); this._pendingChanges.push(e); })); @@ -61,6 +76,10 @@ export class TextMateWorkerTokenizerController extends Disposable { }); } + get shouldLog() { + return this._loggingEnabled.get(); + } + public override dispose(): void { super.dispose(); this._worker.acceptRemovedModel(this._model.uri.toString()); @@ -74,6 +93,23 @@ export class TextMateWorkerTokenizerController extends Disposable { // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^ // | past changes | future states + let tokens = ContiguousMultilineTokensBuilder.deserialize( + new Uint8Array(rawTokens) + ); + + if (this.shouldLog) { + console.log('received background tokenization result', { + fileName: this._model.uri.fsPath.split('\\').pop(), + updatedTokenLines: tokens.map((t) => t.getLineRange()).join(' & '), + updatedStateLines: stateDeltas.map((s) => new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length).toString()).join(' & '), + }); + } + + if (this.shouldLog) { + const changes = this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes).map(c => changesToString(c)).join(' then '); + console.log('Applying changes to local states', changes); + } + // Apply past changes to _states while ( this._pendingChanges.length > 0 && @@ -84,11 +120,12 @@ export class TextMateWorkerTokenizerController extends Disposable { op.applyTo(this._states); } - let tokens = ContiguousMultilineTokensBuilder.deserialize( - new Uint8Array(rawTokens) - ); - if (this._pendingChanges.length > 0) { + if (this.shouldLog) { + const changes = this._pendingChanges.map(c => c.changes).map(c => changesToString(c)).join(' then '); + console.log('Considering non-processed changes', changes); + } + const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany( this._pendingChanges.map((c) => new ArrayEdit( c.changes.map( @@ -169,3 +206,18 @@ function lineArrayEditFromModelContentChange(c: IModelContentChange[]): ArrayEdi ) ); } + +function changesToString(changes: IModelContentChange[]): string { + return changes.map(c => Range.lift(c.range).toString() + ' => ' + c.text).join(' & '); +} + +function observableConfigValue(key: string, defaultValue: T, configurationService: IConfigurationService): IObservable { + return observableFromEvent( + (handleChange) => configurationService.onDidChangeConfiguration(e => { + if (e.affectsConfiguration(key)) { + handleChange(e); + } + }), + () => configurationService.getValue(key) ?? defaultValue, + ); +}