Move lineRange to editor/common/core and adds editor.experimental.asyncTokenizationLogging (#176408)

* Move lineRange to editor/common/core and adds editor.experimental.asyncTokenizationLogging

* Fixes CI

build/monaco/monaco.d.ts.recipe

@@ -74,7 +74,8 @@ export interface ICommandHandler {
 #includeAll(vs/editor/common/model): IScrollEvent
 #include(vs/editor/common/diff/smartLinesDiffComputer): IChange, ICharChange, ILineChange
 #include(vs/editor/common/diff/documentDiffProvider): IDocumentDiffProvider, IDocumentDiffProviderOptions, IDocumentDiff
-#include(vs/editor/common/diff/linesDiffComputer): LineRangeMapping, LineRange, RangeMapping
+#include(vs/editor/common/core/lineRange): LineRange
+#include(vs/editor/common/diff/linesDiffComputer): LineRangeMapping, RangeMapping
 #include(vs/editor/common/core/dimension): IDimension
 #includeAll(vs/editor/common/editorCommon): IScrollEvent
 #includeAll(vs/editor/common/textModelEvents):

src/vs/editor/browser/services/editorWorkerService.ts

@@ -27,7 +27,8 @@ import { IEditorWorkerHost } from 'vs/editor/common/services/editorWorkerHost';
 import { ILanguageFeaturesService } from 'vs/editor/common/services/languageFeatures';
 import { IChange } from 'vs/editor/common/diff/smartLinesDiffComputer';
 import { IDocumentDiff, IDocumentDiffProviderOptions } from 'vs/editor/common/diff/documentDiffProvider';
-import { LineRangeMapping, LineRange, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 
 /**
  * Stop syncing a model to the worker if it was not needed for 1 min.

src/vs/editor/common/config/editorConfigurationSchema.ts

@@ -101,6 +101,11 @@ const editorConfiguration: IConfigurationNode = {
 			description: nls.localize('editor.experimental.asyncTokenization', "Controls whether the tokenization should happen asynchronously on a web worker."),
 			tags: ['experimental'],
 		},
+		'editor.experimental.asyncTokenizationLogging': {
+			type: 'boolean',
+			default: false,
+			description: nls.localize('editor.experimental.asyncTokenizationLogging', "Controls whether async tokenization should be logged. For debugging only."),
+		},
 		'editor.language.brackets': {
 			type: ['array', 'null'],
 			default: null, // We want to distinguish the empty array from not configured.

src/vs/editor/common/core/lineRange.ts

@@ -0,0 +1,62 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+/**
+ * A range of lines (1-based).
+ */
+export class LineRange {
+	/**
+	 * The start line number.
+	 */
+	public readonly startLineNumber: number;
+
+	/**
+	 * The end line number (exclusive).
+	 */
+	public readonly endLineNumberExclusive: number;
+
+	constructor(
+		startLineNumber: number,
+		endLineNumberExclusive: number,
+	) {
+		this.startLineNumber = startLineNumber;
+		this.endLineNumberExclusive = endLineNumberExclusive;
+	}
+
+	/**
+	 * Indicates if this line range is empty.
+	 */
+	get isEmpty(): boolean {
+		return this.startLineNumber === this.endLineNumberExclusive;
+	}
+
+	/**
+	 * Moves this line range by the given offset of line numbers.
+	 */
+	public delta(offset: number): LineRange {
+		return new LineRange(this.startLineNumber + offset, this.endLineNumberExclusive + offset);
+	}
+
+	/**
+	 * The number of lines this line range spans.
+	 */
+	public get length(): number {
+		return this.endLineNumberExclusive - this.startLineNumber;
+	}
+
+	/**
+	 * Creates a line range that combines this and the given line range.
+	 */
+	public join(other: LineRange): LineRange {
+		return new LineRange(
+			Math.min(this.startLineNumber, other.startLineNumber),
+			Math.max(this.endLineNumberExclusive, other.endLineNumberExclusive)
+		);
+	}
+
+	public toString(): string {
+		return `[${this.startLineNumber},${this.endLineNumberExclusive})`;
+	}
+}

src/vs/editor/common/diff/linesDiffComputer.ts

@@ -3,6 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { LineRange } from 'vs/editor/common/core/lineRange';
 import { Range } from 'vs/editor/common/core/range';
 
 export interface ILinesDiffComputer {
@@ -83,61 +84,3 @@ export class RangeMapping {
 		return `{${this.originalRange.toString()}->${this.modifiedRange.toString()}}`;
 	}
 }
-
-/**
- * A range of lines (1-based).
- */
-export class LineRange {
-	/**
-	 * The start line number.
-	 */
-	public readonly startLineNumber: number;
-
-	/**
-	 * The end line number (exclusive).
-	 */
-	public readonly endLineNumberExclusive: number;
-
-	constructor(
-		startLineNumber: number,
-		endLineNumberExclusive: number,
-	) {
-		this.startLineNumber = startLineNumber;
-		this.endLineNumberExclusive = endLineNumberExclusive;
-	}
-
-	/**
-	 * Indicates if this line range is empty.
-	 */
-	get isEmpty(): boolean {
-		return this.startLineNumber === this.endLineNumberExclusive;
-	}
-
-	/**
-	 * Moves this line range by the given offset of line numbers.
-	 */
-	public delta(offset: number): LineRange {
-		return new LineRange(this.startLineNumber + offset, this.endLineNumberExclusive + offset);
-	}
-
-	/**
-	 * The number of lines this line range spans.
-	 */
-	public get length(): number {
-		return this.endLineNumberExclusive - this.startLineNumber;
-	}
-
-	/**
-	 * Creates a line range that combines this and the given line range.
-	 */
-	public join(other: LineRange): LineRange {
-		return new LineRange(
-			Math.min(this.startLineNumber, other.startLineNumber),
-			Math.max(this.endLineNumberExclusive, other.endLineNumberExclusive)
-		);
-	}
-
-	public toString(): string {
-		return `[${this.startLineNumber},${this.endLineNumberExclusive})`;
-	}
-}

src/vs/editor/common/diff/smartLinesDiffComputer.ts

@@ -5,10 +5,11 @@
 
 import { CharCode } from 'vs/base/common/charCode';
 import { IDiffChange, ISequence, LcsDiff, IDiffResult } from 'vs/base/common/diff/diff';
-import { ILinesDiffComputer, ILinesDiff, ILinesDiffComputerOptions, LineRange, RangeMapping, LineRangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { ILinesDiffComputer, ILinesDiff, ILinesDiffComputerOptions, RangeMapping, LineRangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
 import * as strings from 'vs/base/common/strings';
 import { Range } from 'vs/editor/common/core/range';
 import { assertFn, checkAdjacentItems } from 'vs/base/common/assert';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 
 const MINIMUM_MATCHING_CHARACTER_LENGTH = 3;
 

src/vs/editor/common/diff/standardLinesDiffComputer.ts

@@ -5,13 +5,14 @@
 
 import { assertFn, checkAdjacentItems } from 'vs/base/common/assert';
 import { CharCode } from 'vs/base/common/charCode';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 import { Position } from 'vs/editor/common/core/position';
 import { Range } from 'vs/editor/common/core/range';
 import { OffsetRange, SequenceDiff, ISequence } from 'vs/editor/common/diff/algorithms/diffAlgorithm';
 import { DynamicProgrammingDiffing } from 'vs/editor/common/diff/algorithms/dynamicProgrammingDiffing';
 import { optimizeSequenceDiffs, smoothenSequenceDiffs } from 'vs/editor/common/diff/algorithms/joinSequenceDiffs';
 import { MyersDiffAlgorithm } from 'vs/editor/common/diff/algorithms/myersDiffAlgorithm';
-import { ILinesDiff, ILinesDiffComputer, ILinesDiffComputerOptions, LineRange, LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { ILinesDiff, ILinesDiffComputer, ILinesDiffComputerOptions, LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
 
 export class StandardLinesDiffComputer implements ILinesDiffComputer {
 	private readonly dynamicProgrammingDiffing = new DynamicProgrammingDiffing();

src/vs/editor/common/tokens/contiguousMultilineTokens.ts

@@ -9,12 +9,12 @@ import { Position } from 'vs/editor/common/core/position';
 import { IRange } from 'vs/editor/common/core/range';
 import { countEOL } from 'vs/editor/common/core/eolCounter';
 import { ContiguousTokensEditing } from 'vs/editor/common/tokens/contiguousTokensEditing';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 
 /**
  * Represents contiguous tokens over a contiguous range of lines.
  */
 export class ContiguousMultilineTokens {
-
 	public static deserialize(buff: Uint8Array, offset: number, result: ContiguousMultilineTokens[]): number {
 		const view32 = new Uint32Array(buff.buffer);
 		const startLineNumber = readUInt32BE(buff, offset); offset += 4;
@@ -64,6 +64,10 @@ export class ContiguousMultilineTokens {
 		this._tokens = tokens;
 	}
 
+	getLineRange(): LineRange {
+		return new LineRange(this._startLineNumber, this._startLineNumber + this._tokens.length);
+	}
+
 	/**
 	 * @see {@link _tokens}
 	 */

src/vs/editor/standalone/browser/standaloneEditor.ts

@@ -34,7 +34,8 @@ import { EditorCommand, ServicesAccessor } from 'vs/editor/browser/editorExtensi
 import { IMenuItem, MenuRegistry, MenuId } from 'vs/platform/actions/common/actions';
 import { ContextKeyExpr } from 'vs/platform/contextkey/common/contextkey';
 import { PLAINTEXT_LANGUAGE_ID } from 'vs/editor/common/languages/modesRegistry';
-import { LineRange, LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { LineRangeMapping, RangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { LineRange } from 'vs/editor/common/core/lineRange';
 
 /**
  * Create a new editor under `domElement`.

src/vs/monaco.d.ts

@@ -2294,30 +2294,6 @@ declare namespace monaco.editor {
 		 */
 		readonly changes: LineRangeMapping[];
 	}
-
-	/**
-	 * Maps a line range in the original text model to a line range in the modified text model.
-	 */
-	export class LineRangeMapping {
-		/**
-		 * The line range in the original text model.
-		 */
-		readonly originalRange: LineRange;
-		/**
-		 * The line range in the modified text model.
-		 */
-		readonly modifiedRange: LineRange;
-		/**
-		 * If inner changes have not been computed, this is set to undefined.
-		 * Otherwise, it represents the character-level diff in this line range.
-		 * The original range of each range mapping should be contained in the original line range (same for modified).
-		 * Must not be an empty array.
-		 */
-		readonly innerChanges: RangeMapping[] | undefined;
-		constructor(originalRange: LineRange, modifiedRange: LineRange, innerChanges: RangeMapping[] | undefined);
-		toString(): string;
-	}
-
 	/**
 	 * A range of lines (1-based).
 	 */
@@ -2350,6 +2326,29 @@ declare namespace monaco.editor {
 		toString(): string;
 	}
 
+	/**
+	 * Maps a line range in the original text model to a line range in the modified text model.
+	 */
+	export class LineRangeMapping {
+		/**
+		 * The line range in the original text model.
+		 */
+		readonly originalRange: LineRange;
+		/**
+		 * The line range in the modified text model.
+		 */
+		readonly modifiedRange: LineRange;
+		/**
+		 * If inner changes have not been computed, this is set to undefined.
+		 * Otherwise, it represents the character-level diff in this line range.
+		 * The original range of each range mapping should be contained in the original line range (same for modified).
+		 * Must not be an empty array.
+		 */
+		readonly innerChanges: RangeMapping[] | undefined;
+		constructor(originalRange: LineRange, modifiedRange: LineRange, innerChanges: RangeMapping[] | undefined);
+		toString(): string;
+	}
+
 	/**
 	 * Maps a range in the original text model to a range in the modified text model.
 	 */

src/vs/workbench/contrib/mergeEditor/browser/model/diffComputer.ts

@@ -5,13 +5,14 @@
 
 import { assertFn, checkAdjacentItems } from 'vs/base/common/assert';
 import { IReader } from 'vs/base/common/observable';
-import { LineRange as DiffLineRange, RangeMapping as DiffRangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
+import { RangeMapping as DiffRangeMapping } from 'vs/editor/common/diff/linesDiffComputer';
 import { ITextModel } from 'vs/editor/common/model';
 import { IEditorWorkerService } from 'vs/editor/common/services/editorWorker';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { LineRange } from 'vs/workbench/contrib/mergeEditor/browser/model/lineRange';
 import { DetailedLineRangeMapping, RangeMapping } from 'vs/workbench/contrib/mergeEditor/browser/model/mapping';
 import { observableConfigValue } from 'vs/workbench/contrib/mergeEditor/browser/utils';
+import { LineRange as DiffLineRange } from 'vs/editor/common/core/lineRange';
 
 export interface IMergeDiffComputer {
 	computeDiff(textModel1: ITextModel, textModel2: ITextModel, reader: IReader): Promise<IMergeDiffComputerResult>;

src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts

@@ -144,7 +144,7 @@ export class TextMateWorkerHost implements IDisposable {
 			}
 
 			store.add(keepAliveWhenAttached(textModel, () => {
-				const controller = new TextMateWorkerTokenizerController(textModel, workerProxy, this._languageService.languageIdCodec, tokenStore, INITIAL);
+				const controller = new TextMateWorkerTokenizerController(textModel, workerProxy, this._languageService.languageIdCodec, tokenStore, INITIAL, this._configurationService);
 				this._workerTokenizerControllers.set(textModel.uri.toString(), controller);
 
 				return toDisposable(() => {

src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts

@@ -4,12 +4,16 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { Disposable } from 'vs/base/common/lifecycle';
+import { IObservable, keepAlive, observableFromEvent } from 'vs/base/common/observable';
 import { countEOL } from 'vs/editor/common/core/eolCounter';
+import { LineRange } from 'vs/editor/common/core/lineRange';
+import { Range } from 'vs/editor/common/core/range';
 import { IBackgroundTokenizationStore, ILanguageIdCodec } from 'vs/editor/common/languages';
 import { ITextModel } from 'vs/editor/common/model';
 import { ContiguousGrowingArray } from 'vs/editor/common/model/textModelTokens';
 import { IModelContentChange, IModelContentChangedEvent } from 'vs/editor/common/textModelEvents';
 import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/tokens/contiguousMultilineTokensBuilder';
+import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { ArrayEdit, MonotonousIndexTransformer, SingleArrayEdit } from 'vs/workbench/services/textMate/browser/arrayOperation';
 import { TextMateTokenizationWorker } from 'vs/workbench/services/textMate/browser/worker/textMate.worker';
 import type { StateDeltas } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost';
@@ -24,16 +28,27 @@ export class TextMateWorkerTokenizerController extends Disposable {
 	 */
 	private readonly _states = new ContiguousGrowingArray<StateStack | null>(null);
 
+	private readonly _loggingEnabled = observableConfigValue('editor.experimental.asyncTokenizationLogging', false, this._configurationService);
+
 	constructor(
 		private readonly _model: ITextModel,
 		private readonly _worker: TextMateTokenizationWorker,
 		private readonly _languageIdCodec: ILanguageIdCodec,
 		private readonly _backgroundTokenizationStore: IBackgroundTokenizationStore,
 		private readonly _initialState: StateStack,
+		private readonly _configurationService: IConfigurationService,
 	) {
 		super();
 
+		this._register(keepAlive(this._loggingEnabled));
+
 		this._register(this._model.onDidChangeContent((e) => {
+			if (this.shouldLog) {
+				console.log('model change', {
+					fileName: this._model.uri.fsPath.split('\\').pop(),
+					changes: changesToString(e.changes),
+				});
+			}
 			this._worker.acceptModelChanged(this._model.uri.toString(), e);
 			this._pendingChanges.push(e);
 		}));
@@ -61,6 +76,10 @@ export class TextMateWorkerTokenizerController extends Disposable {
 		});
 	}
 
+	get shouldLog() {
+		return this._loggingEnabled.get();
+	}
+
 	public override dispose(): void {
 		super.dispose();
 		this._worker.acceptRemovedModel(this._model.uri.toString());
@@ -74,6 +93,23 @@ export class TextMateWorkerTokenizerController extends Disposable {
 		//                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^                                ^^^^^^^^^^^^^^^^^^^^^^^^^
 		//                | past changes                                                   | future states
 
+		let tokens = ContiguousMultilineTokensBuilder.deserialize(
+			new Uint8Array(rawTokens)
+		);
+
+		if (this.shouldLog) {
+			console.log('received background tokenization result', {
+				fileName: this._model.uri.fsPath.split('\\').pop(),
+				updatedTokenLines: tokens.map((t) => t.getLineRange()).join(' & '),
+				updatedStateLines: stateDeltas.map((s) => new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length).toString()).join(' & '),
+			});
+		}
+
+		if (this.shouldLog) {
+			const changes = this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes).map(c => changesToString(c)).join(' then ');
+			console.log('Applying changes to local states', changes);
+		}
+
 		// Apply past changes to _states
 		while (
 			this._pendingChanges.length > 0 &&
@@ -84,11 +120,12 @@ export class TextMateWorkerTokenizerController extends Disposable {
 			op.applyTo(this._states);
 		}
 
-		let tokens = ContiguousMultilineTokensBuilder.deserialize(
-			new Uint8Array(rawTokens)
-		);
-
 		if (this._pendingChanges.length > 0) {
+			if (this.shouldLog) {
+				const changes = this._pendingChanges.map(c => c.changes).map(c => changesToString(c)).join(' then ');
+				console.log('Considering non-processed changes', changes);
+			}
+
 			const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(
 				this._pendingChanges.map((c) => new ArrayEdit(
 					c.changes.map(
@@ -169,3 +206,18 @@ function lineArrayEditFromModelContentChange(c: IModelContentChange[]): ArrayEdi
 		)
 	);
 }
+
+function changesToString(changes: IModelContentChange[]): string {
+	return changes.map(c => Range.lift(c.range).toString() + ' => ' + c.text).join(' & ');
+}
+
+function observableConfigValue<T>(key: string, defaultValue: T, configurationService: IConfigurationService): IObservable<T> {
+	return observableFromEvent(
+		(handleChange) => configurationService.onDidChangeConfiguration(e => {
+			if (e.affectsConfiguration(key)) {
+				handleChange(e);
+			}
+		}),
+		() => configurationService.getValue<T>(key) ?? defaultValue,
+	);
+}