From 79e3994280ff914844114eb9a28a725a4f16780a Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Tue, 16 Dec 2025 12:44:11 +0100 Subject: [PATCH] Adding font info in colorTokenCustomization (#263403) Adding font info in colorTokenCustomization --- package-lock.json | 11 +- package.json | 2 +- remote/package-lock.json | 8 +- remote/package.json | 2 +- remote/web/package-lock.json | 8 +- remote/web/package.json | 2 +- src/vs/editor/common/languages.ts | 15 + .../editor/common/languages/nullTokenize.ts | 2 +- .../common/languages/supports/tokenization.ts | 31 ++ .../editor/common/model/decorationProvider.ts | 29 +- src/vs/editor/common/model/textModel.ts | 54 +- .../tokens/abstractSyntaxTokenBackend.ts | 6 +- .../editor/common/model/tokens/annotations.ts | 281 ++++++++++ .../tokenizationFontDecorationsProvider.ts | 162 ++++++ .../model/tokens/tokenizationTextModelPart.ts | 14 +- .../tokens/tokenizerSyntaxTokenBackend.ts | 12 +- src/vs/editor/common/textModelEvents.ts | 58 ++ .../test/browser/lineCommentCommand.test.ts | 2 +- .../test/browser/indentation.test.ts | 2 +- .../suggest/test/browser/suggestModel.test.ts | 2 +- .../standalone/browser/standaloneLanguages.ts | 4 +- .../browser/standaloneThemeService.ts | 6 +- .../standalone/common/monarch/monarchLexer.ts | 1 + .../test/browser/standaloneLanguages.test.ts | 4 +- .../trimTrailingWhitespaceCommand.test.ts | 6 +- .../test/browser/controller/cursor.test.ts | 6 +- .../viewModel/modelLineProjection.test.ts | 2 +- .../test/common/model/annotations.test.ts | 500 ++++++++++++++++++ .../bracketPairColorizer/tokenizer.test.ts | 2 +- .../test/common/model/model.line.test.ts | 2 +- .../test/common/model/model.modes.test.ts | 4 +- src/vs/editor/test/common/model/model.test.ts | 2 +- .../common/model/textModelWithTokens.test.ts | 16 +- .../common/modes/textToHtmlTokenizer.test.ts | 2 +- src/vs/platform/theme/common/themeService.ts | 11 + .../theme/test/common/testThemeService.ts | 6 +- .../codeEditor/test/node/autoindent.test.ts | 2 +- .../test/common/terminalColorRegistry.test.ts | 1 + .../textMateWorkerTokenizerController.ts | 24 +- .../threadedBackgroundTokenizerFactory.ts | 6 +- .../textMateTokenizationWorker.worker.ts | 6 +- .../worker/textMateWorkerHost.ts | 4 +- .../worker/textMateWorkerTokenizer.ts | 41 +- .../textMateTokenizationFeatureImpl.ts | 14 +- .../textMateTokenizationSupport.ts | 4 +- .../services/themes/common/colorThemeData.ts | 70 ++- .../themes/common/colorThemeSchema.ts | 12 + .../themes/common/workbenchThemeService.ts | 3 + 48 files changed, 1369 insertions(+), 95 deletions(-) create mode 100644 src/vs/editor/common/model/tokens/annotations.ts create mode 100644 src/vs/editor/common/model/tokens/tokenizationFontDecorationsProvider.ts create mode 100644 src/vs/editor/test/common/model/annotations.test.ts diff --git a/package-lock.json b/package-lock.json index b1ae7c40d80..57ff81d1f3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,7 +53,7 @@ "v8-inspect-profiler": "^0.1.1", "vscode-oniguruma": "1.7.0", "vscode-regexpp": "^3.1.0", - "vscode-textmate": "^9.2.1", + "vscode-textmate": "^9.3.0", "yauzl": "^3.0.0", "yazl": "^2.4.3" }, @@ -11174,8 +11174,9 @@ "node_modules/json-stringify-safe": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", - "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", "dev": true, + "license": "ISC", "optional": true }, "node_modules/json5": { @@ -17722,9 +17723,9 @@ } }, "node_modules/vscode-textmate": { - "version": "9.2.1", - "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.2.1.tgz", - "integrity": "sha512-eXiUi2yYFv9bdvgrYtJynA7UemCEkpVNE50S9iBBA08LYG5t9+/TB+8IRS/YoYOubCez2OkSyZ1Q12eQMwzbrw==", + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.3.0.tgz", + "integrity": "sha512-zHiZZOdb9xqj5/X1C4a29sbgT2HngdWxPLSl3PyHRQF+5visI4uNM020OHiLJjsMxUssyk/pGVAg/9LCIobrVg==", "license": "MIT" }, "node_modules/vscode-uri": { diff --git a/package.json b/package.json index 9aa3be3e55b..2bed48859e0 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,7 @@ "v8-inspect-profiler": "^0.1.1", "vscode-oniguruma": "1.7.0", "vscode-regexpp": "^3.1.0", - "vscode-textmate": "^9.2.1", + "vscode-textmate": "^9.3.0", "yauzl": "^3.0.0", "yazl": "^2.4.3" }, diff --git a/remote/package-lock.json b/remote/package-lock.json index 95ef4ffb526..444daba9c9a 100644 --- a/remote/package-lock.json +++ b/remote/package-lock.json @@ -42,7 +42,7 @@ "tas-client": "0.3.1", "vscode-oniguruma": "1.7.0", "vscode-regexpp": "^3.1.0", - "vscode-textmate": "^9.2.1", + "vscode-textmate": "^9.3.0", "yauzl": "^3.0.0", "yazl": "^2.4.3" } @@ -1215,9 +1215,9 @@ } }, "node_modules/vscode-textmate": { - "version": "9.2.1", - "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.2.1.tgz", - "integrity": "sha512-eXiUi2yYFv9bdvgrYtJynA7UemCEkpVNE50S9iBBA08LYG5t9+/TB+8IRS/YoYOubCez2OkSyZ1Q12eQMwzbrw==", + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.3.0.tgz", + "integrity": "sha512-zHiZZOdb9xqj5/X1C4a29sbgT2HngdWxPLSl3PyHRQF+5visI4uNM020OHiLJjsMxUssyk/pGVAg/9LCIobrVg==", "license": "MIT" }, "node_modules/wrappy": { diff --git a/remote/package.json b/remote/package.json index 1ef9407ef29..ca94e175246 100644 --- a/remote/package.json +++ b/remote/package.json @@ -37,7 +37,7 @@ "tas-client": "0.3.1", "vscode-oniguruma": "1.7.0", "vscode-regexpp": "^3.1.0", - "vscode-textmate": "^9.2.1", + "vscode-textmate": "^9.3.0", "yauzl": "^3.0.0", "yazl": "^2.4.3" }, diff --git a/remote/web/package-lock.json b/remote/web/package-lock.json index 192ad264db4..6dff130e803 100644 --- a/remote/web/package-lock.json +++ b/remote/web/package-lock.json @@ -26,7 +26,7 @@ "katex": "^0.16.22", "tas-client": "0.3.1", "vscode-oniguruma": "1.7.0", - "vscode-textmate": "^9.2.1" + "vscode-textmate": "^9.3.0" } }, "node_modules/@microsoft/1ds-core-js": { @@ -308,9 +308,9 @@ "integrity": "sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==" }, "node_modules/vscode-textmate": { - "version": "9.2.1", - "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.2.1.tgz", - "integrity": "sha512-eXiUi2yYFv9bdvgrYtJynA7UemCEkpVNE50S9iBBA08LYG5t9+/TB+8IRS/YoYOubCez2OkSyZ1Q12eQMwzbrw==", + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-9.3.0.tgz", + "integrity": "sha512-zHiZZOdb9xqj5/X1C4a29sbgT2HngdWxPLSl3PyHRQF+5visI4uNM020OHiLJjsMxUssyk/pGVAg/9LCIobrVg==", "license": "MIT" }, "node_modules/yallist": { diff --git a/remote/web/package.json b/remote/web/package.json index fbe537240c6..6d811ab94e4 100644 --- a/remote/web/package.json +++ b/remote/web/package.json @@ -21,6 +21,6 @@ "katex": "^0.16.22", "tas-client": "0.3.1", "vscode-oniguruma": "1.7.0", - "vscode-textmate": "^9.2.1" + "vscode-textmate": "^9.3.0" } } diff --git a/src/vs/editor/common/languages.ts b/src/vs/editor/common/languages.ts index 6ce3cce8d17..83710866127 100644 --- a/src/vs/editor/common/languages.ts +++ b/src/vs/editor/common/languages.ts @@ -27,6 +27,7 @@ import { localize } from '../../nls.js'; import { ExtensionIdentifier } from '../../platform/extensions/common/extensions.js'; import { IMarkerData } from '../../platform/markers/common/markers.js'; import { EditDeltaInfo } from './textModelEditSource.js'; +import { FontTokensUpdate } from './textModelEvents.js'; /** * @internal @@ -64,6 +65,17 @@ export class TokenizationResult { } } +/** + * @internal + */ +export interface IFontToken { + readonly startIndex: number; + readonly endIndex: number; + readonly fontFamily: string | null; + readonly fontSize: string | null; + readonly lineHeight: number | null; +} + /** * @internal */ @@ -78,6 +90,7 @@ export class EncodedTokenizationResult { * */ public readonly tokens: Uint32Array, + public readonly fontInfo: IFontToken[], public readonly endState: IState, ) { } @@ -140,6 +153,8 @@ export interface IBackgroundTokenizer extends IDisposable { export interface IBackgroundTokenizationStore { setTokens(tokens: ContiguousMultilineTokens[]): void; + setFontInfo(changes: FontTokensUpdate): void; + setEndState(lineNumber: number, state: IState): void; /** diff --git a/src/vs/editor/common/languages/nullTokenize.ts b/src/vs/editor/common/languages/nullTokenize.ts index 8966ab8b734..2ed15d199fe 100644 --- a/src/vs/editor/common/languages/nullTokenize.ts +++ b/src/vs/editor/common/languages/nullTokenize.ts @@ -30,5 +30,5 @@ export function nullTokenizeEncoded(languageId: LanguageId, state: IState | null | (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET) ) >>> 0; - return new EncodedTokenizationResult(tokens, state === null ? NullState : state); + return new EncodedTokenizationResult(tokens, [], state === null ? NullState : state); } diff --git a/src/vs/editor/common/languages/supports/tokenization.ts b/src/vs/editor/common/languages/supports/tokenization.ts index f6322a09dda..076b443f58f 100644 --- a/src/vs/editor/common/languages/supports/tokenization.ts +++ b/src/vs/editor/common/languages/supports/tokenization.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { Color } from '../../../../base/common/color.js'; +import { IFontTokenOptions } from '../../../../platform/theme/common/themeService.js'; import { LanguageId, FontStyle, ColorId, StandardTokenType, MetadataConsts } from '../../encodedTokenAttributes.js'; export interface ITokenThemeRule { @@ -422,3 +423,33 @@ export function generateTokensCSSForColorMap(colorMap: readonly Color[]): string rules.push('.mtks.mtku { text-decoration: underline line-through; text-underline-position: under; }'); return rules.join('\n'); } + +export function generateTokensCSSForFontMap(fontMap: readonly IFontTokenOptions[]): string { + const rules: string[] = []; + const fonts = new Set(); + for (let i = 1, len = fontMap.length; i < len; i++) { + const font = fontMap[i]; + if (!font.fontFamily && !font.fontSize) { + continue; + } + const className = classNameForFontTokenDecorations(font.fontFamily ?? '', font.fontSize ?? ''); + if (fonts.has(className)) { + continue; + } + fonts.add(className); + let rule = `.${className} {`; + if (font.fontFamily) { + rule += `font-family: ${font.fontFamily};`; + } + if (font.fontSize) { + rule += `font-size: ${font.fontSize};`; + } + rule += `}`; + rules.push(rule); + } + return rules.join('\n'); +} + +export function classNameForFontTokenDecorations(fontFamily: string, fontSize: string): string { + return `font-decoration-${fontFamily.toLowerCase()}-${fontSize.toLowerCase()}`; +} diff --git a/src/vs/editor/common/model/decorationProvider.ts b/src/vs/editor/common/model/decorationProvider.ts index e3c146831de..e8154b7277b 100644 --- a/src/vs/editor/common/model/decorationProvider.ts +++ b/src/vs/editor/common/model/decorationProvider.ts @@ -3,7 +3,6 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { Event } from '../../../base/common/event.js'; import { Range } from '../core/range.js'; import { IModelDecoration } from '../model.js'; @@ -25,5 +24,31 @@ export interface DecorationProvider { */ getAllDecorations(ownerId?: number, filterOutValidation?: boolean, onlyMinimapDecorations?: boolean): IModelDecoration[]; - readonly onDidChange: Event; +} + +export class LineHeightChangingDecoration { + + public static toKey(obj: LineHeightChangingDecoration): string { + return `${obj.ownerId};${obj.decorationId};${obj.lineNumber}`; + } + + constructor( + public readonly ownerId: number, + public readonly decorationId: string, + public readonly lineNumber: number, + public readonly lineHeight: number | null + ) { } +} + +export class LineFontChangingDecoration { + + public static toKey(obj: LineFontChangingDecoration): string { + return `${obj.ownerId};${obj.decorationId};${obj.lineNumber}`; + } + + constructor( + public readonly ownerId: number, + public readonly decorationId: string, + public readonly lineNumber: number + ) { } } diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index e1324152ea7..8cb4f567ba3 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -51,6 +51,8 @@ import { PieceTreeTextBuffer } from './pieceTreeTextBuffer/pieceTreeTextBuffer.j import { PieceTreeTextBufferBuilder } from './pieceTreeTextBuffer/pieceTreeTextBufferBuilder.js'; import { SearchParams, TextModelSearch } from './textModelSearch.js'; import { AttachedViews } from './tokens/abstractSyntaxTokenBackend.js'; +import { TokenizationFontDecorationProvider } from './tokens/tokenizationFontDecorationsProvider.js'; +import { LineFontChangingDecoration, LineHeightChangingDecoration } from './decorationProvider.js'; import { TokenizationTextModelPart } from './tokens/tokenizationTextModelPart.js'; export function createTextBufferFactory(text: string): model.ITextBufferFactory { @@ -292,6 +294,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati private _decorations: { [decorationId: string]: IntervalNode }; private _decorationsTree: DecorationsTrees; private readonly _decorationProvider: ColorizedBracketPairsDecorationProvider; + private readonly _fontTokenDecorationsProvider: TokenizationFontDecorationProvider; //#endregion private readonly _tokenizationTextModelPart: TokenizationTextModelPart; @@ -366,6 +369,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati languageId, this._attachedViews ); + this._fontTokenDecorationsProvider = this._register(new TokenizationFontDecorationProvider(this, this._tokenizationTextModelPart)); this._isTooLargeForSyncing = (bufferTextLength > TextModel._MODEL_SYNC_LIMIT); @@ -392,6 +396,18 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati this._onDidChangeDecorations.fire(); this._onDidChangeDecorations.endDeferredEmit(); })); + this._register(this._fontTokenDecorationsProvider.onDidChangeLineHeight((affectedLineHeights) => { + this._onDidChangeDecorations.beginDeferredEmit(); + this._onDidChangeDecorations.fire(); + this._fireOnDidChangeLineHeight(affectedLineHeights); + this._onDidChangeDecorations.endDeferredEmit(); + })); + this._register(this._fontTokenDecorationsProvider.onDidChangeFont((affectedFontLines) => { + this._onDidChangeDecorations.beginDeferredEmit(); + this._onDidChangeDecorations.fire(); + this._fireOnDidChangeFont(affectedFontLines); + this._onDidChangeDecorations.endDeferredEmit(); + })); this._languageService.requestRichLanguageFeatures(languageId); @@ -454,6 +470,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati } this._tokenizationTextModelPart.handleDidChangeContent(change); this._bracketPairs.handleDidChangeContent(change); + this._fontTokenDecorationsProvider.handleDidChangeContent(change); this._eventEmitter.fire(new InternalModelContentChangeEvent(rawChange, change)); } @@ -1630,11 +1647,19 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati const lineChangeEvents = affectedLines.map(lineNumber => new ModelRawLineChanged(lineNumber, this.getLineContent(lineNumber), this._getInjectedTextInLine(lineNumber))); this._onDidChangeInjectedText.fire(new ModelInjectedTextChangedEvent(lineChangeEvents)); } + this._fireOnDidChangeLineHeight(affectedLineHeights); + this._fireOnDidChangeFont(affectedFontLines); + } + + private _fireOnDidChangeLineHeight(affectedLineHeights: Set | null): void { if (affectedLineHeights && affectedLineHeights.size > 0) { const affectedLines = Array.from(affectedLineHeights); const lineHeightChangeEvent = affectedLines.map(specialLineHeightChange => new ModelLineHeightChanged(specialLineHeightChange.ownerId, specialLineHeightChange.decorationId, specialLineHeightChange.lineNumber, specialLineHeightChange.lineHeight)); this._onDidChangeLineHeight.fire(new ModelLineHeightChangedEvent(lineHeightChangeEvent)); } + } + + private _fireOnDidChangeFont(affectedFontLines: Set | null): void { if (affectedFontLines && affectedFontLines.size > 0) { const affectedLines = Array.from(affectedFontLines); const fontChangeEvent = affectedLines.map(fontChange => new ModelFontChanged(fontChange.ownerId, fontChange.lineNumber)); @@ -1795,6 +1820,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati const decorations = this._getDecorationsInRange(range, ownerId, filterOutValidation, filterFontDecorations, onlyMarginDecorations); pushMany(decorations, this._decorationProvider.getDecorationsInRange(range, ownerId, filterOutValidation)); + pushMany(decorations, this._fontTokenDecorationsProvider.getDecorationsInRange(range, ownerId, filterOutValidation)); return decorations; } @@ -1803,6 +1829,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati const decorations = this._getDecorationsInRange(validatedRange, ownerId, filterOutValidation, filterFontDecorations, onlyMarginDecorations); pushMany(decorations, this._decorationProvider.getDecorationsInRange(validatedRange, ownerId, filterOutValidation, onlyMinimapDecorations)); + pushMany(decorations, this._fontTokenDecorationsProvider.getDecorationsInRange(validatedRange, ownerId, filterOutValidation, onlyMinimapDecorations)); return decorations; } @@ -1835,6 +1862,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati public getAllDecorations(ownerId: number = 0, filterOutValidation: boolean = false, filterFontDecorations: boolean = false): model.IModelDecoration[] { let result = this._decorationsTree.getAll(this, ownerId, filterOutValidation, filterFontDecorations, false, false); result = result.concat(this._decorationProvider.getAllDecorations(ownerId, filterOutValidation)); + result = result.concat(this._fontTokenDecorationsProvider.getAllDecorations(ownerId, filterOutValidation)); return result; } @@ -2510,32 +2538,6 @@ function _normalizeOptions(options: model.IModelDecorationOptions): ModelDecorat return ModelDecorationOptions.createDynamic(options); } -class LineHeightChangingDecoration { - - public static toKey(obj: LineHeightChangingDecoration): string { - return `${obj.ownerId};${obj.decorationId};${obj.lineNumber}`; - } - - constructor( - public readonly ownerId: number, - public readonly decorationId: string, - public readonly lineNumber: number, - public readonly lineHeight: number | null - ) { } -} - -class LineFontChangingDecoration { - - public static toKey(obj: LineFontChangingDecoration): string { - return `${obj.ownerId};${obj.decorationId};${obj.lineNumber}`; - } - - constructor( - public readonly ownerId: number, - public readonly decorationId: string, - public readonly lineNumber: number - ) { } -} class DidChangeDecorationsEmitter extends Disposable { diff --git a/src/vs/editor/common/model/tokens/abstractSyntaxTokenBackend.ts b/src/vs/editor/common/model/tokens/abstractSyntaxTokenBackend.ts index 778f8d89eb9..2daa1d88fc6 100644 --- a/src/vs/editor/common/model/tokens/abstractSyntaxTokenBackend.ts +++ b/src/vs/editor/common/model/tokens/abstractSyntaxTokenBackend.ts @@ -12,7 +12,7 @@ import { StandardTokenType } from '../../encodedTokenAttributes.js'; import { ILanguageIdCodec } from '../../languages.js'; import { IAttachedView } from '../../model.js'; import { TextModel } from '../textModel.js'; -import { IModelContentChangedEvent, IModelTokensChangedEvent } from '../../textModelEvents.js'; +import { IModelContentChangedEvent, IModelTokensChangedEvent, IModelFontTokensChangedEvent } from '../../textModelEvents.js'; import { BackgroundTokenizationState } from '../../tokenizationTextModelPart.js'; import { LineTokens } from '../../tokens/lineTokens.js'; import { derivedOpts, IObservable, ISettableObservable, observableSignal, observableValueOpts } from '../../../../base/common/observable.js'; @@ -145,6 +145,10 @@ export abstract class AbstractSyntaxTokenBackend extends Disposable { /** @internal, should not be exposed by the text model! */ public readonly onDidChangeTokens: Event = this._onDidChangeTokens.event; + protected readonly _onDidChangeFontTokens: Emitter = this._register(new Emitter()); + /** @internal, should not be exposed by the text model! */ + public readonly onDidChangeFontTokens: Event = this._onDidChangeFontTokens.event; + constructor( protected readonly _languageIdCodec: ILanguageIdCodec, protected readonly _textModel: TextModel, diff --git a/src/vs/editor/common/model/tokens/annotations.ts b/src/vs/editor/common/model/tokens/annotations.ts new file mode 100644 index 00000000000..cd763868801 --- /dev/null +++ b/src/vs/editor/common/model/tokens/annotations.ts @@ -0,0 +1,281 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { binarySearch2 } from '../../../../base/common/arrays.js'; +import { StringEdit } from '../../core/edits/stringEdit.js'; +import { OffsetRange } from '../../core/ranges/offsetRange.js'; + +export interface IAnnotation { + range: OffsetRange; + annotation: T; +} + +export interface IAnnotatedString { + /** + * Set annotations for a specific line. + * Annotations should be sorted and non-overlapping. + */ + setAnnotations(annotations: AnnotationsUpdate): void; + /** + * Return annotations intersecting with the given offset range. + */ + getAnnotationsIntersecting(range: OffsetRange): IAnnotation[]; + /** + * Get all the annotations. Method is used for testing. + */ + getAllAnnotations(): IAnnotation[]; + /** + * Apply a string edit to the annotated string. + * @returns The annotations that were deleted (became empty) as a result of the edit. + */ + applyEdit(edit: StringEdit): IAnnotation[]; + /** + * Clone the annotated string. + */ + clone(): IAnnotatedString; +} + +export class AnnotatedString implements IAnnotatedString { + + /** + * Annotations are non intersecting and contiguous in the array. + */ + private _annotations: IAnnotation[] = []; + + constructor(annotations: IAnnotation[] = []) { + this._annotations = annotations; + } + + /** + * Set annotations for a specific range. + * Annotations should be sorted and non-overlapping. + * If the annotation value is undefined, the annotation is removed. + */ + public setAnnotations(annotations: AnnotationsUpdate): void { + for (const annotation of annotations.annotations) { + const startIndex = this._getStartIndexOfIntersectingAnnotation(annotation.range.start); + const endIndexExclusive = this._getEndIndexOfIntersectingAnnotation(annotation.range.endExclusive); + if (annotation.annotation !== undefined) { + this._annotations.splice(startIndex, endIndexExclusive - startIndex, { range: annotation.range, annotation: annotation.annotation }); + } else { + this._annotations.splice(startIndex, endIndexExclusive - startIndex); + } + } + } + + /** + * Returns all annotations that intersect with the given offset range. + */ + public getAnnotationsIntersecting(range: OffsetRange): IAnnotation[] { + const startIndex = this._getStartIndexOfIntersectingAnnotation(range.start); + const endIndexExclusive = this._getEndIndexOfIntersectingAnnotation(range.endExclusive); + return this._annotations.slice(startIndex, endIndexExclusive); + } + + private _getStartIndexOfIntersectingAnnotation(offset: number): number { + // Find index to the left of the offset + const startIndexWhereToReplace = binarySearch2(this._annotations.length, (index) => { + return this._annotations[index].range.start - offset; + }); + let startIndex: number; + if (startIndexWhereToReplace >= 0) { + startIndex = startIndexWhereToReplace; + } else { + const candidate = this._annotations[- (startIndexWhereToReplace + 2)]?.range; + if (candidate && offset >= candidate.start && offset <= candidate.endExclusive) { + startIndex = - (startIndexWhereToReplace + 2); + } else { + startIndex = - (startIndexWhereToReplace + 1); + } + } + return startIndex; + } + + private _getEndIndexOfIntersectingAnnotation(offset: number): number { + // Find index to the right of the offset + const endIndexWhereToReplace = binarySearch2(this._annotations.length, (index) => { + return this._annotations[index].range.endExclusive - offset; + }); + let endIndexExclusive: number; + if (endIndexWhereToReplace >= 0) { + endIndexExclusive = endIndexWhereToReplace + 1; + } else { + const candidate = this._annotations[-(endIndexWhereToReplace + 1)]?.range; + if (candidate && offset >= candidate.start && offset <= candidate.endExclusive) { + endIndexExclusive = - endIndexWhereToReplace; + } else { + endIndexExclusive = - (endIndexWhereToReplace + 1); + } + } + return endIndexExclusive; + } + + /** + * Returns a copy of all annotations. + */ + public getAllAnnotations(): IAnnotation[] { + return this._annotations.slice(); + } + + /** + * Applies a string edit to the annotated string, updating annotation ranges accordingly. + * @param edit The string edit to apply. + * @returns The annotations that were deleted (became empty) as a result of the edit. + */ + public applyEdit(edit: StringEdit): IAnnotation[] { + const annotations = this._annotations.slice(); + + // treat edits as deletion of the replace range and then as insertion that extends the first range + const finalAnnotations: IAnnotation[] = []; + const deletedAnnotations: IAnnotation[] = []; + + let offset = 0; + + for (const e of edit.replacements) { + while (true) { + // ranges before the current edit + const annotation = annotations[0]; + if (!annotation) { + break; + } + const range = annotation.range; + if (range.endExclusive >= e.replaceRange.start) { + break; + } + annotations.shift(); + const newAnnotation = { range: range.delta(offset), annotation: annotation.annotation }; + if (!newAnnotation.range.isEmpty) { + finalAnnotations.push(newAnnotation); + } else { + deletedAnnotations.push(newAnnotation); + } + } + + const intersecting: IAnnotation[] = []; + while (true) { + const annotation = annotations[0]; + if (!annotation) { + break; + } + const range = annotation.range; + if (!range.intersectsOrTouches(e.replaceRange)) { + break; + } + annotations.shift(); + intersecting.push(annotation); + } + + for (let i = intersecting.length - 1; i >= 0; i--) { + const annotation = intersecting[i]; + let r = annotation.range; + + // Inserted text will extend the first intersecting annotation, if the edit truly overlaps it + const shouldExtend = i === 0 && (e.replaceRange.endExclusive > r.start) && (e.replaceRange.start < r.endExclusive); + // Annotation shrinks by the overlap then grows with the new text length + const overlap = r.intersect(e.replaceRange)!.length; + r = r.deltaEnd(-overlap + (shouldExtend ? e.newText.length : 0)); + + // If the annotation starts after the edit start, shift left to the edit start position + const rangeAheadOfReplaceRange = r.start - e.replaceRange.start; + if (rangeAheadOfReplaceRange > 0) { + r = r.delta(-rangeAheadOfReplaceRange); + } + + // If annotation shouldn't be extended AND it is after or on edit start, move it after the newly inserted text + if (!shouldExtend && rangeAheadOfReplaceRange >= 0) { + r = r.delta(e.newText.length); + } + + // We already took our offset into account. + // Because we add r back to the queue (which then adds offset again), + // we have to remove it here so as to not double count it. + r = r.delta(-(e.newText.length - e.replaceRange.length)); + + annotations.unshift({ annotation: annotation.annotation, range: r }); + } + + offset += e.newText.length - e.replaceRange.length; + } + + while (true) { + const annotation = annotations[0]; + if (!annotation) { + break; + } + annotations.shift(); + const newAnnotation = { annotation: annotation.annotation, range: annotation.range.delta(offset) }; + if (!newAnnotation.range.isEmpty) { + finalAnnotations.push(newAnnotation); + } else { + deletedAnnotations.push(newAnnotation); + } + } + this._annotations = finalAnnotations; + return deletedAnnotations; + } + + /** + * Creates a shallow clone of this annotated string. + */ + public clone(): IAnnotatedString { + return new AnnotatedString(this._annotations.slice()); + } +} + +export interface IAnnotationUpdate { + range: OffsetRange; + annotation: T | undefined; +} + +type DefinedValue = object | string | number | boolean; + +export type ISerializedAnnotation = { + range: { start: number; endExclusive: number }; + annotation: TSerializedProperty | undefined; +}; + +export class AnnotationsUpdate { + + public static create(annotations: IAnnotationUpdate[]): AnnotationsUpdate { + return new AnnotationsUpdate(annotations); + } + + private _annotations: IAnnotationUpdate[]; + + private constructor(annotations: IAnnotationUpdate[]) { + this._annotations = annotations; + } + + get annotations(): IAnnotationUpdate[] { + return this._annotations; + } + + public rebase(edit: StringEdit): void { + const annotatedString = new AnnotatedString(this._annotations); + annotatedString.applyEdit(edit); + this._annotations = annotatedString.getAllAnnotations(); + } + + public serialize(serializingFunc: (annotation: T) => TSerializedProperty): ISerializedAnnotation[] { + return this._annotations.map(annotation => { + const range = { start: annotation.range.start, endExclusive: annotation.range.endExclusive }; + if (!annotation.annotation) { + return { range, annotation: undefined }; + } + return { range, annotation: serializingFunc(annotation.annotation) }; + }); + } + + static deserialize(serializedAnnotations: ISerializedAnnotation[], deserializingFunc: (annotation: TSerializedProperty) => T): AnnotationsUpdate { + const annotations: IAnnotationUpdate[] = serializedAnnotations.map(serializedAnnotation => { + const range = new OffsetRange(serializedAnnotation.range.start, serializedAnnotation.range.endExclusive); + if (!serializedAnnotation.annotation) { + return { range, annotation: undefined }; + } + return { range, annotation: deserializingFunc(serializedAnnotation.annotation) }; + }); + return new AnnotationsUpdate(annotations); + } +} diff --git a/src/vs/editor/common/model/tokens/tokenizationFontDecorationsProvider.ts b/src/vs/editor/common/model/tokens/tokenizationFontDecorationsProvider.ts new file mode 100644 index 00000000000..ccf4b297be3 --- /dev/null +++ b/src/vs/editor/common/model/tokens/tokenizationFontDecorationsProvider.ts @@ -0,0 +1,162 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Disposable } from '../../../../base/common/lifecycle.js'; +import { IModelDecoration, ITextModel } from '../../model.js'; +import { TokenizationTextModelPart } from './tokenizationTextModelPart.js'; +import { Range } from '../../core/range.js'; +import { DecorationProvider, LineFontChangingDecoration, LineHeightChangingDecoration } from '../decorationProvider.js'; +import { Emitter } from '../../../../base/common/event.js'; +import { IFontTokenOption, IModelContentChangedEvent } from '../../textModelEvents.js'; +import { classNameForFontTokenDecorations } from '../../languages/supports/tokenization.js'; +import { Position } from '../../core/position.js'; +import { AnnotatedString, AnnotationsUpdate, IAnnotatedString, IAnnotationUpdate } from './annotations.js'; +import { OffsetRange } from '../../core/ranges/offsetRange.js'; +import { offsetEditFromContentChanges } from '../textModelStringEdit.js'; + +export interface IFontTokenAnnotation { + decorationId: string; + fontToken: IFontTokenOption; +} + +export class TokenizationFontDecorationProvider extends Disposable implements DecorationProvider { + + private static DECORATION_COUNT = 0; + + private readonly _onDidChangeLineHeight = new Emitter>(); + public readonly onDidChangeLineHeight = this._onDidChangeLineHeight.event; + + private readonly _onDidChangeFont = new Emitter>(); + public readonly onDidChangeFont = this._onDidChangeFont.event; + + private _fontAnnotatedString: IAnnotatedString = new AnnotatedString(); + + constructor( + private readonly textModel: ITextModel, + private readonly tokenizationTextModelPart: TokenizationTextModelPart + ) { + super(); + this._register(this.tokenizationTextModelPart.onDidChangeFontTokens(fontChanges => { + + const linesChanged = new Set(); + const fontTokenAnnotations: IAnnotationUpdate[] = []; + + const affectedLineHeights = new Set(); + const affectedLineFonts = new Set(); + + for (const annotation of fontChanges.changes.annotations) { + + const startPosition = this.textModel.getPositionAt(annotation.range.start); + const endPosition = this.textModel.getPositionAt(annotation.range.endExclusive); + + if (startPosition.lineNumber !== endPosition.lineNumber) { + // The token should be always on a single line + continue; + } + const lineNumber = startPosition.lineNumber; + + let fontTokenAnnotation: IAnnotationUpdate; + if (annotation.annotation === undefined) { + fontTokenAnnotation = { + range: annotation.range, + annotation: undefined + }; + } else { + const decorationId = `tokenization-font-decoration-${TokenizationFontDecorationProvider.DECORATION_COUNT}`; + const fontTokenDecoration: IFontTokenAnnotation = { + fontToken: annotation.annotation, + decorationId + }; + fontTokenAnnotation = { + range: annotation.range, + annotation: fontTokenDecoration + }; + TokenizationFontDecorationProvider.DECORATION_COUNT++; + + if (annotation.annotation.lineHeight) { + affectedLineHeights.add(new LineHeightChangingDecoration(0, decorationId, lineNumber, annotation.annotation.lineHeight)); + } + affectedLineFonts.add(new LineFontChangingDecoration(0, decorationId, lineNumber)); + + } + fontTokenAnnotations.push(fontTokenAnnotation); + + if (!linesChanged.has(lineNumber)) { + // Signal the removal of the font tokenization decorations on the line number + const lineNumberStartOffset = this.textModel.getOffsetAt(new Position(lineNumber, 1)); + const lineNumberEndOffset = this.textModel.getOffsetAt(new Position(lineNumber, this.textModel.getLineMaxColumn(lineNumber))); + const lineOffsetRange = new OffsetRange(lineNumberStartOffset, lineNumberEndOffset); + const lineAnnotations = this._fontAnnotatedString.getAnnotationsIntersecting(lineOffsetRange); + for (const annotation of lineAnnotations) { + const decorationId = annotation.annotation.decorationId; + affectedLineHeights.add(new LineHeightChangingDecoration(0, decorationId, lineNumber, null)); + affectedLineFonts.add(new LineFontChangingDecoration(0, decorationId, lineNumber)); + } + linesChanged.add(lineNumber); + } + } + this._fontAnnotatedString.setAnnotations(AnnotationsUpdate.create(fontTokenAnnotations)); + this._onDidChangeLineHeight.fire(affectedLineHeights); + this._onDidChangeFont.fire(affectedLineFonts); + })); + } + + public handleDidChangeContent(change: IModelContentChangedEvent) { + const edits = offsetEditFromContentChanges(change.changes); + const deletedAnnotations = this._fontAnnotatedString.applyEdit(edits); + if (deletedAnnotations.length === 0) { + return; + } + /* We should fire line and font change events if decorations have been added or removed + * No decorations are added on edit, but they can be removed */ + const affectedLineHeights = new Set(); + const affectedLineFonts = new Set(); + for (const deletedAnnotation of deletedAnnotations) { + const startPosition = this.textModel.getPositionAt(deletedAnnotation.range.start); + const lineNumber = startPosition.lineNumber; + const decorationId = deletedAnnotation.annotation.decorationId; + affectedLineHeights.add(new LineHeightChangingDecoration(0, decorationId, lineNumber, null)); + affectedLineFonts.add(new LineFontChangingDecoration(0, decorationId, lineNumber)); + } + this._onDidChangeLineHeight.fire(affectedLineHeights); + this._onDidChangeFont.fire(affectedLineFonts); + } + + public getDecorationsInRange(range: Range, ownerId?: number, filterOutValidation?: boolean, onlyMinimapDecorations?: boolean): IModelDecoration[] { + const startOffsetOfRange = this.textModel.getOffsetAt(range.getStartPosition()); + const endOffsetOfRange = this.textModel.getOffsetAt(range.getEndPosition()); + const annotations = this._fontAnnotatedString.getAnnotationsIntersecting(new OffsetRange(startOffsetOfRange, endOffsetOfRange)); + + const decorations: IModelDecoration[] = []; + for (const annotation of annotations) { + const annotationStartPosition = this.textModel.getPositionAt(annotation.range.start); + const annotationEndPosition = this.textModel.getPositionAt(annotation.range.endExclusive); + const range = Range.fromPositions(annotationStartPosition, annotationEndPosition); + const anno = annotation.annotation; + const className = classNameForFontTokenDecorations(anno.fontToken.fontFamily ?? '', anno.fontToken.fontSize ?? ''); + const affectsFont = !!(anno.fontToken.fontFamily || anno.fontToken.fontSize); + const id = anno.decorationId; + decorations.push({ + id: id, + options: { + description: 'FontOptionDecoration', + inlineClassName: className, + affectsFont + }, + ownerId: 0, + range + }); + } + return decorations; + } + + public getAllDecorations(ownerId?: number, filterOutValidation?: boolean): IModelDecoration[] { + return this.getDecorationsInRange( + new Range(1, 1, this.textModel.getLineCount(), 1), + ownerId, + filterOutValidation + ); + } +} diff --git a/src/vs/editor/common/model/tokens/tokenizationTextModelPart.ts b/src/vs/editor/common/model/tokens/tokenizationTextModelPart.ts index ab162dca21c..e04f159946e 100644 --- a/src/vs/editor/common/model/tokens/tokenizationTextModelPart.ts +++ b/src/vs/editor/common/model/tokens/tokenizationTextModelPart.ts @@ -18,7 +18,7 @@ import { TextModel } from '../textModel.js'; import { TextModelPart } from '../textModelPart.js'; import { AbstractSyntaxTokenBackend, AttachedViews } from './abstractSyntaxTokenBackend.js'; import { TreeSitterSyntaxTokenBackend } from './treeSitter/treeSitterSyntaxTokenBackend.js'; -import { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent } from '../../textModelEvents.js'; +import { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent, IModelFontTokensChangedEvent } from '../../textModelEvents.js'; import { ITokenizationTextModelPart } from '../../tokenizationTextModelPart.js'; import { LineTokens } from '../../tokens/lineTokens.js'; import { SparseMultilineTokens } from '../../tokens/sparseMultilineTokens.js'; @@ -40,6 +40,9 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz private readonly _onDidChangeTokens: Emitter; public readonly onDidChangeTokens: Event; + private readonly _onDidChangeFontTokens: Emitter = this._register(new Emitter()); + public readonly onDidChangeFontTokens: Event = this._onDidChangeFontTokens.event; + public readonly tokens: IObservable; private readonly _useTreeSitter: IObservable; private readonly _languageIdObs: ISettableObservable; @@ -80,6 +83,11 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz reader.store.add(tokens.onDidChangeTokens(e => { this._emitModelTokensChangedEvent(e); })); + reader.store.add(tokens.onDidChangeFontTokens(e => { + if (!this._textModel._isDisposing()) { + this._onDidChangeFontTokens.fire(e); + } + })); reader.store.add(tokens.onDidChangeBackgroundTokenizationState(e => { this._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState(); @@ -104,9 +112,13 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz this.onDidChangeLanguageConfiguration = this._onDidChangeLanguageConfiguration.event; this._onDidChangeTokens = this._register(new Emitter()); this.onDidChangeTokens = this._onDidChangeTokens.event; + this._onDidChangeFontTokens = this._register(new Emitter()); + this.onDidChangeFontTokens = this._onDidChangeFontTokens.event; } _hasListeners(): boolean { + // Note: _onDidChangeFontTokens is intentionally excluded because it's an internal event + // that TokenizationFontDecorationProvider subscribes to during TextModel construction return (this._onDidChangeLanguage.hasListeners() || this._onDidChangeLanguageConfiguration.hasListeners() || this._onDidChangeTokens.hasListeners()); diff --git a/src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts b/src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts index 004accbdbcd..176bb35fc27 100644 --- a/src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts +++ b/src/vs/editor/common/model/tokens/tokenizerSyntaxTokenBackend.ts @@ -12,7 +12,7 @@ import { LineRange } from '../../core/ranges/lineRange.js'; import { StandardTokenType } from '../../encodedTokenAttributes.js'; import { IBackgroundTokenizer, IState, ILanguageIdCodec, TokenizationRegistry, ITokenizationSupport, IBackgroundTokenizationStore } from '../../languages.js'; import { IAttachedView } from '../../model.js'; -import { IModelContentChangedEvent } from '../../textModelEvents.js'; +import { FontTokensUpdate, IModelContentChangedEvent } from '../../textModelEvents.js'; import { BackgroundTokenizationState } from '../../tokenizationTextModelPart.js'; import { ContiguousMultilineTokens } from '../../tokens/contiguousMultilineTokens.js'; import { ContiguousMultilineTokensBuilder } from '../../tokens/contiguousMultilineTokensBuilder.js'; @@ -123,6 +123,9 @@ export class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend { setTokens: (tokens) => { this.setTokens(tokens); }, + setFontInfo: (changes: FontTokensUpdate) => { + this.setFontInfo(changes); + }, backgroundTokenizationFinished: () => { if (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) { // We already did a full tokenization and don't go back to progressing. @@ -159,6 +162,9 @@ export class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend { setTokens: (tokens) => { this._debugBackgroundTokens?.setMultilineTokens(tokens, this._textModel); }, + setFontInfo: (changes: FontTokensUpdate) => { + this.setFontInfo(changes); + }, backgroundTokenizationFinished() { // NO OP }, @@ -210,6 +216,10 @@ export class TokenizerSyntaxTokenBackend extends AbstractSyntaxTokenBackend { return { changes: changes }; } + private setFontInfo(changes: FontTokensUpdate): void { + this._onDidChangeFontTokens.fire({ changes }); + } + private refreshAllVisibleLineTokens(): void { const ranges = LineRange.joinMany([...this._attachedViewStates].map(([_, s]) => s.lineRanges)); this.refreshRanges(ranges); diff --git a/src/vs/editor/common/textModelEvents.ts b/src/vs/editor/common/textModelEvents.ts index 945bb35b73b..cc142ebb8c5 100644 --- a/src/vs/editor/common/textModelEvents.ts +++ b/src/vs/editor/common/textModelEvents.ts @@ -8,6 +8,7 @@ import { IRange, Range } from './core/range.js'; import { Selection } from './core/selection.js'; import { IModelDecoration, InjectedTextOptions } from './model.js'; import { IModelContentChange } from './model/mirrorTextModel.js'; +import { AnnotationsUpdate } from './model/tokens/annotations.js'; import { TextModelEditSource } from './textModelEditSource.js'; /** @@ -150,6 +151,63 @@ export interface IModelTokensChangedEvent { }[]; } +/** + * @internal + */ +export interface IFontTokenOption { + /** + * Font family of the token. + */ + readonly fontFamily?: string; + /** + * Font size of the token. + */ + readonly fontSize?: string; + /** + * Line height of the token. + */ + readonly lineHeight?: number; +} + +/** + * An event describing a token font change event + * @internal + */ +export interface IModelFontTokensChangedEvent { + changes: FontTokensUpdate; +} + +/** + * @internal + */ +export type FontTokensUpdate = AnnotationsUpdate; + +/** + * @internal + */ +export function serializeFontTokenOptions(): (options: IFontTokenOption) => IFontTokenOption { + return (annotation: IFontTokenOption) => { + return { + fontFamily: annotation.fontFamily ?? '', + fontSize: annotation.fontSize ?? '', + lineHeight: annotation.lineHeight ?? 0 + }; + }; +} + +/** + * @internal + */ +export function deserializeFontTokenOptions(): (options: IFontTokenOption) => IFontTokenOption { + return (annotation: IFontTokenOption) => { + return { + fontFamily: annotation.fontFamily ? String(annotation.fontFamily) : undefined, + fontSize: annotation.fontSize ? String(annotation.fontSize) : undefined, + lineHeight: annotation.lineHeight ? Number(annotation.lineHeight) : undefined + }; + }; +} + export interface IModelOptionsChangedEvent { readonly tabSize: boolean; readonly indentSize: boolean; diff --git a/src/vs/editor/contrib/comment/test/browser/lineCommentCommand.test.ts b/src/vs/editor/contrib/comment/test/browser/lineCommentCommand.test.ts index 0d8bd9f4151..8c4e0b6dce2 100644 --- a/src/vs/editor/contrib/comment/test/browser/lineCommentCommand.test.ts +++ b/src/vs/editor/contrib/comment/test/browser/lineCommentCommand.test.ts @@ -1145,7 +1145,7 @@ suite('Editor Contrib - Line Comment in mixed modes', () => { (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET) | (encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) ); - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } })); } diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index 0dcfd898c47..16a300fa56f 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -132,7 +132,7 @@ export function registerTokenizationSupport(instantiationService: TestInstantiat | (tokensOnLine[i].standardTokenType << MetadataConsts.TOKEN_TYPE_OFFSET) ); } - return new EncodedTokenizationResult(result, state); + return new EncodedTokenizationResult(result, [], state); } }; return TokenizationRegistry.register(languageId, tokenizationSupport); diff --git a/src/vs/editor/contrib/suggest/test/browser/suggestModel.test.ts b/src/vs/editor/contrib/suggest/test/browser/suggestModel.test.ts index e957e5c6a74..dccb55ef441 100644 --- a/src/vs/editor/contrib/suggest/test/browser/suggestModel.test.ts +++ b/src/vs/editor/contrib/suggest/test/browser/suggestModel.test.ts @@ -110,7 +110,7 @@ suite('SuggestModel - Context', function () { for (let i = 0; i < tokens.length; i++) { tokens[i] = tokensArr[i]; } - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } })); } diff --git a/src/vs/editor/standalone/browser/standaloneLanguages.ts b/src/vs/editor/standalone/browser/standaloneLanguages.ts index 31b294e1d03..c06acad60f1 100644 --- a/src/vs/editor/standalone/browser/standaloneLanguages.ts +++ b/src/vs/editor/standalone/browser/standaloneLanguages.ts @@ -131,7 +131,7 @@ export class EncodedTokenizationSupportAdapter implements languages.ITokenizatio public tokenizeEncoded(line: string, hasEOL: boolean, state: languages.IState): languages.EncodedTokenizationResult { const result = this._actual.tokenizeEncoded(line, state); - return new languages.EncodedTokenizationResult(result.tokens, result.endState); + return new languages.EncodedTokenizationResult(result.tokens, [], result.endState); } } @@ -249,7 +249,7 @@ export class TokenizationSupportAdapter implements languages.ITokenizationSuppor endState = actualResult.endState; } - return new languages.EncodedTokenizationResult(tokens, endState); + return new languages.EncodedTokenizationResult(tokens, [], endState); } } diff --git a/src/vs/editor/standalone/browser/standaloneThemeService.ts b/src/vs/editor/standalone/browser/standaloneThemeService.ts index 0ef9470da84..67fe9f8420d 100644 --- a/src/vs/editor/standalone/browser/standaloneThemeService.ts +++ b/src/vs/editor/standalone/browser/standaloneThemeService.ts @@ -16,7 +16,7 @@ import { hc_black, hc_light, vs, vs_dark } from '../common/themes.js'; import { IEnvironmentService } from '../../../platform/environment/common/environment.js'; import { Registry } from '../../../platform/registry/common/platform.js'; import { asCssVariableName, ColorIdentifier, Extensions, IColorRegistry } from '../../../platform/theme/common/colorRegistry.js'; -import { Extensions as ThemingExtensions, ICssStyleCollector, IFileIconTheme, IProductIconTheme, IThemingRegistry, ITokenStyle } from '../../../platform/theme/common/themeService.js'; +import { Extensions as ThemingExtensions, ICssStyleCollector, IFileIconTheme, IProductIconTheme, IThemingRegistry, ITokenStyle, IFontTokenOptions } from '../../../platform/theme/common/themeService.js'; import { IDisposable, Disposable } from '../../../base/common/lifecycle.js'; import { ColorScheme, isDark, isHighContrast } from '../../../platform/theme/common/theme.js'; import { getIconsStyleSheet, UnthemedProductIconTheme } from '../../../platform/theme/browser/iconsStyleSheet.js'; @@ -179,6 +179,10 @@ class StandaloneTheme implements IStandaloneTheme { return []; } + public get tokenFontMap(): IFontTokenOptions[] { + return []; + } + public readonly semanticHighlighting = false; } diff --git a/src/vs/editor/standalone/common/monarch/monarchLexer.ts b/src/vs/editor/standalone/common/monarch/monarchLexer.ts index 9e82b00116b..bb68a158bdf 100644 --- a/src/vs/editor/standalone/common/monarch/monarchLexer.ts +++ b/src/vs/editor/standalone/common/monarch/monarchLexer.ts @@ -380,6 +380,7 @@ class MonarchModernTokensCollector implements IMonarchTokensCollector { public finalize(endState: MonarchLineState): languages.EncodedTokenizationResult { return new languages.EncodedTokenizationResult( MonarchModernTokensCollector._merge(this._prependTokens, this._tokens, null), + [], endState ); } diff --git a/src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts b/src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts index 0fa82cf782b..92943556aaa 100644 --- a/src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts +++ b/src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts @@ -74,7 +74,9 @@ suite('TokenizationSupport2Adapter', () => { semanticHighlighting: false, - tokenColorMap: [] + tokenColorMap: [], + + tokenFontMap: [] }; } setColorMapOverride(colorMapOverride: Color[] | null): void { diff --git a/src/vs/editor/test/browser/commands/trimTrailingWhitespaceCommand.test.ts b/src/vs/editor/test/browser/commands/trimTrailingWhitespaceCommand.test.ts index 503dc55f875..3de2f356e81 100644 --- a/src/vs/editor/test/browser/commands/trimTrailingWhitespaceCommand.test.ts +++ b/src/vs/editor/test/browser/commands/trimTrailingWhitespaceCommand.test.ts @@ -146,20 +146,20 @@ suite('Editor Commands - Trim Trailing Whitespace Command', () => { 0, otherMetadata, 10, stringMetadata, ]); - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } case ' a string ': { const tokens = new Uint32Array([ 0, stringMetadata, ]); - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } case '`; ': { const tokens = new Uint32Array([ 0, stringMetadata, 1, otherMetadata ]); - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } } throw new Error(`Unexpected`); diff --git a/src/vs/editor/test/browser/controller/cursor.test.ts b/src/vs/editor/test/browser/controller/cursor.test.ts index 1322aae8fba..47b7dd7b5a2 100644 --- a/src/vs/editor/test/browser/controller/cursor.test.ts +++ b/src/vs/editor/test/browser/controller/cursor.test.ts @@ -1335,7 +1335,7 @@ suite('Editor Controller - Cursor', () => { getInitialState: () => NullState, tokenize: undefined!, tokenizeEncoded: (line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult => { - return new EncodedTokenizationResult(new Uint32Array(0), state); + return new EncodedTokenizationResult(new Uint32Array(0), [], state); } }; @@ -1533,7 +1533,7 @@ suite('Editor Controller', () => { ); startIndex += tokens[i].length; } - return new EncodedTokenizationResult(result, state); + return new EncodedTokenizationResult(result, [], state); function advance(): void { if (state instanceof BaseState) { @@ -2794,7 +2794,7 @@ suite('Editor Controller', () => { getInitialState: () => NullState, tokenize: undefined!, tokenizeEncoded: (line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult => { - return new EncodedTokenizationResult(new Uint32Array(0), state); + return new EncodedTokenizationResult(new Uint32Array(0), [], state); } }; diff --git a/src/vs/editor/test/browser/viewModel/modelLineProjection.test.ts b/src/vs/editor/test/browser/viewModel/modelLineProjection.test.ts index aecf9a0621d..ddbb919151a 100644 --- a/src/vs/editor/test/browser/viewModel/modelLineProjection.test.ts +++ b/src/vs/editor/test/browser/viewModel/modelLineProjection.test.ts @@ -346,7 +346,7 @@ suite('SplitLinesCollection', () => { tokens[i].value << MetadataConsts.FOREGROUND_OFFSET ); } - return new languages.EncodedTokenizationResult(result, state); + return new languages.EncodedTokenizationResult(result, [], state); } }; const LANGUAGE_ID = 'modelModeTest1'; diff --git a/src/vs/editor/test/common/model/annotations.test.ts b/src/vs/editor/test/common/model/annotations.test.ts new file mode 100644 index 00000000000..6f5bc3b12dd --- /dev/null +++ b/src/vs/editor/test/common/model/annotations.test.ts @@ -0,0 +1,500 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; +import { AnnotatedString, AnnotationsUpdate, IAnnotation, IAnnotationUpdate } from '../../../common/model/tokens/annotations.js'; +import { OffsetRange } from '../../../common/core/ranges/offsetRange.js'; +import { StringEdit } from '../../../common/core/edits/stringEdit.js'; + +// ============================================================================ +// Visual Annotation Test Infrastructure +// ============================================================================ +// This infrastructure allows representing annotations visually using brackets: +// - '[id:text]' marks an annotation with the given id covering 'text' +// - Plain text represents unannotated content +// +// Example: "Lorem [1:ipsum] dolor [2:sit] amet" represents: +// - annotation "1" at offset 6-11 (content "ipsum") +// - annotation "2" at offset 18-21 (content "sit") +// +// For updates: +// - '[id:text]' sets an annotation +// - '' deletes an annotation in that range +// ============================================================================ + +/** + * Parses a visual string representation into annotations. + * The visual string uses '[id:text]' to mark annotation boundaries. + * The id becomes the annotation value, and text is the annotated content. + */ +function parseVisualAnnotations(visual: string): { annotations: IAnnotation[]; baseString: string } { + const annotations: IAnnotation[] = []; + let baseString = ''; + let i = 0; + + while (i < visual.length) { + if (visual[i] === '[') { + // Find the colon and closing bracket + const colonIdx = visual.indexOf(':', i + 1); + const closeIdx = visual.indexOf(']', colonIdx + 1); + if (colonIdx === -1 || closeIdx === -1) { + throw new Error(`Invalid annotation format at position ${i}`); + } + const id = visual.substring(i + 1, colonIdx); + const text = visual.substring(colonIdx + 1, closeIdx); + const startOffset = baseString.length; + baseString += text; + annotations.push({ range: new OffsetRange(startOffset, baseString.length), annotation: id }); + i = closeIdx + 1; + } else { + baseString += visual[i]; + i++; + } + } + + return { annotations, baseString }; +} + +/** + * Converts annotations to a visual string representation. + * Uses '[id:text]' to mark annotation boundaries. + * + * @param annotations - The annotations to visualize + * @param baseString - The base string content + */ +function toVisualString( + annotations: IAnnotation[], + baseString: string +): string { + if (annotations.length === 0) { + return baseString; + } + + // Sort annotations by start position + const sortedAnnotations = [...annotations].sort((a, b) => a.range.start - b.range.start); + + // Build the visual representation + let result = ''; + let pos = 0; + + for (const ann of sortedAnnotations) { + // Add plain text before this annotation + result += baseString.substring(pos, ann.range.start); + // Add annotated content with id + const annotatedText = baseString.substring(ann.range.start, ann.range.endExclusive); + result += `[${ann.annotation}:${annotatedText}]`; + pos = ann.range.endExclusive; + } + + // Add remaining text after last annotation + result += baseString.substring(pos); + + return result; +} + +/** + * Represents an AnnotatedString with its base string for visual testing. + */ +class VisualAnnotatedString { + constructor( + public readonly annotatedString: AnnotatedString, + public baseString: string + ) { } + + setAnnotations(update: AnnotationsUpdate): void { + this.annotatedString.setAnnotations(update); + } + + applyEdit(edit: StringEdit): void { + this.annotatedString.applyEdit(edit); + this.baseString = edit.apply(this.baseString); + } + + getAnnotationsIntersecting(range: OffsetRange): IAnnotation[] { + return this.annotatedString.getAnnotationsIntersecting(range); + } + + getAllAnnotations(): IAnnotation[] { + return this.annotatedString.getAllAnnotations(); + } + + clone(): VisualAnnotatedString { + return new VisualAnnotatedString(this.annotatedString.clone() as AnnotatedString, this.baseString); + } +} + +/** + * Creates a VisualAnnotatedString from a visual representation. + */ +function fromVisual(visual: string): VisualAnnotatedString { + const { annotations, baseString } = parseVisualAnnotations(visual); + return new VisualAnnotatedString(new AnnotatedString(annotations), baseString); +} + +/** + * Converts a VisualAnnotatedString to a visual representation. + */ +function toVisual(vas: VisualAnnotatedString): string { + return toVisualString(vas.getAllAnnotations(), vas.baseString); +} + +/** + * Parses visual update annotations, where: + * - '[id:text]' represents an annotation to set + * - '' represents an annotation to delete (range is tracked but annotation is undefined) + */ +function parseVisualUpdate(visual: string): { updates: IAnnotationUpdate[]; baseString: string } { + const updates: IAnnotationUpdate[] = []; + let baseString = ''; + let i = 0; + + while (i < visual.length) { + if (visual[i] === '[') { + // Set annotation: [id:text] + const colonIdx = visual.indexOf(':', i + 1); + const closeIdx = visual.indexOf(']', colonIdx + 1); + if (colonIdx === -1 || closeIdx === -1) { + throw new Error(`Invalid annotation format at position ${i}`); + } + const id = visual.substring(i + 1, colonIdx); + const text = visual.substring(colonIdx + 1, closeIdx); + const startOffset = baseString.length; + baseString += text; + updates.push({ range: new OffsetRange(startOffset, baseString.length), annotation: id }); + i = closeIdx + 1; + } else if (visual[i] === '<') { + // Delete annotation: + const colonIdx = visual.indexOf(':', i + 1); + const closeIdx = visual.indexOf('>', colonIdx + 1); + if (colonIdx === -1 || closeIdx === -1) { + throw new Error(`Invalid delete format at position ${i}`); + } + const text = visual.substring(colonIdx + 1, closeIdx); + const startOffset = baseString.length; + baseString += text; + updates.push({ range: new OffsetRange(startOffset, baseString.length), annotation: undefined }); + i = closeIdx + 1; + } else { + baseString += visual[i]; + i++; + } + } + + return { updates, baseString }; +} + +/** + * Creates an AnnotationsUpdate from a visual representation. + */ +function updateFromVisual(...visuals: string[]): AnnotationsUpdate { + const updates: IAnnotationUpdate[] = []; + + for (const visual of visuals) { + const { updates: parsedUpdates } = parseVisualUpdate(visual); + updates.push(...parsedUpdates); + } + + return AnnotationsUpdate.create(updates); +} + +/** + * Helper to create a StringEdit from visual notation. + * Uses a pattern matching approach where: + * - 'd' marks positions to delete + * - 'i:text:' inserts 'text' at the marked position + * + * Simpler approach: just use offset-based helpers + */ +function editDelete(start: number, end: number): StringEdit { + return StringEdit.replace(new OffsetRange(start, end), ''); +} + +function editInsert(pos: number, text: string): StringEdit { + return StringEdit.insert(pos, text); +} + +function editReplace(start: number, end: number, text: string): StringEdit { + return StringEdit.replace(new OffsetRange(start, end), text); +} + +/** + * Asserts that a VisualAnnotatedString matches the expected visual representation. + * Only compares annotations, not the base string (since setAnnotations doesn't change the base string). + */ +function assertVisual(vas: VisualAnnotatedString, expectedVisual: string): void { + const actual = toVisual(vas); + const { annotations: expectedAnnotations } = parseVisualAnnotations(expectedVisual); + const actualAnnotations = vas.getAllAnnotations(); + + // Compare annotations for better error messages + if (actualAnnotations.length !== expectedAnnotations.length) { + assert.fail( + `Annotation count mismatch.\n` + + ` Expected: ${expectedVisual}\n` + + ` Actual: ${actual}\n` + + ` Expected ${expectedAnnotations.length} annotations, got ${actualAnnotations.length}` + ); + } + + for (let i = 0; i < actualAnnotations.length; i++) { + const expected = expectedAnnotations[i]; + const actualAnn = actualAnnotations[i]; + if (actualAnn.range.start !== expected.range.start || actualAnn.range.endExclusive !== expected.range.endExclusive) { + assert.fail( + `Annotation ${i} range mismatch.\n` + + ` Expected: (${expected.range.start}, ${expected.range.endExclusive})\n` + + ` Actual: (${actualAnn.range.start}, ${actualAnn.range.endExclusive})\n` + + ` Expected visual: ${expectedVisual}\n` + + ` Actual visual: ${actual}` + ); + } + if (actualAnn.annotation !== expected.annotation) { + assert.fail( + `Annotation ${i} value mismatch.\n` + + ` Expected: "${expected.annotation}"\n` + + ` Actual: "${actualAnn.annotation}"` + ); + } + } +} + +/** + * Helper to visualize the effect of an edit on annotations. + * Returns both before and after states as visual strings. + */ +function visualizeEdit( + beforeAnnotations: string, + edit: StringEdit +): { before: string; after: string } { + const vas = fromVisual(beforeAnnotations); + const before = toVisual(vas); + + vas.applyEdit(edit); + + const after = toVisual(vas); + return { before, after }; +} + +// ============================================================================ +// Visual Annotations Test Suite +// ============================================================================ +// These tests use a visual representation for better readability: +// - '[id:text]' marks annotated regions with id and content +// - Plain text represents unannotated content +// - '' marks regions to delete (in updates) +// +// Example: "Lorem [1:ipsum] dolor [2:sit] amet" represents two annotations: +// "1" at (6,11) covering "ipsum", "2" at (18,21) covering "sit" +// ============================================================================ + +suite('Annotations Suite', () => { + + ensureNoDisposablesAreLeakedInTestSuite(); + + test('setAnnotations 1', () => { + const vas = fromVisual('[1:Lorem] ipsum [2:dolor] sit [3:amet]'); + vas.setAnnotations(updateFromVisual('[4:Lorem i]')); + assertVisual(vas, '[4:Lorem i]psum [2:dolor] sit [3:amet]'); + vas.setAnnotations(updateFromVisual('Lorem ip[5:s]')); + assertVisual(vas, '[4:Lorem i]p[5:s]um [2:dolor] sit [3:amet]'); + }); + + test('setAnnotations 2', () => { + const vas = fromVisual('[1:Lorem] ipsum [2:dolor] sit [3:amet]'); + vas.setAnnotations(updateFromVisual( + 'L<_:orem ipsum d>', + '[4:Lorem ]' + )); + assertVisual(vas, '[4:Lorem ]ipsum dolor sit [3:amet]'); + vas.setAnnotations(updateFromVisual( + 'Lorem <_:ipsum dolor sit amet>', + '[5:Lor]' + )); + assertVisual(vas, '[5:Lor]em ipsum dolor sit amet'); + vas.setAnnotations(updateFromVisual('L[6:or]')); + assertVisual(vas, 'L[6:or]em ipsum dolor sit amet'); + }); + + test('setAnnotations 3', () => { + const vas = fromVisual('[1:Lorem] ipsum [2:dolor] sit [3:amet]'); + vas.setAnnotations(updateFromVisual('Lore[4:m ipsum dolor ]')); + assertVisual(vas, 'Lore[4:m ipsum dolor ]sit [3:amet]'); + vas.setAnnotations(updateFromVisual('Lorem ipsum dolor sit [5:a]')); + assertVisual(vas, 'Lore[4:m ipsum dolor ]sit [5:a]met'); + }); + + test('getAnnotationsIntersecting 1', () => { + const vas = fromVisual('[1:Lorem] ipsum [2:dolor] sit [3:amet]'); + const result1 = vas.getAnnotationsIntersecting(new OffsetRange(0, 13)); + assert.strictEqual(result1.length, 2); + assert.deepStrictEqual(result1.map(a => a.annotation), ['1', '2']); + const result2 = vas.getAnnotationsIntersecting(new OffsetRange(0, 22)); + assert.strictEqual(result2.length, 3); + assert.deepStrictEqual(result2.map(a => a.annotation), ['1', '2', '3']); + }); + + test('getAnnotationsIntersecting 2', () => { + const vas = fromVisual('[1:Lorem] [2:i]p[3:s]'); + + const result1 = vas.getAnnotationsIntersecting(new OffsetRange(5, 7)); + assert.strictEqual(result1.length, 2); + assert.deepStrictEqual(result1.map(a => a.annotation), ['1', '2']); + const result2 = vas.getAnnotationsIntersecting(new OffsetRange(5, 9)); + assert.strictEqual(result2.length, 3); + assert.deepStrictEqual(result2.map(a => a.annotation), ['1', '2', '3']); + }); + + test('getAnnotationsIntersecting 3', () => { + const vas = fromVisual('[1:Lorem] ipsum [2:dolor]'); + const result1 = vas.getAnnotationsIntersecting(new OffsetRange(4, 13)); + assert.strictEqual(result1.length, 2); + assert.deepStrictEqual(result1.map(a => a.annotation), ['1', '2']); + vas.setAnnotations(updateFromVisual('[3:Lore]m[4: ipsu]')); + assertVisual(vas, '[3:Lore]m[4: ipsu]m [2:dolor]'); + const result2 = vas.getAnnotationsIntersecting(new OffsetRange(7, 13)); + assert.strictEqual(result2.length, 2); + assert.deepStrictEqual(result2.map(a => a.annotation), ['4', '2']); + }); + + test('getAnnotationsIntersecting 4', () => { + const vas = fromVisual('[1:Lorem ipsum] sit'); + vas.setAnnotations(updateFromVisual('Lorem ipsum [2:sit]')); + const result = vas.getAnnotationsIntersecting(new OffsetRange(2, 8)); + assert.strictEqual(result.length, 1); + assert.deepStrictEqual(result.map(a => a.annotation), ['1']); + }); + + test('getAnnotationsIntersecting 5', () => { + const vas = fromVisual('[1:Lorem ipsum] [2:dol] [3:or]'); + const result = vas.getAnnotationsIntersecting(new OffsetRange(1, 16)); + assert.strictEqual(result.length, 3); + assert.deepStrictEqual(result.map(a => a.annotation), ['1', '2', '3']); + }); + + test('applyEdit 1 - deletion within annotation', () => { + const result = visualizeEdit( + '[1:Lorem] ipsum [2:dolor] sit [3:amet]', + editDelete(0, 3) + ); + assert.strictEqual(result.after, '[1:em] ipsum [2:dolor] sit [3:amet]'); + }); + + test('applyEdit 2 - deletion and insertion within annotation', () => { + const result = visualizeEdit( + '[1:Lorem] ipsum [2:dolor] sit [3:amet]', + editReplace(1, 3, 'XXXXX') + ); + assert.strictEqual(result.after, '[1:LXXXXXem] ipsum [2:dolor] sit [3:amet]'); + }); + + test('applyEdit 3 - deletion across several annotations', () => { + const result = visualizeEdit( + '[1:Lorem] ipsum [2:dolor] sit [3:amet]', + editReplace(4, 22, 'XXXXX') + ); + assert.strictEqual(result.after, '[1:LoreXXXXX][3:amet]'); + }); + + test('applyEdit 4 - deletion between annotations', () => { + const result = visualizeEdit( + '[1:Lorem ip]sum and [2:dolor] sit [3:amet]', + editDelete(10, 12) + ); + assert.strictEqual(result.after, '[1:Lorem ip]suand [2:dolor] sit [3:amet]'); + }); + + test('applyEdit 5 - deletion that covers annotation', () => { + const result = visualizeEdit( + '[1:Lorem] ipsum [2:dolor] sit [3:amet]', + editDelete(0, 5) + ); + assert.strictEqual(result.after, ' ipsum [2:dolor] sit [3:amet]'); + }); + + test('applyEdit 6 - several edits', () => { + const vas = fromVisual('[1:Lorem] ipsum [2:dolor] sit [3:amet]'); + const edit = StringEdit.compose([ + StringEdit.replace(new OffsetRange(0, 6), ''), + StringEdit.replace(new OffsetRange(6, 12), ''), + StringEdit.replace(new OffsetRange(12, 17), '') + ]); + vas.applyEdit(edit); + assertVisual(vas, 'ipsum sit [3:am]'); + }); + + test('applyEdit 7 - several edits', () => { + const vas = fromVisual('[1:Lorem] ipsum [2:dolor] sit [3:amet]'); + const edit1 = StringEdit.replace(new OffsetRange(0, 3), 'XXXX'); + const edit2 = StringEdit.replace(new OffsetRange(0, 2), ''); + vas.applyEdit(edit1.compose(edit2)); + assertVisual(vas, '[1:XXem] ipsum [2:dolor] sit [3:amet]'); + }); + + test('applyEdit 9 - insertion at end of annotation', () => { + const result = visualizeEdit( + '[1:Lorem] ipsum [2:dolor] sit [3:amet]', + editInsert(17, 'XXX') + ); + assert.strictEqual(result.after, '[1:Lorem] ipsum [2:dolor]XXX sit [3:amet]'); + }); + + test('applyEdit 10 - insertion in middle of annotation', () => { + const result = visualizeEdit( + '[1:Lorem] ipsum [2:dolor] sit [3:amet]', + editInsert(14, 'XXX') + ); + assert.strictEqual(result.after, '[1:Lorem] ipsum [2:doXXXlor] sit [3:amet]'); + }); + + test('applyEdit 11 - replacement consuming annotation', () => { + const result = visualizeEdit( + '[1:L]o[2:rem] [3:i]', + editReplace(1, 6, 'X') + ); + assert.strictEqual(result.after, '[1:L]X[3:i]'); + }); + + test('applyEdit 12 - multiple disjoint edits', () => { + const vas = fromVisual('[1:Lorem] ipsum [2:dolor] sit [3:amet!] [4:done]'); + + const edit = StringEdit.compose([ + StringEdit.insert(0, 'X'), + StringEdit.delete(new OffsetRange(12, 13)), + StringEdit.replace(new OffsetRange(21, 22), 'YY'), + StringEdit.replace(new OffsetRange(28, 32), 'Z') + ]); + vas.applyEdit(edit); + assertVisual(vas, 'X[1:Lorem] ipsum[2:dolor] sitYY[3:amet!]Z[4:e]'); + }); + + test('applyEdit 13 - edit on the left border', () => { + const result = visualizeEdit( + 'lorem ipsum dolor[1: ]', + editInsert(17, 'X') + ); + assert.strictEqual(result.after, 'lorem ipsum dolorX[1: ]'); + }); + + test('rebase', () => { + const a = new VisualAnnotatedString( + new AnnotatedString([{ range: new OffsetRange(2, 5), annotation: '1' }]), + 'sitamet' + ); + const b = a.clone(); + const update: AnnotationsUpdate = AnnotationsUpdate.create([{ range: new OffsetRange(4, 5), annotation: '2' }]); + + b.setAnnotations(update); + const edit: StringEdit = StringEdit.replace(new OffsetRange(1, 6), 'XXX'); + + a.applyEdit(edit); + b.applyEdit(edit); + + update.rebase(edit); + + a.setAnnotations(update); + assert.deepStrictEqual(a.getAllAnnotations(), b.getAllAnnotations()); + }); +}); diff --git a/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts b/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts index 3b18e1d835c..2e8099a60a6 100644 --- a/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts +++ b/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts @@ -187,7 +187,7 @@ export class TokenizedDocument { offset += t.text.length; } - return new EncodedTokenizationResult(new Uint32Array(arr), new State(state2.lineNumber + 1)); + return new EncodedTokenizationResult(new Uint32Array(arr), [], new State(state2.lineNumber + 1)); } }; } diff --git a/src/vs/editor/test/common/model/model.line.test.ts b/src/vs/editor/test/common/model/model.line.test.ts index 446e7acaf42..b28e6ea067e 100644 --- a/src/vs/editor/test/common/model/model.line.test.ts +++ b/src/vs/editor/test/common/model/model.line.test.ts @@ -122,7 +122,7 @@ class ManualTokenizationSupport implements ITokenizationSupport { tokenizeEncoded(line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult { const s = state as LineState; - return new EncodedTokenizationResult(this.tokens.get(s.lineNumber)!, new LineState(s.lineNumber + 1)); + return new EncodedTokenizationResult(this.tokens.get(s.lineNumber)!, [], new LineState(s.lineNumber + 1)); } /** diff --git a/src/vs/editor/test/common/model/model.modes.test.ts b/src/vs/editor/test/common/model/model.modes.test.ts index a7ad097c019..d65eb62519c 100644 --- a/src/vs/editor/test/common/model/model.modes.test.ts +++ b/src/vs/editor/test/common/model/model.modes.test.ts @@ -31,7 +31,7 @@ suite('Editor Model - Model Modes 1', () => { tokenize: undefined!, tokenizeEncoded: (line: string, hasEOL: boolean, state: languages.IState): languages.EncodedTokenizationResult => { calledFor.push(line.charAt(0)); - return new languages.EncodedTokenizationResult(new Uint32Array(0), state); + return new languages.EncodedTokenizationResult(new Uint32Array(0), [], state); } }; @@ -188,7 +188,7 @@ suite('Editor Model - Model Modes 2', () => { tokenizeEncoded: (line: string, hasEOL: boolean, state: languages.IState): languages.EncodedTokenizationResult => { calledFor.push(line); (state).prevLineContent = line; - return new languages.EncodedTokenizationResult(new Uint32Array(0), state); + return new languages.EncodedTokenizationResult(new Uint32Array(0), [], state); } }; diff --git a/src/vs/editor/test/common/model/model.test.ts b/src/vs/editor/test/common/model/model.test.ts index e18b8438525..e6544a65608 100644 --- a/src/vs/editor/test/common/model/model.test.ts +++ b/src/vs/editor/test/common/model/model.test.ts @@ -428,7 +428,7 @@ suite('Editor Model - Words', () => { for (let i = 0; i < tokens.length; i++) { tokens[i] = tokensArr[i]; } - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } })); } diff --git a/src/vs/editor/test/common/model/textModelWithTokens.test.ts b/src/vs/editor/test/common/model/textModelWithTokens.test.ts index f5118870ca0..bcf65679059 100644 --- a/src/vs/editor/test/common/model/textModelWithTokens.test.ts +++ b/src/vs/editor/test/common/model/textModelWithTokens.test.ts @@ -390,7 +390,7 @@ suite('TextModelWithTokens 2', () => { 12, otherMetadata1, 13, otherMetadata1, ]); - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } case ' return

{true}

;': { const tokens = new Uint32Array([ @@ -408,13 +408,13 @@ suite('TextModelWithTokens 2', () => { 21, otherMetadata2, 22, otherMetadata2, ]); - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } case '}': { const tokens = new Uint32Array([ 0, otherMetadata1 ]); - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } } throw new Error(`Unexpected`); @@ -487,7 +487,7 @@ suite('TextModelWithTokens 2', () => { const tokens = new Uint32Array([ 0, otherMetadata ]); - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } case ' console.log(`${100}`);': { const tokens = new Uint32Array([ @@ -497,13 +497,13 @@ suite('TextModelWithTokens 2', () => { 22, stringMetadata, 24, otherMetadata, ]); - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } case '}': { const tokens = new Uint32Array([ 0, otherMetadata ]); - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } } throw new Error(`Unexpected`); @@ -585,7 +585,7 @@ suite('TextModelWithTokens regression tests', () => { tokens[1] = ( myId << MetadataConsts.FOREGROUND_OFFSET ) >>> 0; - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } }; @@ -694,7 +694,7 @@ suite('TextModelWithTokens regression tests', () => { tokens[1] = ( encodedInnerMode << MetadataConsts.LANGUAGEID_OFFSET ) >>> 0; - return new EncodedTokenizationResult(tokens, state); + return new EncodedTokenizationResult(tokens, [], state); } }; diff --git a/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts b/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts index 7c0ab5d725b..59e562aec8c 100644 --- a/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts +++ b/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts @@ -393,7 +393,7 @@ class Mode extends Disposable { for (let i = 0; i < tokens.length; i++) { tokens[i] = tokensArr[i]; } - return new EncodedTokenizationResult(tokens, null!); + return new EncodedTokenizationResult(tokens, [], null!); } })); } diff --git a/src/vs/platform/theme/common/themeService.ts b/src/vs/platform/theme/common/themeService.ts index c0d0065b3a2..9a4657d9a7a 100644 --- a/src/vs/platform/theme/common/themeService.ts +++ b/src/vs/platform/theme/common/themeService.ts @@ -70,12 +70,23 @@ export interface IColorTheme { */ readonly tokenColorMap: string[]; + /** + * List of all the fonts used with tokens. + */ + readonly tokenFontMap: IFontTokenOptions[]; + /** * Defines whether semantic highlighting should be enabled for the theme. */ readonly semanticHighlighting: boolean; } +export class IFontTokenOptions { + fontFamily?: string; + fontSize?: string; + lineHeight?: number; +} + export interface IFileIconTheme { readonly hasFileIcons: boolean; readonly hasFolderIcons: boolean; diff --git a/src/vs/platform/theme/test/common/testThemeService.ts b/src/vs/platform/theme/test/common/testThemeService.ts index 8ee388d4dbd..09f4162992a 100644 --- a/src/vs/platform/theme/test/common/testThemeService.ts +++ b/src/vs/platform/theme/test/common/testThemeService.ts @@ -7,7 +7,7 @@ import { Color } from '../../../../base/common/color.js'; import { Emitter, Event } from '../../../../base/common/event.js'; import { IconContribution } from '../../common/iconRegistry.js'; import { ColorScheme } from '../../common/theme.js'; -import { IColorTheme, IFileIconTheme, IProductIconTheme, IThemeService, ITokenStyle } from '../../common/themeService.js'; +import { IColorTheme, IFileIconTheme, IProductIconTheme, IThemeService, IFontTokenOptions, ITokenStyle } from '../../common/themeService.js'; export class TestColorTheme implements IColorTheme { @@ -38,6 +38,10 @@ export class TestColorTheme implements IColorTheme { get tokenColorMap(): string[] { return []; } + + get tokenFontMap(): IFontTokenOptions[] { + return []; + } } class TestFileIconTheme implements IFileIconTheme { diff --git a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts index cc144be8284..7bfd3ded3e3 100644 --- a/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts +++ b/src/vs/workbench/contrib/codeEditor/test/node/autoindent.test.ts @@ -89,7 +89,7 @@ function registerTokenizationSupport(instantiationService: TestInstantiationServ ((encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET) | (tokensOnLine[i].standardTokenType << MetadataConsts.TOKEN_TYPE_OFFSET)); } - return new EncodedTokenizationResult(result, state); + return new EncodedTokenizationResult(result, [], state); } }; return TokenizationRegistry.register(languageId, tokenizationSupport); diff --git a/src/vs/workbench/contrib/terminal/test/common/terminalColorRegistry.test.ts b/src/vs/workbench/contrib/terminal/test/common/terminalColorRegistry.test.ts index 865ea509b49..1f9bb62b4de 100644 --- a/src/vs/workbench/contrib/terminal/test/common/terminalColorRegistry.test.ts +++ b/src/vs/workbench/contrib/terminal/test/common/terminalColorRegistry.test.ts @@ -24,6 +24,7 @@ function getMockTheme(type: ColorScheme): IColorTheme { defines: () => true, getTokenStyleMetadata: () => undefined, tokenColorMap: [], + tokenFontMap: [], semanticHighlighting: false }; return theme; diff --git a/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.ts b/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.ts index 86cad433574..61fee6de827 100644 --- a/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.ts +++ b/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.ts @@ -12,7 +12,7 @@ import { Range } from '../../../../../editor/common/core/range.js'; import { IBackgroundTokenizationStore, ILanguageIdCodec } from '../../../../../editor/common/languages.js'; import { ITextModel } from '../../../../../editor/common/model.js'; import { TokenizationStateStore } from '../../../../../editor/common/model/textModelTokens.js'; -import { IModelContentChangedEvent } from '../../../../../editor/common/textModelEvents.js'; +import { deserializeFontTokenOptions, IFontTokenOption, IModelContentChangedEvent } from '../../../../../editor/common/textModelEvents.js'; import { IModelContentChange } from '../../../../../editor/common/model/mirrorTextModel.js'; import { ContiguousMultilineTokensBuilder } from '../../../../../editor/common/tokens/contiguousMultilineTokensBuilder.js'; import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js'; @@ -21,6 +21,9 @@ import { MonotonousIndexTransformer } from '../indexTransformer.js'; import type { StateDeltas, TextMateTokenizationWorker } from './worker/textMateTokenizationWorker.worker.js'; import type { applyStateStackDiff, StateStack } from 'vscode-textmate'; import { linesLengthEditFromModelContentChange } from '../../../../../editor/common/model/textModelStringEdit.js'; +import { StringEdit } from '../../../../../editor/common/core/edits/stringEdit.js'; +import { OffsetRange } from '../../../../../editor/common/core/ranges/offsetRange.js'; +import { AnnotationsUpdate, ISerializedAnnotation } from '../../../../../editor/common/model/tokens/annotations.js'; export class TextMateWorkerTokenizerController extends Disposable { private static _id = 0; @@ -109,7 +112,7 @@ export class TextMateWorkerTokenizerController extends Disposable { /** * This method is called from the worker through the worker host. */ - public async setTokensAndStates(controllerId: number, versionId: number, rawTokens: Uint8Array, stateDeltas: StateDeltas[]): Promise { + public async setTokensAndStates(controllerId: number, versionId: number, rawTokens: Uint8Array, fontTokens: ISerializedAnnotation[], stateDeltas: StateDeltas[]): Promise { if (this.controllerId !== controllerId) { // This event is for an outdated controller (the worker didn't receive the delete/create messages yet), ignore the event. return; @@ -122,6 +125,7 @@ export class TextMateWorkerTokenizerController extends Disposable { let tokens = ContiguousMultilineTokensBuilder.deserialize( new Uint8Array(rawTokens) ); + const fontTokensUpdate = AnnotationsUpdate.deserialize(fontTokens, deserializeFontTokenOptions()); if (this._shouldLog) { console.log('received background tokenization result', { @@ -178,6 +182,7 @@ export class TextMateWorkerTokenizerController extends Disposable { } } } + fontTokensUpdate.rebase(this._stringEditFromChanges(this._model, this._pendingChanges)); } const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany( @@ -220,6 +225,21 @@ export class TextMateWorkerTokenizerController extends Disposable { } // First set states, then tokens, so that events fired from set tokens don't read invalid states this._backgroundTokenizationStore.setTokens(tokens); + this._backgroundTokenizationStore.setFontInfo(fontTokensUpdate); + } + + private _stringEditFromChanges(model: ITextModel, pendingChanges: IModelContentChangedEvent[]): StringEdit { + const edits: StringEdit[] = []; + for (const change of pendingChanges) { + for (const innerChanges of change.changes) { + const range = Range.lift(innerChanges.range); + const text = innerChanges.text; + const offsetEditStart = model.getOffsetAt(range.getStartPosition()); + const offsetEditEnd = model.getOffsetAt(range.getEndPosition()); + edits.push(StringEdit.replace(new OffsetRange(offsetEditStart, offsetEditEnd), text)); + } + } + return StringEdit.compose(edits); } private get _shouldLog() { return this._loggingEnabled.get(); } diff --git a/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.ts b/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.ts index 3662b13b377..9f8b2fdc9d3 100644 --- a/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.ts +++ b/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.ts @@ -25,6 +25,8 @@ import type { IRawTheme } from 'vscode-textmate'; import { WebWorkerDescriptor } from '../../../../../platform/webWorker/browser/webWorkerDescriptor.js'; import { IWebWorkerService } from '../../../../../platform/webWorker/browser/webWorkerService.js'; import { IWebWorkerClient, Proxied } from '../../../../../base/common/worker/webWorker.js'; +import { ISerializedAnnotation } from '../../../../../editor/common/model/tokens/annotations.js'; +import { IFontTokenOption } from '../../../../../editor/common/textModelEvents.js'; export class ThreadedBackgroundTokenizerFactory implements IDisposable { private static _reportedMismatchingTokens = false; @@ -150,13 +152,13 @@ export class ThreadedBackgroundTokenizerFactory implements IDisposable { const resource = URI.revive(_resource); return this._extensionResourceLoaderService.readExtensionResource(resource); }, - $setTokensAndStates: async (controllerId: number, versionId: number, tokens: Uint8Array, lineEndStateDeltas: StateDeltas[]): Promise => { + $setTokensAndStates: async (controllerId: number, versionId: number, tokens: Uint8Array, fontTokens: ISerializedAnnotation[], lineEndStateDeltas: StateDeltas[]): Promise => { const controller = this._workerTokenizerControllers.get(controllerId); // When a model detaches, it is removed synchronously from the map. // However, the worker might still be sending tokens for that model, // so we ignore the event when there is no controller. if (controller) { - controller.setTokensAndStates(controllerId, versionId, tokens, lineEndStateDeltas); + controller.setTokensAndStates(controllerId, versionId, tokens, fontTokens, lineEndStateDeltas); } }, $reportTokenizationTime: (timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void => { diff --git a/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.ts b/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.ts index 124a298e0bd..157e314ba7d 100644 --- a/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.ts +++ b/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.ts @@ -13,6 +13,8 @@ import { TextMateWorkerTokenizer } from './textMateWorkerTokenizer.js'; import { importAMDNodeModule } from '../../../../../../amdX.js'; import { IWebWorkerServerRequestHandler, IWebWorkerServer } from '../../../../../../base/common/worker/webWorker.js'; import { TextMateWorkerHost } from './textMateWorkerHost.js'; +import { ISerializedAnnotation } from '../../../../../../editor/common/model/tokens/annotations.js'; +import { IFontTokenOption } from '../../../../../../editor/common/textModelEvents.js'; export function create(workerServer: IWebWorkerServer): TextMateTokenizationWorker { return new TextMateTokenizationWorker(workerServer); @@ -109,8 +111,8 @@ export class TextMateTokenizationWorker implements IWebWorkerServerRequestHandle } return that._grammarCache[encodedLanguageId]; }, - setTokensAndStates(versionId: number, tokens: Uint8Array, stateDeltas: StateDeltas[]): void { - that._host.$setTokensAndStates(data.controllerId, versionId, tokens, stateDeltas); + setTokensAndStates(versionId: number, tokens: Uint8Array, fontTokens: ISerializedAnnotation[], stateDeltas: StateDeltas[]): void { + that._host.$setTokensAndStates(data.controllerId, versionId, tokens, fontTokens, stateDeltas); }, reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void { that._host.$reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample); diff --git a/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerHost.ts b/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerHost.ts index e84330da915..c289c9b85e0 100644 --- a/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerHost.ts +++ b/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerHost.ts @@ -5,6 +5,8 @@ import { UriComponents } from '../../../../../../base/common/uri.js'; import { IWebWorkerServer, IWebWorkerClient } from '../../../../../../base/common/worker/webWorker.js'; +import { ISerializedAnnotation } from '../../../../../../editor/common/model/tokens/annotations.js'; +import { IFontTokenOption } from '../../../../../../editor/common/textModelEvents.js'; import { StateDeltas } from './textMateTokenizationWorker.worker.js'; export abstract class TextMateWorkerHost { @@ -17,6 +19,6 @@ export abstract class TextMateWorkerHost { } abstract $readFile(_resource: UriComponents): Promise; - abstract $setTokensAndStates(controllerId: number, versionId: number, tokens: Uint8Array, lineEndStateDeltas: StateDeltas[]): Promise; + abstract $setTokensAndStates(controllerId: number, versionId: number, tokens: Uint8Array, fontTokens: ISerializedAnnotation[], lineEndStateDeltas: StateDeltas[]): Promise; abstract $reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void; } diff --git a/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.ts b/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.ts index fee62cac570..d410a975a99 100644 --- a/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.ts +++ b/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.ts @@ -20,10 +20,14 @@ import type { StackDiff, StateStack, diffStateStacksRefEq } from 'vscode-textmat import { ICreateGrammarResult } from '../../../common/TMGrammarFactory.js'; import { StateDeltas } from './textMateTokenizationWorker.worker.js'; import { Disposable } from '../../../../../../base/common/lifecycle.js'; +import { IFontTokenOption, serializeFontTokenOptions } from '../../../../../../editor/common/textModelEvents.js'; +import { AnnotationsUpdate, IAnnotationUpdate, ISerializedAnnotation } from '../../../../../../editor/common/model/tokens/annotations.js'; +import { OffsetRange } from '../../../../../../editor/common/core/ranges/offsetRange.js'; +import { EncodedTokenizationResult } from '../../../../../../editor/common/languages.js'; export interface TextMateModelTokenizerHost { getOrCreateGrammar(languageId: string, encodedLanguageId: LanguageId): Promise; - setTokensAndStates(versionId: number, tokens: Uint8Array, stateDeltas: StateDeltas[]): void; + setTokensAndStates(versionId: number, tokens: Uint8Array, fontTokens: ISerializedAnnotation[], stateDeltas: StateDeltas[]): void; reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void; } @@ -125,6 +129,7 @@ export class TextMateWorkerTokenizer extends MirrorTextModel { let tokenizedLines = 0; const tokenBuilder = new ContiguousMultilineTokensBuilder(); const stateDeltaBuilder = new StateDeltaBuilder(); + const fontTokensUpdate: IAnnotationUpdate[] = []; while (true) { const lineToTokenize = this._tokenizerWithStateStore.getFirstInvalidLine(); @@ -145,6 +150,7 @@ export class TextMateWorkerTokenizer extends MirrorTextModel { LineTokens.convertToEndOffset(r.tokens, text.length); tokenBuilder.add(lineToTokenize.lineNumber, r.tokens); + fontTokensUpdate.push(...this._getFontTokensUpdate(lineToTokenize.lineNumber, r)); const deltaMs = new Date().getTime() - startTime; if (deltaMs > 20) { @@ -157,10 +163,13 @@ export class TextMateWorkerTokenizer extends MirrorTextModel { break; } + const fontUpdate = AnnotationsUpdate.create(fontTokensUpdate); + const serializedFontUpdate = fontUpdate.serialize(serializeFontTokenOptions()); const stateDeltas = stateDeltaBuilder.getStateDeltas(); this._host.setTokensAndStates( this._versionId, tokenBuilder.serialize(), + serializedFontUpdate, stateDeltas ); @@ -172,6 +181,36 @@ export class TextMateWorkerTokenizer extends MirrorTextModel { } } } + + private _getFontTokensUpdate(lineNumber: number, r: EncodedTokenizationResult): IAnnotationUpdate[] { + const fontTokens: IAnnotationUpdate[] = []; + const offsetAtLineStart = this._getOffsetAtLineStart(lineNumber); + const offsetAtNextLineStart = this._getOffsetAtLineStart(lineNumber + 1); + const offsetAtLineEnd = offsetAtNextLineStart > 0 ? offsetAtNextLineStart - 1 : 0; + fontTokens.push({ + range: new OffsetRange(offsetAtLineStart, offsetAtLineEnd), + annotation: undefined + }); + if (r.fontInfo.length) { + for (const fontInfo of r.fontInfo) { + const offsetAtLineStart = this._getOffsetAtLineStart(lineNumber); + fontTokens.push({ + range: new OffsetRange(offsetAtLineStart + fontInfo.startIndex, offsetAtLineStart + fontInfo.endIndex), + annotation: { + fontFamily: fontInfo.fontFamily ?? undefined, + fontSize: fontInfo.fontSize ?? undefined, + lineHeight: fontInfo.lineHeight ?? undefined + } + }); + } + } + return fontTokens; + } + + private _getOffsetAtLineStart(lineNumber: number): number { + this._ensureLineStarts(); + return lineNumber - 1 > 0 ? this._lineStarts!.getPrefixSum(lineNumber - 2) : 0; + } } class StateDeltaBuilder { diff --git a/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts b/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts index 34ee2bfb8b7..0e9f18f3d10 100644 --- a/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts +++ b/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts @@ -18,7 +18,7 @@ import { URI } from '../../../../base/common/uri.js'; import { StandardTokenType } from '../../../../editor/common/encodedTokenAttributes.js'; import { ITokenizationSupport, LazyTokenizationSupport, TokenizationRegistry } from '../../../../editor/common/languages.js'; import { ILanguageService } from '../../../../editor/common/languages/language.js'; -import { generateTokensCSSForColorMap } from '../../../../editor/common/languages/supports/tokenization.js'; +import { generateTokensCSSForColorMap, generateTokensCSSForFontMap } from '../../../../editor/common/languages/supports/tokenization.js'; import * as nls from '../../../../nls.js'; import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js'; import { IExtensionResourceLoaderService } from '../../../../platform/extensionResourceLoader/common/extensionResourceLoader.js'; @@ -38,6 +38,7 @@ import { ITMSyntaxExtensionPoint, grammarsExtPoint } from '../common/TMGrammars. import { IValidEmbeddedLanguagesMap, IValidGrammarDefinition, IValidTokenTypeMap } from '../common/TMScopeRegistry.js'; import { ITextMateThemingRule, IWorkbenchColorTheme, IWorkbenchThemeService } from '../../themes/common/workbenchThemeService.js'; import type { IGrammar, IOnigLib, IRawTheme } from 'vscode-textmate'; +import { IFontTokenOptions } from '../../../../platform/theme/common/themeService.js'; export class TextMateTokenizationFeature extends Disposable implements ITextMateTokenizationService { private static reportTokenizationTimeCounter = { sync: 0, async: 0 }; @@ -55,6 +56,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate private readonly _tokenizersRegistrations; private _currentTheme: IRawTheme | null; private _currentTokenColorMap: string[] | null; + private _currentTokenFontMap: IFontTokenOptions[] | null; private readonly _threadedBackgroundTokenizerFactory; constructor( @@ -79,6 +81,7 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate this._tokenizersRegistrations = this._register(new DisposableStore()); this._currentTheme = null; this._currentTokenColorMap = null; + this._currentTokenFontMap = null; this._threadedBackgroundTokenizerFactory = this._instantiationService.createInstance( ThreadedBackgroundTokenizerFactory, (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample), @@ -335,16 +338,19 @@ export class TextMateTokenizationFeature extends Disposable implements ITextMate private _updateTheme(colorTheme: IWorkbenchColorTheme, forceUpdate: boolean): void { if (!forceUpdate && this._currentTheme && this._currentTokenColorMap && equalsTokenRules(this._currentTheme.settings, colorTheme.tokenColors) - && equalArray(this._currentTokenColorMap, colorTheme.tokenColorMap)) { + && equalArray(this._currentTokenColorMap, colorTheme.tokenColorMap) && this._currentTokenFontMap && equalArray(this._currentTokenFontMap, colorTheme.tokenFontMap)) { return; } this._currentTheme = { name: colorTheme.label, settings: colorTheme.tokenColors }; this._currentTokenColorMap = colorTheme.tokenColorMap; + this._currentTokenFontMap = colorTheme.tokenFontMap; this._grammarFactory?.setTheme(this._currentTheme, this._currentTokenColorMap); const colorMap = toColorMap(this._currentTokenColorMap); - const cssRules = generateTokensCSSForColorMap(colorMap); - this._styleElement.textContent = cssRules; + const colorCssRules = generateTokensCSSForColorMap(colorMap); + const fontCssRules = generateTokensCSSForFontMap(this._currentTokenFontMap); + + this._styleElement.textContent = colorCssRules + fontCssRules; TokenizationRegistry.setColorMap(colorMap); if (this._currentTheme && this._currentTokenColorMap) { diff --git a/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.ts b/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.ts index b38487e0101..37605fea5cb 100644 --- a/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.ts +++ b/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.ts @@ -62,7 +62,7 @@ export class TextMateTokenizationSupport extends Disposable implements ITokeniza if (textMateResult.stoppedEarly) { console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`); // return the state at the beginning of the line - return new EncodedTokenizationResult(textMateResult.tokens, state); + return new EncodedTokenizationResult(textMateResult.tokens, textMateResult.fonts, state); } if (this._containsEmbeddedLanguages) { @@ -89,6 +89,6 @@ export class TextMateTokenizationSupport extends Disposable implements ITokeniza endState = textMateResult.ruleStack; } - return new EncodedTokenizationResult(textMateResult.tokens, endState); + return new EncodedTokenizationResult(textMateResult.tokens, textMateResult.fonts, endState); } } diff --git a/src/vs/workbench/services/themes/common/colorThemeData.ts b/src/vs/workbench/services/themes/common/colorThemeData.ts index 2a088b1df3f..386d668f89c 100644 --- a/src/vs/workbench/services/themes/common/colorThemeData.ts +++ b/src/vs/workbench/services/themes/common/colorThemeData.ts @@ -12,7 +12,7 @@ import * as nls from '../../../../nls.js'; import * as types from '../../../../base/common/types.js'; import * as resources from '../../../../base/common/resources.js'; import { Extensions as ColorRegistryExtensions, IColorRegistry, ColorIdentifier, editorBackground, editorForeground, DEFAULT_COLOR_CONFIG_VALUE } from '../../../../platform/theme/common/colorRegistry.js'; -import { ITokenStyle, getThemeTypeSelector } from '../../../../platform/theme/common/themeService.js'; +import { IFontTokenOptions, ITokenStyle, getThemeTypeSelector } from '../../../../platform/theme/common/themeService.js'; import { Registry } from '../../../../platform/registry/common/platform.js'; import { getParseErrorMessage } from '../../../../base/common/jsonErrorMessages.js'; import { URI } from '../../../../base/common/uri.js'; @@ -81,6 +81,7 @@ export class ColorThemeData implements IWorkbenchColorTheme { private textMateThemingRules: ITextMateThemingRule[] | undefined = undefined; // created on demand private tokenColorIndex: TokenColorIndex | undefined = undefined; // created on demand + private tokenFontIndex: TokenFontIndex | undefined = undefined; // created on demand private constructor(id: string, label: string, settingsId: string) { this.id = id; @@ -120,7 +121,17 @@ export class ColorThemeData implements IWorkbenchColorTheme { if (rule.scope === 'token.info-token') { hasDefaultTokens = true; } - result.push({ scope: rule.scope, settings: { foreground: normalizeColor(rule.settings.foreground), background: normalizeColor(rule.settings.background), fontStyle: rule.settings.fontStyle } }); + const ruleSettings = rule.settings; + result.push({ + scope: rule.scope, settings: { + foreground: normalizeColor(ruleSettings.foreground), + background: normalizeColor(ruleSettings.background), + fontStyle: ruleSettings.fontStyle, + fontSize: ruleSettings.fontSize, + fontFamily: ruleSettings.fontFamily, + lineHeight: ruleSettings.lineHeight + } + }); } } @@ -167,7 +178,10 @@ export class ColorThemeData implements IWorkbenchColorTheme { bold: -1, underline: -1, strikethrough: -1, - italic: -1 + italic: -1, + fontFamily: -1, + fontSize: -1, + lineHeight: -1 }; function _processStyle(matchScore: number, style: TokenStyle, definition: TokenStyleDefinition) { @@ -270,10 +284,24 @@ export class ColorThemeData implements IWorkbenchColorTheme { return this.tokenColorIndex; } + + public getTokenFontIndex(): TokenFontIndex { + if (!this.tokenFontIndex) { + const index = new TokenFontIndex(); + this.tokenColors.forEach(r => index.add(r.settings.fontFamily, r.settings.fontSize, r.settings.lineHeight)); + this.tokenFontIndex = index; + } + return this.tokenFontIndex; + } + public get tokenColorMap(): string[] { return this.getTokenColorIndex().asArray(); } + public get tokenFontMap(): IFontTokenOptions[] { + return this.getTokenFontIndex().asArray(); + } + public getTokenStyleMetadata(typeWithLanguage: string, modifiers: string[], defaultLanguage: string, useDefault = true, definitions: TokenStyleDefinitions = {}): ITokenStyle | undefined { const { type, language } = parseClassifierString(typeWithLanguage, defaultLanguage); const style = this.getTokenStyle(type, modifiers, language, useDefault, definitions); @@ -972,7 +1000,43 @@ class TokenColorIndex { public asArray(): string[] { return this._id2color.slice(0); } +} +class TokenFontIndex { + + private _lastFontId: number; + private _id2font: IFontTokenOptions[]; + private _font2id: Map; + + constructor() { + this._lastFontId = 0; + this._id2font = []; + this._font2id = new Map(); + } + + public add(fontFamily: string | undefined, fontSize: string | undefined, lineHeight: number | undefined): number { + const font: IFontTokenOptions = { fontFamily, fontSize, lineHeight }; + let value = this._font2id.get(font); + if (value) { + return value; + } + value = ++this._lastFontId; + this._font2id.set(font, value); + this._id2font[value] = font; + return value; + } + + public get(font: IFontTokenOptions): number { + const value = this._font2id.get(font); + if (value) { + return value; + } + return 0; + } + + public asArray(): IFontTokenOptions[] { + return this._id2font.slice(0); + } } function normalizeColor(color: string | Color | undefined | null): string | undefined { diff --git a/src/vs/workbench/services/themes/common/colorThemeSchema.ts b/src/vs/workbench/services/themes/common/colorThemeSchema.ts index bb0bdeae99d..ddcc9f57c09 100644 --- a/src/vs/workbench/services/themes/common/colorThemeSchema.ts +++ b/src/vs/workbench/services/themes/common/colorThemeSchema.ts @@ -169,6 +169,18 @@ const textmateColorSchema: IJSONSchema = { { body: 'bold underline strikethrough' }, { body: 'italic bold underline strikethrough' } ] + }, + fontFamily: { + type: 'string', + description: nls.localize('schema.token.fontFamily', 'Font family for the token (e.g., "Fira Code", "JetBrains Mono").') + }, + fontSize: { + type: 'string', + description: nls.localize('schema.token.fontSize', 'Font size string for the token (e.g., "14px", "1.2em").') + }, + lineHeight: { + type: 'number', + description: nls.localize('schema.token.lineHeight', 'Line height number for the token (e.g., "20").') } }, additionalProperties: false, diff --git a/src/vs/workbench/services/themes/common/workbenchThemeService.ts b/src/vs/workbench/services/themes/common/workbenchThemeService.ts index 9c0f9e254d3..679f93e9385 100644 --- a/src/vs/workbench/services/themes/common/workbenchThemeService.ts +++ b/src/vs/workbench/services/themes/common/workbenchThemeService.ts @@ -477,6 +477,9 @@ export interface ITokenColorizationSetting { foreground?: string; background?: string; fontStyle?: string; /* [italic|bold|underline|strikethrough] */ + fontFamily?: string; + fontSize?: string; + lineHeight?: number; } export interface ISemanticTokenColorizationSetting {