diff --git a/src/vs/editor/common/model.ts b/src/vs/editor/common/model.ts index ff16a9a93cc..0e15f872a2d 100644 --- a/src/vs/editor/common/model.ts +++ b/src/vs/editor/common/model.ts @@ -827,7 +827,17 @@ export interface ITextModel { /** * @internal */ - setSemanticTokens(tokens: MultilineTokens2[] | null): void; + setSemanticTokens(tokens: MultilineTokens2[] | null, isComplete: boolean): void; + + /** + * @internal + */ + setPartialSemanticTokens(tokens: MultilineTokens2[] | null): void; + + /** + * @internal + */ + hasSemanticTokens(): boolean; /** * Flush all tokenization state. diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index 73a171cefee..3591172a3a1 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -1793,8 +1793,8 @@ export class TextModel extends Disposable implements model.ITextModel { } } - public setSemanticTokens(tokens: MultilineTokens2[] | null): void { - this._tokens2.set(tokens); + public setSemanticTokens(tokens: MultilineTokens2[] | null, isComplete: boolean): void { + this._tokens2.set(tokens, isComplete); this._emitModelTokensChangedEvent({ tokenizationSupportChanged: false, @@ -1803,6 +1803,17 @@ export class TextModel extends Disposable implements model.ITextModel { }); } + public hasSemanticTokens(): boolean { + return this._tokens2.isComplete(); + } + + public setPartialSemanticTokens(tokens: MultilineTokens2[]): void { + if (this.hasSemanticTokens()) { + return; + } + this.setSemanticTokens(tokens, false); + } + public tokenizeViewport(startLineNumber: number, endLineNumber: number): void { startLineNumber = Math.max(1, startLineNumber); endLineNumber = Math.min(this._buffer.getLineCount(), endLineNumber); diff --git a/src/vs/editor/common/model/tokensStore.ts b/src/vs/editor/common/model/tokensStore.ts index d86331d6f9f..b63c5f065ad 100644 --- a/src/vs/editor/common/model/tokensStore.ts +++ b/src/vs/editor/common/model/tokensStore.ts @@ -737,17 +737,25 @@ function toUint32Array(arr: Uint32Array | ArrayBuffer): Uint32Array { export class TokensStore2 { private _pieces: MultilineTokens2[]; + private _isComplete: boolean; constructor() { this._pieces = []; + this._isComplete = false; } public flush(): void { this._pieces = []; + this._isComplete = false; } - public set(pieces: MultilineTokens2[] | null) { + public set(pieces: MultilineTokens2[] | null, isComplete: boolean) { this._pieces = pieces || []; + this._isComplete = isComplete; + } + + public isComplete(): boolean { + return this._isComplete; } public addSemanticTokens(lineNumber: number, aTokens: LineTokens): LineTokens { diff --git a/src/vs/editor/common/services/modelService.ts b/src/vs/editor/common/services/modelService.ts index 1c4a4514404..60ce39358dc 100644 --- a/src/vs/editor/common/services/modelService.ts +++ b/src/vs/editor/common/services/modelService.ts @@ -8,9 +8,13 @@ import { URI } from 'vs/base/common/uri'; import { ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model'; import { ILanguageSelection } from 'vs/editor/common/services/modeService'; import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; +import { DocumentSemanticTokensProvider, DocumentRangeSemanticTokensProvider } from 'vs/editor/common/modes'; +import { SemanticTokensProviderStyling } from 'vs/editor/common/services/semanticTokensProviderStyling'; export const IModelService = createDecorator('modelService'); +export type DocumentTokensProvider = DocumentSemanticTokensProvider | DocumentRangeSemanticTokensProvider; + export interface IModelService { _serviceBrand: undefined; @@ -28,6 +32,8 @@ export interface IModelService { getModel(resource: URI): ITextModel | null; + getSemanticTokensProviderStyling(provider: DocumentTokensProvider): SemanticTokensProviderStyling; + onModelAdded: Event; onModelRemoved: Event; diff --git a/src/vs/editor/common/services/modelServiceImpl.ts b/src/vs/editor/common/services/modelServiceImpl.ts index 611d1d4c7bd..3cd20849fca 100644 --- a/src/vs/editor/common/services/modelServiceImpl.ts +++ b/src/vs/editor/common/services/modelServiceImpl.ts @@ -12,26 +12,26 @@ import { URI } from 'vs/base/common/uri'; import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/config/editorOptions'; import { EditOperation } from 'vs/editor/common/core/editOperation'; import { Range } from 'vs/editor/common/core/range'; -import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions, IValidEditOperation } from 'vs/editor/common/model'; +import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model'; import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel'; import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents'; -import { LanguageIdentifier, DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokensLegend, SemanticTokens, SemanticTokensEdits, TokenMetadata, FontStyle, MetadataConsts } from 'vs/editor/common/modes'; +import { LanguageIdentifier, DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokens, SemanticTokensEdits } from 'vs/editor/common/modes'; import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry'; import { ILanguageSelection } from 'vs/editor/common/services/modeService'; -import { IModelService } from 'vs/editor/common/services/modelService'; +import { IModelService, DocumentTokensProvider } from 'vs/editor/common/services/modelService'; import { ITextResourcePropertiesService } from 'vs/editor/common/services/textResourceConfigurationService'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { RunOnceScheduler } from 'vs/base/common/async'; import { CancellationTokenSource } from 'vs/base/common/cancellation'; -import { SparseEncodedTokens, MultilineTokens2 } from 'vs/editor/common/model/tokensStore'; import { IThemeService } from 'vs/platform/theme/common/themeService'; -import { ILogService, LogLevel } from 'vs/platform/log/common/log'; +import { ILogService } from 'vs/platform/log/common/log'; import { IUndoRedoService, IUndoRedoElement, IPastFutureElements } from 'vs/platform/undoRedo/common/undoRedo'; import { StringSHA1 } from 'vs/base/common/hash'; import { SingleModelEditStackElement, MultiModelEditStackElement, EditStackElement } from 'vs/editor/common/model/editStack'; import { IDialogService } from 'vs/platform/dialogs/common/dialogs'; import { Schemas } from 'vs/base/common/network'; import Severity from 'vs/base/common/severity'; +import { SemanticTokensProviderStyling, toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling'; export const MAINTAIN_UNDO_REDO_STACK = true; @@ -171,6 +171,7 @@ export class ModelServiceImpl extends Disposable implements IModelService { */ private readonly _models: { [modelId: string]: ModelData; }; private readonly _disposedModels: Map; + private readonly _semanticStyling: SemanticStyling; constructor( @IConfigurationService private readonly _configurationService: IConfigurationService, @@ -184,11 +185,12 @@ export class ModelServiceImpl extends Disposable implements IModelService { this._modelCreationOptionsByLanguageAndResource = Object.create(null); this._models = {}; this._disposedModels = new Map(); + this._semanticStyling = this._register(new SemanticStyling(this._themeService, this._logService)); - this._register(this._configurationService.onDidChangeConfiguration(e => this._updateModelOptions())); + this._register(this._configurationService.onDidChangeConfiguration(() => this._updateModelOptions())); this._updateModelOptions(); - this._register(new SemanticColoringFeature(this, this._themeService, this._configurationService, this._logService)); + this._register(new SemanticColoringFeature(this, this._themeService, this._configurationService, this._semanticStyling)); } private static _readModelOptions(config: IRawConfig, isForSimpleWidget: boolean): ITextModelCreationOptions { @@ -380,7 +382,7 @@ export class ModelServiceImpl extends Disposable implements IModelService { model.pushEditOperations( [], ModelServiceImpl._computeEdits(model, textBuffer), - (inverseEditOperations: IValidEditOperation[]) => [] + () => [] ); model.pushStackElement(); } @@ -542,6 +544,10 @@ export class ModelServiceImpl extends Disposable implements IModelService { return modelData.model; } + public getSemanticTokensProviderStyling(provider: DocumentTokensProvider): SemanticTokensProviderStyling { + return this._semanticStyling.get(provider); + } + // --- end IModelService private _onWillDispose(model: ITextModel): void { @@ -575,13 +581,13 @@ class SemanticColoringFeature extends Disposable { private static readonly SETTING_ID = 'editor.semanticHighlighting'; - private _watchers: Record; - private _semanticStyling: SemanticStyling; + private readonly _watchers: Record; + private readonly _semanticStyling: SemanticStyling; - constructor(modelService: IModelService, themeService: IThemeService, configurationService: IConfigurationService, logService: ILogService) { + constructor(modelService: IModelService, themeService: IThemeService, configurationService: IConfigurationService, semanticStyling: SemanticStyling) { super(); this._watchers = Object.create(null); - this._semanticStyling = this._register(new SemanticStyling(themeService, logService)); + this._semanticStyling = semanticStyling; const isSemanticColoringEnabled = (model: ITextModel) => { if (!themeService.getColorTheme().semanticHighlighting) { @@ -633,201 +639,27 @@ class SemanticColoringFeature extends Disposable { class SemanticStyling extends Disposable { - private _caches: WeakMap; + private _caches: WeakMap; constructor( private readonly _themeService: IThemeService, private readonly _logService: ILogService ) { super(); - this._caches = new WeakMap(); + this._caches = new WeakMap(); this._register(this._themeService.onDidColorThemeChange(() => { - this._caches = new WeakMap(); + this._caches = new WeakMap(); })); } - public get(provider: DocumentSemanticTokensProvider): SemanticColoringProviderStyling { + public get(provider: DocumentTokensProvider): SemanticTokensProviderStyling { if (!this._caches.has(provider)) { - this._caches.set(provider, new SemanticColoringProviderStyling(provider.getLegend(), this._themeService, this._logService)); + this._caches.set(provider, new SemanticTokensProviderStyling(provider.getLegend(), this._themeService, this._logService)); } return this._caches.get(provider)!; } } -const enum Constants { - NO_STYLING = 0b01111111111111111111111111111111 -} - -class HashTableEntry { - public readonly tokenTypeIndex: number; - public readonly tokenModifierSet: number; - public readonly metadata: number; - public next: HashTableEntry | null; - - constructor(tokenTypeIndex: number, tokenModifierSet: number, metadata: number) { - this.tokenTypeIndex = tokenTypeIndex; - this.tokenModifierSet = tokenModifierSet; - this.metadata = metadata; - this.next = null; - } -} - -class HashTable { - - private static _SIZES = [3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, 2097143]; - - private _elementsCount: number; - private _currentLengthIndex: number; - private _currentLength: number; - private _growCount: number; - private _elements: (HashTableEntry | null)[]; - - constructor() { - this._elementsCount = 0; - this._currentLengthIndex = 0; - this._currentLength = HashTable._SIZES[this._currentLengthIndex]; - this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0); - this._elements = []; - HashTable._nullOutEntries(this._elements, this._currentLength); - } - - private static _nullOutEntries(entries: (HashTableEntry | null)[], length: number): void { - for (let i = 0; i < length; i++) { - entries[i] = null; - } - } - - private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number): number { - return ((((tokenTypeIndex << 5) - tokenTypeIndex) + tokenModifierSet) | 0) % this._currentLength; // tokenTypeIndex * 31 + tokenModifierSet, keep as int32 - } - - public get(tokenTypeIndex: number, tokenModifierSet: number): HashTableEntry | null { - const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet); - - let p = this._elements[hash]; - while (p) { - if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet) { - return p; - } - p = p.next; - } - - return null; - } - - public add(tokenTypeIndex: number, tokenModifierSet: number, metadata: number): void { - this._elementsCount++; - if (this._growCount !== 0 && this._elementsCount >= this._growCount) { - // expand! - const oldElements = this._elements; - - this._currentLengthIndex++; - this._currentLength = HashTable._SIZES[this._currentLengthIndex]; - this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0); - this._elements = []; - HashTable._nullOutEntries(this._elements, this._currentLength); - - for (const first of oldElements) { - let p = first; - while (p) { - const oldNext = p.next; - p.next = null; - this._add(p); - p = oldNext; - } - } - } - this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, metadata)); - } - - private _add(element: HashTableEntry): void { - const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet); - element.next = this._elements[hash]; - this._elements[hash] = element; - } -} - -class SemanticColoringProviderStyling { - - private readonly _hashTable: HashTable; - - constructor( - private readonly _legend: SemanticTokensLegend, - private readonly _themeService: IThemeService, - private readonly _logService: ILogService - ) { - this._hashTable = new HashTable(); - } - - public getMetadata(tokenTypeIndex: number, tokenModifierSet: number): number { - const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet); - let metadata: number; - if (entry) { - metadata = entry.metadata; - } else { - const tokenType = this._legend.tokenTypes[tokenTypeIndex]; - const tokenModifiers: string[] = []; - let modifierSet = tokenModifierSet; - for (let modifierIndex = 0; modifierSet > 0 && modifierIndex < this._legend.tokenModifiers.length; modifierIndex++) { - if (modifierSet & 1) { - tokenModifiers.push(this._legend.tokenModifiers[modifierIndex]); - } - modifierSet = modifierSet >> 1; - } - - const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers); - if (typeof tokenStyle === 'undefined') { - metadata = Constants.NO_STYLING; - } else { - metadata = 0; - if (typeof tokenStyle.italic !== 'undefined') { - const italicBit = (tokenStyle.italic ? FontStyle.Italic : 0) << MetadataConsts.FONT_STYLE_OFFSET; - metadata |= italicBit | MetadataConsts.SEMANTIC_USE_ITALIC; - } - if (typeof tokenStyle.bold !== 'undefined') { - const boldBit = (tokenStyle.bold ? FontStyle.Bold : 0) << MetadataConsts.FONT_STYLE_OFFSET; - metadata |= boldBit | MetadataConsts.SEMANTIC_USE_BOLD; - } - if (typeof tokenStyle.underline !== 'undefined') { - const underlineBit = (tokenStyle.underline ? FontStyle.Underline : 0) << MetadataConsts.FONT_STYLE_OFFSET; - metadata |= underlineBit | MetadataConsts.SEMANTIC_USE_UNDERLINE; - } - if (tokenStyle.foreground) { - const foregroundBits = (tokenStyle.foreground) << MetadataConsts.FOREGROUND_OFFSET; - metadata |= foregroundBits | MetadataConsts.SEMANTIC_USE_FOREGROUND; - } - if (metadata === 0) { - // Nothing! - metadata = Constants.NO_STYLING; - } - } - this._hashTable.add(tokenTypeIndex, tokenModifierSet, metadata); - } - if (this._logService.getLevel() === LogLevel.Trace) { - const type = this._legend.tokenTypes[tokenTypeIndex]; - const modifiers = tokenModifierSet ? ' ' + this._legend.tokenModifiers.filter((_, i) => tokenModifierSet & (1 << i)).join(' ') : ''; - this._logService.trace(`tokenStyleMetadata ${entry ? '[CACHED] ' : ''}${type}${modifiers}: foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`); - } - return metadata; - } - - -} - -const enum SemanticColoringConstants { - /** - * Let's aim at having 8KB buffers if possible... - * So that would be 8192 / (5 * 4) = 409.6 tokens per area - */ - DesiredTokensPerArea = 400, - - /** - * Try to keep the total number of areas under 1024 if possible, - * simply compensate by having more tokens per area... - */ - DesiredMaxAreas = 1024, -} - class SemanticTokensResponse { constructor( private readonly _provider: DocumentSemanticTokensProvider, @@ -861,7 +693,7 @@ class ModelSemanticColoring extends Disposable { this._currentDocumentRequestCancellationTokenSource = null; this._documentProvidersChangeListeners = []; - this._register(this._model.onDidChangeContent(e => { + this._register(this._model.onDidChangeContent(() => { if (!this._fetchDocumentSemanticTokens.isScheduled()) { this._fetchDocumentSemanticTokens.schedule(); } @@ -876,7 +708,7 @@ class ModelSemanticColoring extends Disposable { } }; bindDocumentChangeListeners(); - this._register(DocumentSemanticTokensProviderRegistry.onDidChange(e => { + this._register(DocumentSemanticTokensProviderRegistry.onDidChange(() => { bindDocumentChangeListeners(); this._fetchDocumentSemanticTokens.schedule(); })); @@ -963,7 +795,7 @@ class ModelSemanticColoring extends Disposable { } } - private _setDocumentSemanticTokens(provider: DocumentSemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticColoringProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void { + private _setDocumentSemanticTokens(provider: DocumentSemanticTokensProvider | null, tokens: SemanticTokens | SemanticTokensEdits | null, styling: SemanticTokensProviderStyling | null, pendingChanges: IModelContentChangedEvent[]): void { const currentResponse = this._currentDocumentResponse; if (this._currentDocumentResponse) { this._currentDocumentResponse.dispose(); @@ -976,15 +808,19 @@ class ModelSemanticColoring extends Disposable { } return; } - if (!provider || !tokens || !styling) { - this._model.setSemanticTokens(null); + if (!provider || !styling) { + this._model.setSemanticTokens(null, false); + return; + } + if (!tokens) { + this._model.setSemanticTokens(null, true); return; } if (ModelSemanticColoring._isSemanticTokensEdits(tokens)) { if (!currentResponse) { // not possible! - this._model.setSemanticTokens(null); + this._model.setSemanticTokens(null, true); return; } if (tokens.edits.length === 0) { @@ -1036,76 +872,7 @@ class ModelSemanticColoring extends Disposable { this._currentDocumentResponse = new SemanticTokensResponse(provider, tokens.resultId, tokens.data); - const srcData = tokens.data; - const tokenCount = (tokens.data.length / 5) | 0; - const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea); - - const result: MultilineTokens2[] = []; - - let tokenIndex = 0; - let lastLineNumber = 1; - let lastStartCharacter = 0; - while (tokenIndex < tokenCount) { - const tokenStartIndex = tokenIndex; - let tokenEndIndex = Math.min(tokenStartIndex + tokensPerArea, tokenCount); - - // Keep tokens on the same line in the same area... - if (tokenEndIndex < tokenCount) { - - let smallTokenEndIndex = tokenEndIndex; - while (smallTokenEndIndex - 1 > tokenStartIndex && srcData[5 * smallTokenEndIndex] === 0) { - smallTokenEndIndex--; - } - - if (smallTokenEndIndex - 1 === tokenStartIndex) { - // there are so many tokens on this line that our area would be empty, we must now go right - let bigTokenEndIndex = tokenEndIndex; - while (bigTokenEndIndex + 1 < tokenCount && srcData[5 * bigTokenEndIndex] === 0) { - bigTokenEndIndex++; - } - tokenEndIndex = bigTokenEndIndex; - } else { - tokenEndIndex = smallTokenEndIndex; - } - } - - let destData = new Uint32Array((tokenEndIndex - tokenStartIndex) * 4); - let destOffset = 0; - let areaLine = 0; - while (tokenIndex < tokenEndIndex) { - const srcOffset = 5 * tokenIndex; - const deltaLine = srcData[srcOffset]; - const deltaCharacter = srcData[srcOffset + 1]; - const lineNumber = lastLineNumber + deltaLine; - const startCharacter = (deltaLine === 0 ? lastStartCharacter + deltaCharacter : deltaCharacter); - const length = srcData[srcOffset + 2]; - const tokenTypeIndex = srcData[srcOffset + 3]; - const tokenModifierSet = srcData[srcOffset + 4]; - const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet); - - if (metadata !== Constants.NO_STYLING) { - if (areaLine === 0) { - areaLine = lineNumber; - } - destData[destOffset] = lineNumber - areaLine; - destData[destOffset + 1] = startCharacter; - destData[destOffset + 2] = startCharacter + length; - destData[destOffset + 3] = metadata; - destOffset += 4; - } - - lastLineNumber = lineNumber; - lastStartCharacter = startCharacter; - tokenIndex++; - } - - if (destOffset !== destData.length) { - destData = destData.subarray(0, destOffset); - } - - const tokens = new MultilineTokens2(areaLine, new SparseEncodedTokens(destData)); - result.push(tokens); - } + const result = toMultilineTokens2(tokens, styling); // Adjust incoming semantic tokens if (pendingChanges.length > 0) { @@ -1126,11 +893,11 @@ class ModelSemanticColoring extends Disposable { } } - this._model.setSemanticTokens(result); + this._model.setSemanticTokens(result, true); return; } - this._model.setSemanticTokens(null); + this._model.setSemanticTokens(null, true); } private _getSemanticColoringProvider(): DocumentSemanticTokensProvider | null { diff --git a/src/vs/editor/common/services/semanticTokensProviderStyling.ts b/src/vs/editor/common/services/semanticTokensProviderStyling.ts new file mode 100644 index 00000000000..00abe2785e0 --- /dev/null +++ b/src/vs/editor/common/services/semanticTokensProviderStyling.ts @@ -0,0 +1,255 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { SemanticTokensLegend, TokenMetadata, FontStyle, MetadataConsts, SemanticTokens } from 'vs/editor/common/modes'; +import { IThemeService } from 'vs/platform/theme/common/themeService'; +import { ILogService, LogLevel } from 'vs/platform/log/common/log'; +import { MultilineTokens2, SparseEncodedTokens } from 'vs/editor/common/model/tokensStore'; + +export const enum SemanticTokensProviderStylingConstants { + NO_STYLING = 0b01111111111111111111111111111111 +} + +export class SemanticTokensProviderStyling { + + private readonly _hashTable: HashTable; + + constructor( + private readonly _legend: SemanticTokensLegend, + private readonly _themeService: IThemeService, + private readonly _logService: ILogService + ) { + this._hashTable = new HashTable(); + } + + public getMetadata(tokenTypeIndex: number, tokenModifierSet: number): number { + const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet); + let metadata: number; + if (entry) { + metadata = entry.metadata; + } else { + const tokenType = this._legend.tokenTypes[tokenTypeIndex]; + const tokenModifiers: string[] = []; + let modifierSet = tokenModifierSet; + for (let modifierIndex = 0; modifierSet > 0 && modifierIndex < this._legend.tokenModifiers.length; modifierIndex++) { + if (modifierSet & 1) { + tokenModifiers.push(this._legend.tokenModifiers[modifierIndex]); + } + modifierSet = modifierSet >> 1; + } + + const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers); + if (typeof tokenStyle === 'undefined') { + metadata = SemanticTokensProviderStylingConstants.NO_STYLING; + } else { + metadata = 0; + if (typeof tokenStyle.italic !== 'undefined') { + const italicBit = (tokenStyle.italic ? FontStyle.Italic : 0) << MetadataConsts.FONT_STYLE_OFFSET; + metadata |= italicBit | MetadataConsts.SEMANTIC_USE_ITALIC; + } + if (typeof tokenStyle.bold !== 'undefined') { + const boldBit = (tokenStyle.bold ? FontStyle.Bold : 0) << MetadataConsts.FONT_STYLE_OFFSET; + metadata |= boldBit | MetadataConsts.SEMANTIC_USE_BOLD; + } + if (typeof tokenStyle.underline !== 'undefined') { + const underlineBit = (tokenStyle.underline ? FontStyle.Underline : 0) << MetadataConsts.FONT_STYLE_OFFSET; + metadata |= underlineBit | MetadataConsts.SEMANTIC_USE_UNDERLINE; + } + if (tokenStyle.foreground) { + const foregroundBits = (tokenStyle.foreground) << MetadataConsts.FOREGROUND_OFFSET; + metadata |= foregroundBits | MetadataConsts.SEMANTIC_USE_FOREGROUND; + } + if (metadata === 0) { + // Nothing! + metadata = SemanticTokensProviderStylingConstants.NO_STYLING; + } + } + this._hashTable.add(tokenTypeIndex, tokenModifierSet, metadata); + } + if (this._logService.getLevel() === LogLevel.Trace) { + const type = this._legend.tokenTypes[tokenTypeIndex]; + const modifiers = tokenModifierSet ? ' ' + this._legend.tokenModifiers.filter((_, i) => tokenModifierSet & (1 << i)).join(' ') : ''; + this._logService.trace(`tokenStyleMetadata ${entry ? '[CACHED] ' : ''}${type}${modifiers}: foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`); + } + return metadata; + } +} + +const enum SemanticColoringConstants { + /** + * Let's aim at having 8KB buffers if possible... + * So that would be 8192 / (5 * 4) = 409.6 tokens per area + */ + DesiredTokensPerArea = 400, + + /** + * Try to keep the total number of areas under 1024 if possible, + * simply compensate by having more tokens per area... + */ + DesiredMaxAreas = 1024, +} + +export function toMultilineTokens2(tokens: SemanticTokens, styling: SemanticTokensProviderStyling): MultilineTokens2[] { + const srcData = tokens.data; + const tokenCount = (tokens.data.length / 5) | 0; + const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea); + const result: MultilineTokens2[] = []; + + let tokenIndex = 0; + let lastLineNumber = 1; + let lastStartCharacter = 0; + while (tokenIndex < tokenCount) { + const tokenStartIndex = tokenIndex; + let tokenEndIndex = Math.min(tokenStartIndex + tokensPerArea, tokenCount); + + // Keep tokens on the same line in the same area... + if (tokenEndIndex < tokenCount) { + + let smallTokenEndIndex = tokenEndIndex; + while (smallTokenEndIndex - 1 > tokenStartIndex && srcData[5 * smallTokenEndIndex] === 0) { + smallTokenEndIndex--; + } + + if (smallTokenEndIndex - 1 === tokenStartIndex) { + // there are so many tokens on this line that our area would be empty, we must now go right + let bigTokenEndIndex = tokenEndIndex; + while (bigTokenEndIndex + 1 < tokenCount && srcData[5 * bigTokenEndIndex] === 0) { + bigTokenEndIndex++; + } + tokenEndIndex = bigTokenEndIndex; + } else { + tokenEndIndex = smallTokenEndIndex; + } + } + + let destData = new Uint32Array((tokenEndIndex - tokenStartIndex) * 4); + let destOffset = 0; + let areaLine = 0; + while (tokenIndex < tokenEndIndex) { + const srcOffset = 5 * tokenIndex; + const deltaLine = srcData[srcOffset]; + const deltaCharacter = srcData[srcOffset + 1]; + const lineNumber = lastLineNumber + deltaLine; + const startCharacter = (deltaLine === 0 ? lastStartCharacter + deltaCharacter : deltaCharacter); + const length = srcData[srcOffset + 2]; + const tokenTypeIndex = srcData[srcOffset + 3]; + const tokenModifierSet = srcData[srcOffset + 4]; + const metadata = styling.getMetadata(tokenTypeIndex, tokenModifierSet); + + if (metadata !== SemanticTokensProviderStylingConstants.NO_STYLING) { + if (areaLine === 0) { + areaLine = lineNumber; + } + destData[destOffset] = lineNumber - areaLine; + destData[destOffset + 1] = startCharacter; + destData[destOffset + 2] = startCharacter + length; + destData[destOffset + 3] = metadata; + destOffset += 4; + } + + lastLineNumber = lineNumber; + lastStartCharacter = startCharacter; + tokenIndex++; + } + + if (destOffset !== destData.length) { + destData = destData.subarray(0, destOffset); + } + + const tokens = new MultilineTokens2(areaLine, new SparseEncodedTokens(destData)); + result.push(tokens); + } + + return result; +} + +class HashTableEntry { + public readonly tokenTypeIndex: number; + public readonly tokenModifierSet: number; + public readonly metadata: number; + public next: HashTableEntry | null; + + constructor(tokenTypeIndex: number, tokenModifierSet: number, metadata: number) { + this.tokenTypeIndex = tokenTypeIndex; + this.tokenModifierSet = tokenModifierSet; + this.metadata = metadata; + this.next = null; + } +} + +class HashTable { + + private static _SIZES = [3, 7, 13, 31, 61, 127, 251, 509, 1021, 2039, 4093, 8191, 16381, 32749, 65521, 131071, 262139, 524287, 1048573, 2097143]; + + private _elementsCount: number; + private _currentLengthIndex: number; + private _currentLength: number; + private _growCount: number; + private _elements: (HashTableEntry | null)[]; + + constructor() { + this._elementsCount = 0; + this._currentLengthIndex = 0; + this._currentLength = HashTable._SIZES[this._currentLengthIndex]; + this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0); + this._elements = []; + HashTable._nullOutEntries(this._elements, this._currentLength); + } + + private static _nullOutEntries(entries: (HashTableEntry | null)[], length: number): void { + for (let i = 0; i < length; i++) { + entries[i] = null; + } + } + + private _hashFunc(tokenTypeIndex: number, tokenModifierSet: number): number { + return ((((tokenTypeIndex << 5) - tokenTypeIndex) + tokenModifierSet) | 0) % this._currentLength; // tokenTypeIndex * 31 + tokenModifierSet, keep as int32 + } + + public get(tokenTypeIndex: number, tokenModifierSet: number): HashTableEntry | null { + const hash = this._hashFunc(tokenTypeIndex, tokenModifierSet); + + let p = this._elements[hash]; + while (p) { + if (p.tokenTypeIndex === tokenTypeIndex && p.tokenModifierSet === tokenModifierSet) { + return p; + } + p = p.next; + } + + return null; + } + + public add(tokenTypeIndex: number, tokenModifierSet: number, metadata: number): void { + this._elementsCount++; + if (this._growCount !== 0 && this._elementsCount >= this._growCount) { + // expand! + const oldElements = this._elements; + + this._currentLengthIndex++; + this._currentLength = HashTable._SIZES[this._currentLengthIndex]; + this._growCount = Math.round(this._currentLengthIndex + 1 < HashTable._SIZES.length ? 2 / 3 * this._currentLength : 0); + this._elements = []; + HashTable._nullOutEntries(this._elements, this._currentLength); + + for (const first of oldElements) { + let p = first; + while (p) { + const oldNext = p.next; + p.next = null; + this._add(p); + p = oldNext; + } + } + } + this._add(new HashTableEntry(tokenTypeIndex, tokenModifierSet, metadata)); + } + + private _add(element: HashTableEntry): void { + const hash = this._hashFunc(element.tokenTypeIndex, element.tokenModifierSet); + element.next = this._elements[hash]; + this._elements[hash] = element; + } +} diff --git a/src/vs/editor/contrib/viewportSemanticTokens/viewportSemanticTokens.ts b/src/vs/editor/contrib/viewportSemanticTokens/viewportSemanticTokens.ts index 226bd19fd90..9f4f0dd6948 100644 --- a/src/vs/editor/contrib/viewportSemanticTokens/viewportSemanticTokens.ts +++ b/src/vs/editor/contrib/viewportSemanticTokens/viewportSemanticTokens.ts @@ -3,13 +3,15 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { RunOnceScheduler } from 'vs/base/common/async'; +import { RunOnceScheduler, createCancelablePromise } from 'vs/base/common/async'; import { Disposable } from 'vs/base/common/lifecycle'; import { ICodeEditor } from 'vs/editor/browser/editorBrowser'; import { registerEditorContribution } from 'vs/editor/browser/editorExtensions'; import { IEditorContribution } from 'vs/editor/common/editorCommon'; import { ITextModel } from 'vs/editor/common/model'; import { DocumentRangeSemanticTokensProviderRegistry, DocumentRangeSemanticTokensProvider } from 'vs/editor/common/modes'; +import { IModelService } from 'vs/editor/common/services/modelService'; +import { toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling'; class ViewportSemanticTokensContribution extends Disposable implements IEditorContribution { @@ -22,7 +24,10 @@ class ViewportSemanticTokensContribution extends Disposable implements IEditorCo private readonly _editor: ICodeEditor; private readonly _tokenizeViewport: RunOnceScheduler; - constructor(editor: ICodeEditor) { + constructor( + editor: ICodeEditor, + @IModelService private readonly _modelService: IModelService + ) { super(); this._editor = editor; this._tokenizeViewport = new RunOnceScheduler(() => this._tokenizeViewportNow(), 100); @@ -47,13 +52,23 @@ class ViewportSemanticTokensContribution extends Disposable implements IEditorCo return; } const model = this._editor.getModel(); + if (model.hasSemanticTokens()) { + return; + } const provider = ViewportSemanticTokensContribution._getSemanticColoringProvider(model); if (!provider) { return; } - - // const visibleRanges = this._editor.getVisibleRanges(); - // console.log(`_tokenizeViewportNow ---> ${visibleRanges.map(r => r.toString()).join(', ')}`); + const styling = this._modelService.getSemanticTokensProviderStyling(provider); + const visibleRanges = this._editor.getVisibleRanges(); + const request = createCancelablePromise(token => Promise.resolve(provider.provideDocumentRangeSemanticTokens(model, visibleRanges[0], token))); + request.then((r) => { + if (!r || model.isDisposed()) { + return; + } + const tokens = toMultilineTokens2(r, styling); + model.setPartialSemanticTokens(tokens); + }); } } diff --git a/src/vs/editor/test/common/model/tokensStore.test.ts b/src/vs/editor/test/common/model/tokensStore.test.ts index 05d8c435b28..e8eacbbdaf4 100644 --- a/src/vs/editor/test/common/model/tokensStore.test.ts +++ b/src/vs/editor/test/common/model/tokensStore.test.ts @@ -98,7 +98,7 @@ suite('TokensStore', () => { function testTokensAdjustment(rawInitialState: string[], edits: IIdentifiedSingleEditOperation[], rawFinalState: string[]) { const initialState = parseTokensState(rawInitialState); const model = createTextModel(initialState.text); - model.setSemanticTokens([initialState.tokens]); + model.setSemanticTokens([initialState.tokens], true); model.applyEdits(edits); @@ -183,7 +183,7 @@ suite('TokensStore', () => { 0, 38, 42, 245768, 0, 43, 47, 180232, ]))) - ]); + ], true); const lineTokens = model.getLineTokens(1); let decodedTokens: number[] = []; for (let i = 0, len = lineTokens.getCount(); i < len; i++) { diff --git a/src/vs/workbench/api/common/shared/semanticTokensDto.ts b/src/vs/workbench/api/common/shared/semanticTokensDto.ts index 5af91f5def9..a5f3738272d 100644 --- a/src/vs/workbench/api/common/shared/semanticTokensDto.ts +++ b/src/vs/workbench/api/common/shared/semanticTokensDto.ts @@ -54,7 +54,14 @@ function fromLittleEndianBuffer(buff: VSBuffer): Uint32Array { // the byte order must be changed reverseEndianness(uint8Arr); } - return new Uint32Array(uint8Arr.buffer, uint8Arr.byteOffset); + if (uint8Arr.byteOffset % 4 === 0) { + return new Uint32Array(uint8Arr.buffer, uint8Arr.byteOffset); + } else { + // unaligned memory access doesn't work on all platforms + const data = new Uint8Array(uint8Arr.byteLength); + data.set(uint8Arr); + return new Uint32Array(data.buffer, data.byteOffset); + } } export function encodeSemanticTokensDto(semanticTokens: ISemanticTokensDto): VSBuffer {