From 1bc85eb39b158b1d039fbb38d1bbdb77944fc638 Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Thu, 30 Dec 2021 18:27:49 +0100 Subject: [PATCH] Split tokensStore into multiple files --- .../editor/browser/widget/diffEditorWidget.ts | 2 +- src/vs/editor/browser/widget/diffReview.ts | 2 +- .../editor/common/controller/cursorCommon.ts | 2 +- src/vs/editor/common/model.ts | 11 +- .../bracketPairsImpl.ts | 2 +- .../bracketPairsTree/tokenizer.ts | 2 +- .../model/pieceTreeTextBuffer/eolCounter.ts | 51 + .../pieceTreeTextBuffer.ts | 2 +- src/vs/editor/common/model/textModel.ts | 45 +- src/vs/editor/common/model/textModelTokens.ts | 23 +- .../model/tokens/contiguousMultilineTokens.ts | 225 +++ .../contiguousMultilineTokensBuilder.ts | 66 + .../model/tokens/contiguousTokensEditing.ts | 144 ++ .../model/tokens/contiguousTokensStore.ts | 215 +++ .../{core => model/tokens}/lineTokens.ts | 6 +- .../model/tokens/sparseMultilineTokens.ts | 594 +++++++ .../common/model/tokens/sparseTokensStore.ts | 238 +++ src/vs/editor/common/model/tokensStore.ts | 1415 ----------------- .../modes/languageConfigurationRegistry.ts | 2 +- src/vs/editor/common/modes/supports.ts | 2 +- .../common/modes/textToHtmlTokenizer.ts | 2 +- .../services/semanticTokensProviderStyling.ts | 8 +- .../common/viewLayout/viewLineRenderer.ts | 2 +- .../common/viewModel/modelLineProjection.ts | 2 +- src/vs/editor/common/viewModel/viewModel.ts | 2 +- .../contrib/folding/hiddenRangeModel.ts | 2 +- .../inlineCompletions/ghostTextWidget.ts | 2 +- src/vs/editor/standalone/browser/colorizer.ts | 2 +- .../viewModel/modelLineProjection.test.ts | 2 +- .../test/common/core/lineTokens.test.ts | 2 +- .../editor/test/common/core/viewLineToken.ts | 2 +- .../test/common/model/model.line.test.ts | 2 +- .../test/common/model/tokensStore.test.ts | 81 +- src/vs/editor/test/common/modesTestUtils.ts | 2 +- .../semanticTokensProviderStyling.test.ts | 6 +- .../viewLayout/viewLineRenderer.test.ts | 2 +- .../inspectEditorTokens.ts | 2 +- .../browser/themes.test.contribution.ts | 2 +- .../textMate/browser/nativeTextMateService.ts | 4 +- .../services/textMate/browser/textMate.ts | 2 +- .../textMate/browser/textMateWorker.ts | 7 +- 41 files changed, 1656 insertions(+), 1531 deletions(-) create mode 100644 src/vs/editor/common/model/pieceTreeTextBuffer/eolCounter.ts create mode 100644 src/vs/editor/common/model/tokens/contiguousMultilineTokens.ts create mode 100644 src/vs/editor/common/model/tokens/contiguousMultilineTokensBuilder.ts create mode 100644 src/vs/editor/common/model/tokens/contiguousTokensEditing.ts create mode 100644 src/vs/editor/common/model/tokens/contiguousTokensStore.ts rename src/vs/editor/common/{core => model/tokens}/lineTokens.ts (98%) create mode 100644 src/vs/editor/common/model/tokens/sparseMultilineTokens.ts create mode 100644 src/vs/editor/common/model/tokens/sparseTokensStore.ts delete mode 100644 src/vs/editor/common/model/tokensStore.ts diff --git a/src/vs/editor/browser/widget/diffEditorWidget.ts b/src/vs/editor/browser/widget/diffEditorWidget.ts index c8708654df1..8e50d73f90a 100644 --- a/src/vs/editor/browser/widget/diffEditorWidget.ts +++ b/src/vs/editor/browser/widget/diffEditorWidget.ts @@ -49,7 +49,7 @@ import { IEditorProgressService, IProgressRunner } from 'vs/platform/progress/co import { ElementSizeObserver } from 'vs/editor/browser/config/elementSizeObserver'; import { Codicon } from 'vs/base/common/codicons'; import { MOUSE_CURSOR_TEXT_CSS_CLASS_NAME } from 'vs/base/browser/ui/mouseCursor/mouseCursor'; -import { IViewLineTokens } from 'vs/editor/common/core/lineTokens'; +import { IViewLineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { FontInfo } from 'vs/editor/common/config/fontInfo'; import { registerIcon } from 'vs/platform/theme/common/iconRegistry'; import { ILineBreaksComputer } from 'vs/editor/common/viewModel/modelLineProjectionData'; diff --git a/src/vs/editor/browser/widget/diffReview.ts b/src/vs/editor/browser/widget/diffReview.ts index c29649cda27..ab3f95cbc93 100644 --- a/src/vs/editor/browser/widget/diffReview.ts +++ b/src/vs/editor/browser/widget/diffReview.ts @@ -18,7 +18,7 @@ import { EditorAction, ServicesAccessor, registerEditorAction } from 'vs/editor/ import { ICodeEditorService } from 'vs/editor/browser/services/codeEditorService'; import { DiffEditorWidget } from 'vs/editor/browser/widget/diffEditorWidget'; import { IComputedEditorOptions, EditorOption, EditorFontLigatures } from 'vs/editor/common/config/editorOptions'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { Position } from 'vs/editor/common/core/position'; import { ILineChange, ScrollType } from 'vs/editor/common/editorCommon'; import { ITextModel, TextModelResolvedOptions } from 'vs/editor/common/model'; diff --git a/src/vs/editor/common/controller/cursorCommon.ts b/src/vs/editor/common/controller/cursorCommon.ts index 8dcf95945a4..7b846b2cbfd 100644 --- a/src/vs/editor/common/controller/cursorCommon.ts +++ b/src/vs/editor/common/controller/cursorCommon.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { ConfigurationChangedEvent, EditorAutoClosingEditStrategy, EditorAutoClosingStrategy, EditorAutoIndentStrategy, EditorAutoSurroundStrategy, EditorOption } from 'vs/editor/common/config/editorOptions'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; import { ISelection, Selection } from 'vs/editor/common/core/selection'; diff --git a/src/vs/editor/common/model.ts b/src/vs/editor/common/model.ts index 16309c7952f..d6305b74e94 100644 --- a/src/vs/editor/common/model.ts +++ b/src/vs/editor/common/model.ts @@ -7,7 +7,7 @@ import { Event } from 'vs/base/common/event'; import { IMarkdownString } from 'vs/base/common/htmlContent'; import { IDisposable } from 'vs/base/common/lifecycle'; import { URI } from 'vs/base/common/uri'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { IPosition, Position } from 'vs/editor/common/core/position'; import { IRange, Range } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; @@ -15,7 +15,8 @@ import { IModelContentChange, IModelContentChangedEvent, IModelDecorationsChange import { SearchData } from 'vs/editor/common/model/textModelSearch'; import { FormattingOptions, StandardTokenType } from 'vs/editor/common/modes'; import { ThemeColor } from 'vs/platform/theme/common/themeService'; -import { MultilineTokens, MultilineTokens2 } from 'vs/editor/common/model/tokensStore'; +import { ContiguousMultilineTokens } from 'vs/editor/common/model/tokens/contiguousMultilineTokens'; +import { SparseMultilineTokens } from 'vs/editor/common/model/tokens/sparseMultilineTokens'; import { TextChange } from 'vs/editor/common/model/textChange'; import { equals } from 'vs/base/common/objects'; import { IBracketPairsTextModelPart } from 'vs/editor/common/model/bracketPairsTextModelPart/bracketPairs'; @@ -875,17 +876,17 @@ export interface ITextModel { /** * @internal */ - setTokens(tokens: MultilineTokens[]): void; + setTokens(tokens: ContiguousMultilineTokens[]): void; /** * @internal */ - setSemanticTokens(tokens: MultilineTokens2[] | null, isComplete: boolean): void; + setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void; /** * @internal */ - setPartialSemanticTokens(range: Range, tokens: MultilineTokens2[] | null): void; + setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[] | null): void; /** * @internal diff --git a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsImpl.ts b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsImpl.ts index 34d2e5c1426..d2ab3e97974 100644 --- a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsImpl.ts +++ b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsImpl.ts @@ -5,7 +5,7 @@ import { Emitter } from 'vs/base/common/event'; import { Disposable, DisposableStore, IDisposable, IReference, MutableDisposable } from 'vs/base/common/lifecycle'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { IPosition, Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; import { BracketPairsTree } from 'vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/bracketPairsTree'; diff --git a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts index 3dadff93291..e980d3c4322 100644 --- a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts +++ b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { NotSupportedError } from 'vs/base/common/errors'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { ITextModel } from 'vs/editor/common/model'; import { SmallImmutableSet } from './smallImmutableSet'; import { StandardTokenType, TokenMetadata } from 'vs/editor/common/modes'; diff --git a/src/vs/editor/common/model/pieceTreeTextBuffer/eolCounter.ts b/src/vs/editor/common/model/pieceTreeTextBuffer/eolCounter.ts new file mode 100644 index 00000000000..702b562cbed --- /dev/null +++ b/src/vs/editor/common/model/pieceTreeTextBuffer/eolCounter.ts @@ -0,0 +1,51 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CharCode } from 'vs/base/common/charCode'; + +export const enum StringEOL { + Unknown = 0, + Invalid = 3, + LF = 1, + CRLF = 2 +} + +export function countEOL(text: string): [number, number, number, StringEOL] { + let eolCount = 0; + let firstLineLength = 0; + let lastLineStart = 0; + let eol: StringEOL = StringEOL.Unknown; + for (let i = 0, len = text.length; i < len; i++) { + const chr = text.charCodeAt(i); + + if (chr === CharCode.CarriageReturn) { + if (eolCount === 0) { + firstLineLength = i; + } + eolCount++; + if (i + 1 < len && text.charCodeAt(i + 1) === CharCode.LineFeed) { + // \r\n... case + eol |= StringEOL.CRLF; + i++; // skip \n + } else { + // \r... case + eol |= StringEOL.Invalid; + } + lastLineStart = i + 1; + } else if (chr === CharCode.LineFeed) { + // \n... case + eol |= StringEOL.LF; + if (eolCount === 0) { + firstLineLength = i; + } + eolCount++; + lastLineStart = i + 1; + } + } + if (eolCount === 0) { + firstLineLength = text.length; + } + return [eolCount, firstLineLength, text.length - lastLineStart, eol]; +} diff --git a/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts b/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts index a57fd663d7a..eaf88b1b926 100644 --- a/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts +++ b/src/vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.ts @@ -10,7 +10,7 @@ import { Range } from 'vs/editor/common/core/range'; import { ApplyEditsResult, EndOfLinePreference, FindMatch, IInternalModelContentChange, ISingleEditOperationIdentifier, ITextBuffer, ITextSnapshot, ValidAnnotatedEditOperation, IValidEditOperation } from 'vs/editor/common/model'; import { PieceTreeBase, StringBuffer } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeBase'; import { SearchData } from 'vs/editor/common/model/textModelSearch'; -import { countEOL, StringEOL } from 'vs/editor/common/model/tokensStore'; +import { countEOL, StringEOL } from 'vs/editor/common/model/pieceTreeTextBuffer/eolCounter'; import { TextChange } from 'vs/editor/common/model/textChange'; import { Disposable } from 'vs/base/common/lifecycle'; diff --git a/src/vs/editor/common/model/textModel.ts b/src/vs/editor/common/model/textModel.ts index 0a2685131bb..aca2d8fa692 100644 --- a/src/vs/editor/common/model/textModel.ts +++ b/src/vs/editor/common/model/textModel.ts @@ -16,7 +16,7 @@ import * as strings from 'vs/base/common/strings'; import { Constants } from 'vs/base/common/uint'; import { URI } from 'vs/base/common/uri'; import { EDITOR_MODEL_DEFAULTS } from 'vs/editor/common/config/editorOptions'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { IPosition, Position } from 'vs/editor/common/core/position'; import { IRange, Range } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; @@ -35,7 +35,11 @@ import { TextChange } from 'vs/editor/common/model/textChange'; import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, InternalModelContentChangeEvent, LineInjectedText, ModelInjectedTextChangedEvent, ModelRawChange, ModelRawContentChangedEvent, ModelRawEOLChanged, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents'; import { SearchData, SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch'; import { TextModelTokenization } from 'vs/editor/common/model/textModelTokens'; -import { countEOL, MultilineTokens, MultilineTokens2, TokensStore, TokensStore2 } from 'vs/editor/common/model/tokensStore'; +import { countEOL } from 'vs/editor/common/model/pieceTreeTextBuffer/eolCounter'; +import { ContiguousMultilineTokens } from 'vs/editor/common/model/tokens/contiguousMultilineTokens'; +import { SparseMultilineTokens } from 'vs/editor/common/model/tokens/sparseMultilineTokens'; +import { ContiguousTokensStore } from 'vs/editor/common/model/tokens/contiguousTokensStore'; +import { SparseTokensStore } from 'vs/editor/common/model/tokens/sparseTokensStore'; import { getWordAtText } from 'vs/editor/common/model/wordHelper'; import { FormattingOptions, StandardTokenType } from 'vs/editor/common/modes'; import { ILanguageConfigurationService, ResolvedLanguageConfiguration } from 'vs/editor/common/modes/languageConfigurationRegistry'; @@ -293,8 +297,8 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati //#region Tokenization private _languageId: string; private readonly _languageRegistryListener: IDisposable; - private readonly _tokens: TokensStore; - private readonly _tokens2: TokensStore2; + private readonly _tokens: ContiguousTokensStore; + private readonly _semanticTokens: SparseTokensStore; private readonly _tokenization: TextModelTokenization; //#endregion @@ -399,8 +403,8 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati this._isRedoing = false; this._trimAutoWhitespaceLines = null; - this._tokens = new TokensStore(this._languageService.languageIdCodec); - this._tokens2 = new TokensStore2(this._languageService.languageIdCodec); + this._tokens = new ContiguousTokensStore(this._languageService.languageIdCodec); + this._semanticTokens = new SparseTokensStore(this._languageService.languageIdCodec); this._tokenization = new TextModelTokenization(this, this._languageService.languageIdCodec); this._bracketPairColorizer = this._register(new BracketPairsTextModelPart(this, this._languageConfigurationService)); @@ -496,7 +500,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati // Flush all tokens this._tokens.flush(); - this._tokens2.flush(); + this._semanticTokens.flush(); // Destroy all my decorations this._decorations = Object.create(null); @@ -1476,7 +1480,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati const change = contentChanges[i]; const [eolCount, firstLineLength, lastLineLength] = countEOL(change.text); this._tokens.acceptEdit(change.range, eolCount, firstLineLength); - this._tokens2.acceptEdit(change.range, eolCount, firstLineLength, lastLineLength, change.text.length > 0 ? change.text.charCodeAt(0) : CharCode.Null); + this._semanticTokens.acceptEdit(change.range, eolCount, firstLineLength, lastLineLength, change.text.length > 0 ? change.text.charCodeAt(0) : CharCode.Null); this._decorationsTree.acceptReplace(change.rangeOffset, change.rangeLength, change.text.length, change.forceMoveMarkers); } @@ -1964,7 +1968,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), tokens, false); } - public setTokens(tokens: MultilineTokens[], backgroundTokenizationCompleted: boolean = false): void { + public setTokens(tokens: ContiguousMultilineTokens[], backgroundTokenizationCompleted: boolean = false): void { if (tokens.length !== 0) { const ranges: { fromLineNumber: number; toLineNumber: number; }[] = []; @@ -1973,13 +1977,12 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati let minChangedLineNumber = 0; let maxChangedLineNumber = 0; let hasChange = false; - for (let j = 0, lenJ = element.tokens.length; j < lenJ; j++) { - const lineNumber = element.startLineNumber + j; + for (let lineNumber = element.startLineNumber; lineNumber <= element.endLineNumber; lineNumber++) { if (hasChange) { - this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.tokens[j], false); + this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.getLineTokens(lineNumber), false); maxChangedLineNumber = lineNumber; } else { - const lineHasChange = this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.tokens[j], true); + const lineHasChange = this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.getLineTokens(lineNumber), true); if (lineHasChange) { hasChange = true; minChangedLineNumber = lineNumber; @@ -2003,8 +2006,8 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati this.handleTokenizationProgress(backgroundTokenizationCompleted); } - public setSemanticTokens(tokens: MultilineTokens2[] | null, isComplete: boolean): void { - this._tokens2.set(tokens, isComplete); + public setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void { + this._semanticTokens.set(tokens, isComplete); this._emitModelTokensChangedEvent({ tokenizationSupportChanged: false, @@ -2014,18 +2017,18 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati } public hasCompleteSemanticTokens(): boolean { - return this._tokens2.isComplete(); + return this._semanticTokens.isComplete(); } public hasSomeSemanticTokens(): boolean { - return !this._tokens2.isEmpty(); + return !this._semanticTokens.isEmpty(); } - public setPartialSemanticTokens(range: Range, tokens: MultilineTokens2[]): void { + public setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[]): void { if (this.hasCompleteSemanticTokens()) { return; } - const changedRange = this._tokens2.setPartial(range, tokens); + const changedRange = this._semanticTokens.setPartial(range, tokens); this._emitModelTokensChangedEvent({ tokenizationSupportChanged: false, @@ -2053,7 +2056,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati } public clearSemanticTokens(): void { - this._tokens2.flush(); + this._semanticTokens.flush(); this._emitModelTokensChangedEvent({ tokenizationSupportChanged: false, @@ -2101,7 +2104,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati private _getLineTokens(lineNumber: number): LineTokens { const lineText = this.getLineContent(lineNumber); const syntacticTokens = this._tokens.getTokens(this._languageId, lineNumber - 1, lineText); - return this._tokens2.addSemanticTokens(lineNumber, syntacticTokens); + return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens); } public getLanguageId(): string { diff --git a/src/vs/editor/common/model/textModelTokens.ts b/src/vs/editor/common/model/textModelTokens.ts index f944e5e5f1d..c0f8653280c 100644 --- a/src/vs/editor/common/model/textModelTokens.ts +++ b/src/vs/editor/common/model/textModelTokens.ts @@ -5,7 +5,7 @@ import * as arrays from 'vs/base/common/arrays'; import { onUnexpectedError } from 'vs/base/common/errors'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { Position } from 'vs/editor/common/core/position'; import { IRange } from 'vs/editor/common/core/range'; import { EncodedTokenizationResult } from 'vs/editor/common/core/token'; @@ -14,7 +14,8 @@ import { nullTokenizeEncoded } from 'vs/editor/common/modes/nullMode'; import { TextModel } from 'vs/editor/common/model/textModel'; import { Disposable } from 'vs/base/common/lifecycle'; import { StopWatch } from 'vs/base/common/stopwatch'; -import { MultilineTokensBuilder, countEOL } from 'vs/editor/common/model/tokensStore'; +import { countEOL } from 'vs/editor/common/model/pieceTreeTextBuffer/eolCounter'; +import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/model/tokens/contiguousMultilineTokensBuilder'; import { runWhenIdle, IdleDeadline } from 'vs/base/common/async'; import { setTimeout0 } from 'vs/base/common/platform'; @@ -303,7 +304,7 @@ export class TextModelTokenization extends Disposable { */ private _backgroundTokenizeForAtLeast1ms(): void { const lineCount = this._textModel.getLineCount(); - const builder = new MultilineTokensBuilder(); + const builder = new ContiguousMultilineTokensBuilder(); const sw = StopWatch.create(false); do { @@ -321,13 +322,13 @@ export class TextModelTokenization extends Disposable { } } while (this._hasLinesToTokenize()); - this._textModel.setTokens(builder.tokens, !this._hasLinesToTokenize()); + this._textModel.setTokens(builder.finalize(), !this._hasLinesToTokenize()); } public tokenizeViewport(startLineNumber: number, endLineNumber: number): void { - const builder = new MultilineTokensBuilder(); + const builder = new ContiguousMultilineTokensBuilder(); this._tokenizeViewport(builder, startLineNumber, endLineNumber); - this._textModel.setTokens(builder.tokens, !this._hasLinesToTokenize()); + this._textModel.setTokens(builder.finalize(), !this._hasLinesToTokenize()); } public reset(): void { @@ -336,9 +337,9 @@ export class TextModelTokenization extends Disposable { } public forceTokenization(lineNumber: number): void { - const builder = new MultilineTokensBuilder(); + const builder = new ContiguousMultilineTokensBuilder(); this._updateTokensUntilLine(builder, lineNumber); - this._textModel.setTokens(builder.tokens, !this._hasLinesToTokenize()); + this._textModel.setTokens(builder.finalize(), !this._hasLinesToTokenize()); } public getTokenTypeIfInsertingCharacter(position: Position, character: string): StandardTokenType { @@ -400,7 +401,7 @@ export class TextModelTokenization extends Disposable { return (this._tokenizationStateStore.invalidLineStartIndex < this._textModel.getLineCount()); } - private _tokenizeOneInvalidLine(builder: MultilineTokensBuilder): number { + private _tokenizeOneInvalidLine(builder: ContiguousMultilineTokensBuilder): number { if (!this._hasLinesToTokenize()) { return this._textModel.getLineCount() + 1; } @@ -409,7 +410,7 @@ export class TextModelTokenization extends Disposable { return lineNumber; } - private _updateTokensUntilLine(builder: MultilineTokensBuilder, lineNumber: number): void { + private _updateTokensUntilLine(builder: ContiguousMultilineTokensBuilder, lineNumber: number): void { if (!this._tokenizationSupport) { return; } @@ -429,7 +430,7 @@ export class TextModelTokenization extends Disposable { } } - private _tokenizeViewport(builder: MultilineTokensBuilder, startLineNumber: number, endLineNumber: number): void { + private _tokenizeViewport(builder: ContiguousMultilineTokensBuilder, startLineNumber: number, endLineNumber: number): void { if (!this._tokenizationSupport) { // nothing to do return; diff --git a/src/vs/editor/common/model/tokens/contiguousMultilineTokens.ts b/src/vs/editor/common/model/tokens/contiguousMultilineTokens.ts new file mode 100644 index 00000000000..b54299359df --- /dev/null +++ b/src/vs/editor/common/model/tokens/contiguousMultilineTokens.ts @@ -0,0 +1,225 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as arrays from 'vs/base/common/arrays'; +import { readUInt32BE, writeUInt32BE } from 'vs/base/common/buffer'; +import { Position } from 'vs/editor/common/core/position'; +import { IRange } from 'vs/editor/common/core/range'; +import { countEOL } from 'vs/editor/common/model/pieceTreeTextBuffer/eolCounter'; +import { ContiguousTokensEditing } from 'vs/editor/common/model/tokens/contiguousTokensEditing'; + +/** + * Represents contiguous tokens over a contiguous range of lines. + */ +export class ContiguousMultilineTokens { + + public static deserialize(buff: Uint8Array, offset: number, result: ContiguousMultilineTokens[]): number { + const view32 = new Uint32Array(buff.buffer); + const startLineNumber = readUInt32BE(buff, offset); offset += 4; + const count = readUInt32BE(buff, offset); offset += 4; + const tokens: Uint32Array[] = []; + for (let i = 0; i < count; i++) { + const byteCount = readUInt32BE(buff, offset); offset += 4; + tokens.push(view32.subarray(offset / 4, offset / 4 + byteCount / 4)); + offset += byteCount; + } + result.push(new ContiguousMultilineTokens(startLineNumber, tokens)); + return offset; + } + + /** + * The start line number for this block of tokens. + */ + private _startLineNumber: number; + + /** + * The tokens are stored in a binary format. There is an element for each line, + * so `tokens[index]` contains all tokens on line `startLineNumber + index`. + * + * On a specific line, each token occupies two array indices. For token i: + * - at offset 2*i => endOffset + * - at offset 2*i + 1 => metadata + * + */ + private _tokens: (Uint32Array | ArrayBuffer | null)[]; + + /** + * (Inclusive) start line number for these tokens. + */ + public get startLineNumber(): number { + return this._startLineNumber; + } + + /** + * (Inclusive) end line number for these tokens. + */ + public get endLineNumber(): number { + return this._startLineNumber + this._tokens.length - 1; + } + + constructor(startLineNumber: number, tokens: Uint32Array[]) { + this._startLineNumber = startLineNumber; + this._tokens = tokens; + } + + /** + * @see {@link _tokens} + */ + public getLineTokens(lineNumber: number): Uint32Array | ArrayBuffer | null { + return this._tokens[lineNumber - this._startLineNumber]; + } + + public appendLineTokens(lineTokens: Uint32Array): void { + this._tokens.push(lineTokens); + } + + public serializeSize(): number { + let result = 0; + result += 4; // 4 bytes for the start line number + result += 4; // 4 bytes for the line count + for (let i = 0; i < this._tokens.length; i++) { + const lineTokens = this._tokens[i]; + if (!(lineTokens instanceof Uint32Array)) { + throw new Error(`Not supported!`); + } + result += 4; // 4 bytes for the byte count + result += lineTokens.byteLength; + } + return result; + } + + public serialize(destination: Uint8Array, offset: number): number { + writeUInt32BE(destination, this._startLineNumber, offset); offset += 4; + writeUInt32BE(destination, this._tokens.length, offset); offset += 4; + for (let i = 0; i < this._tokens.length; i++) { + const lineTokens = this._tokens[i]; + if (!(lineTokens instanceof Uint32Array)) { + throw new Error(`Not supported!`); + } + writeUInt32BE(destination, lineTokens.byteLength, offset); offset += 4; + destination.set(new Uint8Array(lineTokens.buffer), offset); offset += lineTokens.byteLength; + } + return offset; + } + + public applyEdit(range: IRange, text: string): void { + const [eolCount, firstLineLength] = countEOL(text); + this._acceptDeleteRange(range); + this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength); + } + + private _acceptDeleteRange(range: IRange): void { + if (range.startLineNumber === range.endLineNumber && range.startColumn === range.endColumn) { + // Nothing to delete + return; + } + + const firstLineIndex = range.startLineNumber - this._startLineNumber; + const lastLineIndex = range.endLineNumber - this._startLineNumber; + + if (lastLineIndex < 0) { + // this deletion occurs entirely before this block, so we only need to adjust line numbers + const deletedLinesCount = lastLineIndex - firstLineIndex; + this._startLineNumber -= deletedLinesCount; + return; + } + + if (firstLineIndex >= this._tokens.length) { + // this deletion occurs entirely after this block, so there is nothing to do + return; + } + + if (firstLineIndex < 0 && lastLineIndex >= this._tokens.length) { + // this deletion completely encompasses this block + this._startLineNumber = 0; + this._tokens = []; + return; + } + + if (firstLineIndex === lastLineIndex) { + // a delete on a single line + this._tokens[firstLineIndex] = ContiguousTokensEditing.delete(this._tokens[firstLineIndex], range.startColumn - 1, range.endColumn - 1); + return; + } + + if (firstLineIndex >= 0) { + // The first line survives + this._tokens[firstLineIndex] = ContiguousTokensEditing.deleteEnding(this._tokens[firstLineIndex], range.startColumn - 1); + + if (lastLineIndex < this._tokens.length) { + // The last line survives + const lastLineTokens = ContiguousTokensEditing.deleteBeginning(this._tokens[lastLineIndex], range.endColumn - 1); + + // Take remaining text on last line and append it to remaining text on first line + this._tokens[firstLineIndex] = ContiguousTokensEditing.append(this._tokens[firstLineIndex], lastLineTokens); + + // Delete middle lines + this._tokens.splice(firstLineIndex + 1, lastLineIndex - firstLineIndex); + } else { + // The last line does not survive + + // Take remaining text on last line and append it to remaining text on first line + this._tokens[firstLineIndex] = ContiguousTokensEditing.append(this._tokens[firstLineIndex], null); + + // Delete lines + this._tokens = this._tokens.slice(0, firstLineIndex + 1); + } + } else { + // The first line does not survive + + const deletedBefore = -firstLineIndex; + this._startLineNumber -= deletedBefore; + + // Remove beginning from last line + this._tokens[lastLineIndex] = ContiguousTokensEditing.deleteBeginning(this._tokens[lastLineIndex], range.endColumn - 1); + + // Delete lines + this._tokens = this._tokens.slice(lastLineIndex); + } + } + + private _acceptInsertText(position: Position, eolCount: number, firstLineLength: number): void { + + if (eolCount === 0 && firstLineLength === 0) { + // Nothing to insert + return; + } + + const lineIndex = position.lineNumber - this._startLineNumber; + + if (lineIndex < 0) { + // this insertion occurs before this block, so we only need to adjust line numbers + this._startLineNumber += eolCount; + return; + } + + if (lineIndex >= this._tokens.length) { + // this insertion occurs after this block, so there is nothing to do + return; + } + + if (eolCount === 0) { + // Inserting text on one line + this._tokens[lineIndex] = ContiguousTokensEditing.insert(this._tokens[lineIndex], position.column - 1, firstLineLength); + return; + } + + this._tokens[lineIndex] = ContiguousTokensEditing.deleteEnding(this._tokens[lineIndex], position.column - 1); + this._tokens[lineIndex] = ContiguousTokensEditing.insert(this._tokens[lineIndex], position.column - 1, firstLineLength); + + this._insertLines(position.lineNumber, eolCount); + } + + private _insertLines(insertIndex: number, insertCount: number): void { + if (insertCount === 0) { + return; + } + const lineTokens: (Uint32Array | ArrayBuffer | null)[] = []; + for (let i = 0; i < insertCount; i++) { + lineTokens[i] = null; + } + this._tokens = arrays.arrayInsert(this._tokens, insertIndex, lineTokens); + } +} diff --git a/src/vs/editor/common/model/tokens/contiguousMultilineTokensBuilder.ts b/src/vs/editor/common/model/tokens/contiguousMultilineTokensBuilder.ts new file mode 100644 index 00000000000..809c98afedf --- /dev/null +++ b/src/vs/editor/common/model/tokens/contiguousMultilineTokensBuilder.ts @@ -0,0 +1,66 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { readUInt32BE, writeUInt32BE } from 'vs/base/common/buffer'; +import { ContiguousMultilineTokens } from 'vs/editor/common/model/tokens/contiguousMultilineTokens'; + +export class ContiguousMultilineTokensBuilder { + + public static deserialize(buff: Uint8Array): ContiguousMultilineTokens[] { + let offset = 0; + const count = readUInt32BE(buff, offset); offset += 4; + const result: ContiguousMultilineTokens[] = []; + for (let i = 0; i < count; i++) { + offset = ContiguousMultilineTokens.deserialize(buff, offset, result); + } + return result; + } + + private readonly _tokens: ContiguousMultilineTokens[]; + + constructor() { + this._tokens = []; + } + + public add(lineNumber: number, lineTokens: Uint32Array): void { + if (this._tokens.length > 0) { + const last = this._tokens[this._tokens.length - 1]; + if (last.endLineNumber + 1 === lineNumber) { + // append + last.appendLineTokens(lineTokens); + return; + } + } + this._tokens.push(new ContiguousMultilineTokens(lineNumber, [lineTokens])); + } + + public finalize(): ContiguousMultilineTokens[] { + return this._tokens; + } + + public serialize(): Uint8Array { + const size = this._serializeSize(); + const result = new Uint8Array(size); + this._serialize(result); + return result; + } + + private _serializeSize(): number { + let result = 0; + result += 4; // 4 bytes for the count + for (let i = 0; i < this._tokens.length; i++) { + result += this._tokens[i].serializeSize(); + } + return result; + } + + private _serialize(destination: Uint8Array): void { + let offset = 0; + writeUInt32BE(destination, this._tokens.length, offset); offset += 4; + for (let i = 0; i < this._tokens.length; i++) { + offset = this._tokens[i].serialize(destination, offset); + } + } +} diff --git a/src/vs/editor/common/model/tokens/contiguousTokensEditing.ts b/src/vs/editor/common/model/tokens/contiguousTokensEditing.ts new file mode 100644 index 00000000000..cd91aa5f678 --- /dev/null +++ b/src/vs/editor/common/model/tokens/contiguousTokensEditing.ts @@ -0,0 +1,144 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; + +export const EMPTY_LINE_TOKENS = (new Uint32Array(0)).buffer; + +export class ContiguousTokensEditing { + + public static deleteBeginning(lineTokens: Uint32Array | ArrayBuffer | null, toChIndex: number): Uint32Array | ArrayBuffer | null { + if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) { + return lineTokens; + } + return ContiguousTokensEditing.delete(lineTokens, 0, toChIndex); + } + + public static deleteEnding(lineTokens: Uint32Array | ArrayBuffer | null, fromChIndex: number): Uint32Array | ArrayBuffer | null { + if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) { + return lineTokens; + } + + const tokens = toUint32Array(lineTokens); + const lineTextLength = tokens[tokens.length - 2]; + return ContiguousTokensEditing.delete(lineTokens, fromChIndex, lineTextLength); + } + + public static delete(lineTokens: Uint32Array | ArrayBuffer | null, fromChIndex: number, toChIndex: number): Uint32Array | ArrayBuffer | null { + if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS || fromChIndex === toChIndex) { + return lineTokens; + } + + const tokens = toUint32Array(lineTokens); + const tokensCount = (tokens.length >>> 1); + + // special case: deleting everything + if (fromChIndex === 0 && tokens[tokens.length - 2] === toChIndex) { + return EMPTY_LINE_TOKENS; + } + + const fromTokenIndex = LineTokens.findIndexInTokensArray(tokens, fromChIndex); + const fromTokenStartOffset = (fromTokenIndex > 0 ? tokens[(fromTokenIndex - 1) << 1] : 0); + const fromTokenEndOffset = tokens[fromTokenIndex << 1]; + + if (toChIndex < fromTokenEndOffset) { + // the delete range is inside a single token + const delta = (toChIndex - fromChIndex); + for (let i = fromTokenIndex; i < tokensCount; i++) { + tokens[i << 1] -= delta; + } + return lineTokens; + } + + let dest: number; + let lastEnd: number; + if (fromTokenStartOffset !== fromChIndex) { + tokens[fromTokenIndex << 1] = fromChIndex; + dest = ((fromTokenIndex + 1) << 1); + lastEnd = fromChIndex; + } else { + dest = (fromTokenIndex << 1); + lastEnd = fromTokenStartOffset; + } + + const delta = (toChIndex - fromChIndex); + for (let tokenIndex = fromTokenIndex + 1; tokenIndex < tokensCount; tokenIndex++) { + const tokenEndOffset = tokens[tokenIndex << 1] - delta; + if (tokenEndOffset > lastEnd) { + tokens[dest++] = tokenEndOffset; + tokens[dest++] = tokens[(tokenIndex << 1) + 1]; + lastEnd = tokenEndOffset; + } + } + + if (dest === tokens.length) { + // nothing to trim + return lineTokens; + } + + const tmp = new Uint32Array(dest); + tmp.set(tokens.subarray(0, dest), 0); + return tmp.buffer; + } + + public static append(lineTokens: Uint32Array | ArrayBuffer | null, _otherTokens: Uint32Array | ArrayBuffer | null): Uint32Array | ArrayBuffer | null { + if (_otherTokens === EMPTY_LINE_TOKENS) { + return lineTokens; + } + if (lineTokens === EMPTY_LINE_TOKENS) { + return _otherTokens; + } + if (lineTokens === null) { + return lineTokens; + } + if (_otherTokens === null) { + // cannot determine combined line length... + return null; + } + const myTokens = toUint32Array(lineTokens); + const otherTokens = toUint32Array(_otherTokens); + const otherTokensCount = (otherTokens.length >>> 1); + + const result = new Uint32Array(myTokens.length + otherTokens.length); + result.set(myTokens, 0); + let dest = myTokens.length; + const delta = myTokens[myTokens.length - 2]; + for (let i = 0; i < otherTokensCount; i++) { + result[dest++] = otherTokens[(i << 1)] + delta; + result[dest++] = otherTokens[(i << 1) + 1]; + } + return result.buffer; + } + + public static insert(lineTokens: Uint32Array | ArrayBuffer | null, chIndex: number, textLength: number): Uint32Array | ArrayBuffer | null { + if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) { + // nothing to do + return lineTokens; + } + + const tokens = toUint32Array(lineTokens); + const tokensCount = (tokens.length >>> 1); + + let fromTokenIndex = LineTokens.findIndexInTokensArray(tokens, chIndex); + if (fromTokenIndex > 0) { + const fromTokenStartOffset = tokens[(fromTokenIndex - 1) << 1]; + if (fromTokenStartOffset === chIndex) { + fromTokenIndex--; + } + } + for (let tokenIndex = fromTokenIndex; tokenIndex < tokensCount; tokenIndex++) { + tokens[tokenIndex << 1] += textLength; + } + return lineTokens; + } +} + +export function toUint32Array(arr: Uint32Array | ArrayBuffer): Uint32Array { + if (arr instanceof Uint32Array) { + return arr; + } else { + return new Uint32Array(arr); + } +} diff --git a/src/vs/editor/common/model/tokens/contiguousTokensStore.ts b/src/vs/editor/common/model/tokens/contiguousTokensStore.ts new file mode 100644 index 00000000000..fb450605dcb --- /dev/null +++ b/src/vs/editor/common/model/tokens/contiguousTokensStore.ts @@ -0,0 +1,215 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as arrays from 'vs/base/common/arrays'; +import { Position } from 'vs/editor/common/core/position'; +import { IRange } from 'vs/editor/common/core/range'; +import { ContiguousTokensEditing, EMPTY_LINE_TOKENS, toUint32Array } from 'vs/editor/common/model/tokens/contiguousTokensEditing'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; +import { ColorId, FontStyle, ILanguageIdCodec, LanguageId, MetadataConsts, StandardTokenType, TokenMetadata } from 'vs/editor/common/modes'; + +/** + * Represents contiguous tokens in a text model. + */ +export class ContiguousTokensStore { + private _lineTokens: (Uint32Array | ArrayBuffer | null)[]; + private _len: number; + private readonly _languageIdCodec: ILanguageIdCodec; + + constructor(languageIdCodec: ILanguageIdCodec) { + this._lineTokens = []; + this._len = 0; + this._languageIdCodec = languageIdCodec; + } + + public flush(): void { + this._lineTokens = []; + this._len = 0; + } + + public getTokens(topLevelLanguageId: string, lineIndex: number, lineText: string): LineTokens { + let rawLineTokens: Uint32Array | ArrayBuffer | null = null; + if (lineIndex < this._len) { + rawLineTokens = this._lineTokens[lineIndex]; + } + + if (rawLineTokens !== null && rawLineTokens !== EMPTY_LINE_TOKENS) { + return new LineTokens(toUint32Array(rawLineTokens), lineText, this._languageIdCodec); + } + + const lineTokens = new Uint32Array(2); + lineTokens[0] = lineText.length; + lineTokens[1] = getDefaultMetadata(this._languageIdCodec.encodeLanguageId(topLevelLanguageId)); + return new LineTokens(lineTokens, lineText, this._languageIdCodec); + } + + private static _massageTokens(topLevelLanguageId: LanguageId, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null): Uint32Array | ArrayBuffer { + + const tokens = _tokens ? toUint32Array(_tokens) : null; + + if (lineTextLength === 0) { + let hasDifferentLanguageId = false; + if (tokens && tokens.length > 1) { + hasDifferentLanguageId = (TokenMetadata.getLanguageId(tokens[1]) !== topLevelLanguageId); + } + + if (!hasDifferentLanguageId) { + return EMPTY_LINE_TOKENS; + } + } + + if (!tokens || tokens.length === 0) { + const tokens = new Uint32Array(2); + tokens[0] = lineTextLength; + tokens[1] = getDefaultMetadata(topLevelLanguageId); + return tokens.buffer; + } + + // Ensure the last token covers the end of the text + tokens[tokens.length - 2] = lineTextLength; + + if (tokens.byteOffset === 0 && tokens.byteLength === tokens.buffer.byteLength) { + // Store directly the ArrayBuffer pointer to save an object + return tokens.buffer; + } + return tokens; + } + + private _ensureLine(lineIndex: number): void { + while (lineIndex >= this._len) { + this._lineTokens[this._len] = null; + this._len++; + } + } + + private _deleteLines(start: number, deleteCount: number): void { + if (deleteCount === 0) { + return; + } + if (start + deleteCount > this._len) { + deleteCount = this._len - start; + } + this._lineTokens.splice(start, deleteCount); + this._len -= deleteCount; + } + + private _insertLines(insertIndex: number, insertCount: number): void { + if (insertCount === 0) { + return; + } + const lineTokens: (Uint32Array | ArrayBuffer | null)[] = []; + for (let i = 0; i < insertCount; i++) { + lineTokens[i] = null; + } + this._lineTokens = arrays.arrayInsert(this._lineTokens, insertIndex, lineTokens); + this._len += insertCount; + } + + public setTokens(topLevelLanguageId: string, lineIndex: number, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null, checkEquality: boolean): boolean { + const tokens = ContiguousTokensStore._massageTokens(this._languageIdCodec.encodeLanguageId(topLevelLanguageId), lineTextLength, _tokens); + this._ensureLine(lineIndex); + const oldTokens = this._lineTokens[lineIndex]; + this._lineTokens[lineIndex] = tokens; + + if (checkEquality) { + return !ContiguousTokensStore._equals(oldTokens, tokens); + } + return false; + } + + private static _equals(_a: Uint32Array | ArrayBuffer | null, _b: Uint32Array | ArrayBuffer | null) { + if (!_a || !_b) { + return !_a && !_b; + } + + const a = toUint32Array(_a); + const b = toUint32Array(_b); + + if (a.length !== b.length) { + return false; + } + for (let i = 0, len = a.length; i < len; i++) { + if (a[i] !== b[i]) { + return false; + } + } + return true; + } + + //#region Editing + + public acceptEdit(range: IRange, eolCount: number, firstLineLength: number): void { + this._acceptDeleteRange(range); + this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength); + } + + private _acceptDeleteRange(range: IRange): void { + + const firstLineIndex = range.startLineNumber - 1; + if (firstLineIndex >= this._len) { + return; + } + + if (range.startLineNumber === range.endLineNumber) { + if (range.startColumn === range.endColumn) { + // Nothing to delete + return; + } + + this._lineTokens[firstLineIndex] = ContiguousTokensEditing.delete(this._lineTokens[firstLineIndex], range.startColumn - 1, range.endColumn - 1); + return; + } + + this._lineTokens[firstLineIndex] = ContiguousTokensEditing.deleteEnding(this._lineTokens[firstLineIndex], range.startColumn - 1); + + const lastLineIndex = range.endLineNumber - 1; + let lastLineTokens: Uint32Array | ArrayBuffer | null = null; + if (lastLineIndex < this._len) { + lastLineTokens = ContiguousTokensEditing.deleteBeginning(this._lineTokens[lastLineIndex], range.endColumn - 1); + } + + // Take remaining text on last line and append it to remaining text on first line + this._lineTokens[firstLineIndex] = ContiguousTokensEditing.append(this._lineTokens[firstLineIndex], lastLineTokens); + + // Delete middle lines + this._deleteLines(range.startLineNumber, range.endLineNumber - range.startLineNumber); + } + + private _acceptInsertText(position: Position, eolCount: number, firstLineLength: number): void { + + if (eolCount === 0 && firstLineLength === 0) { + // Nothing to insert + return; + } + + const lineIndex = position.lineNumber - 1; + if (lineIndex >= this._len) { + return; + } + + if (eolCount === 0) { + // Inserting text on one line + this._lineTokens[lineIndex] = ContiguousTokensEditing.insert(this._lineTokens[lineIndex], position.column - 1, firstLineLength); + return; + } + + this._lineTokens[lineIndex] = ContiguousTokensEditing.deleteEnding(this._lineTokens[lineIndex], position.column - 1); + this._lineTokens[lineIndex] = ContiguousTokensEditing.insert(this._lineTokens[lineIndex], position.column - 1, firstLineLength); + + this._insertLines(position.lineNumber, eolCount); + } + + //#endregion +} + +function getDefaultMetadata(topLevelLanguageId: LanguageId): number { + return ( + (topLevelLanguageId << MetadataConsts.LANGUAGEID_OFFSET) + | (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET) + | (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET) + | (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET) + | (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET) + ) >>> 0; +} diff --git a/src/vs/editor/common/core/lineTokens.ts b/src/vs/editor/common/model/tokens/lineTokens.ts similarity index 98% rename from src/vs/editor/common/core/lineTokens.ts rename to src/vs/editor/common/model/tokens/lineTokens.ts index 7390a7fb19d..2e06aee6937 100644 --- a/src/vs/editor/common/core/lineTokens.ts +++ b/src/vs/editor/common/model/tokens/lineTokens.ts @@ -135,7 +135,7 @@ export class LineTokens implements IViewLineTokens { } public sliceAndInflate(startOffset: number, endOffset: number, deltaOffset: number): IViewLineTokens { - return new SlicedLineTokens(this, startOffset, endOffset, deltaOffset); + return new SliceLineTokens(this, startOffset, endOffset, deltaOffset); } public static convertToEndOffset(tokens: Uint32Array, lineTextLength: number): void { @@ -220,7 +220,7 @@ export class LineTokens implements IViewLineTokens { } } -export class SlicedLineTokens implements IViewLineTokens { +class SliceLineTokens implements IViewLineTokens { private readonly _source: LineTokens; private readonly _startOffset: number; @@ -252,7 +252,7 @@ export class SlicedLineTokens implements IViewLineTokens { } public equals(other: IViewLineTokens): boolean { - if (other instanceof SlicedLineTokens) { + if (other instanceof SliceLineTokens) { return ( this._startOffset === other._startOffset && this._endOffset === other._endOffset diff --git a/src/vs/editor/common/model/tokens/sparseMultilineTokens.ts b/src/vs/editor/common/model/tokens/sparseMultilineTokens.ts new file mode 100644 index 00000000000..8468a05c26c --- /dev/null +++ b/src/vs/editor/common/model/tokens/sparseMultilineTokens.ts @@ -0,0 +1,594 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { CharCode } from 'vs/base/common/charCode'; +import { Position } from 'vs/editor/common/core/position'; +import { IRange, Range } from 'vs/editor/common/core/range'; +import { countEOL } from 'vs/editor/common/model/pieceTreeTextBuffer/eolCounter'; + +/** + * Represents sparse tokens over a contiguous range of lines. + */ +export class SparseMultilineTokens { + + public static create(startLineNumber: number, tokens: Uint32Array): SparseMultilineTokens { + return new SparseMultilineTokens(startLineNumber, new SparseMultilineTokensStorage(tokens)); + } + + private _startLineNumber: number; + private _endLineNumber: number; + private readonly _tokens: SparseMultilineTokensStorage; + + /** + * (Inclusive) start line number for these tokens. + */ + public get startLineNumber(): number { + return this._startLineNumber; + } + + /** + * (Inclusive) end line number for these tokens. + */ + public get endLineNumber(): number { + return this._endLineNumber; + } + + private constructor(startLineNumber: number, tokens: SparseMultilineTokensStorage) { + this._startLineNumber = startLineNumber; + this._tokens = tokens; + this._endLineNumber = this._startLineNumber + this._tokens.getMaxDeltaLine(); + } + + public toString(): string { + return this._tokens.toString(this._startLineNumber); + } + + private _updateEndLineNumber(): void { + this._endLineNumber = this._startLineNumber + this._tokens.getMaxDeltaLine(); + } + + public isEmpty(): boolean { + return this._tokens.isEmpty(); + } + + public getLineTokens(lineNumber: number): SparseLineTokens | null { + if (this._startLineNumber <= lineNumber && lineNumber <= this._endLineNumber) { + return this._tokens.getLineTokens(lineNumber - this._startLineNumber); + } + return null; + } + + public getRange(): Range | null { + const deltaRange = this._tokens.getRange(); + if (!deltaRange) { + return deltaRange; + } + return new Range(this._startLineNumber + deltaRange.startLineNumber, deltaRange.startColumn, this._startLineNumber + deltaRange.endLineNumber, deltaRange.endColumn); + } + + public removeTokens(range: Range): void { + const startLineIndex = range.startLineNumber - this._startLineNumber; + const endLineIndex = range.endLineNumber - this._startLineNumber; + + this._startLineNumber += this._tokens.removeTokens(startLineIndex, range.startColumn - 1, endLineIndex, range.endColumn - 1); + this._updateEndLineNumber(); + } + + public split(range: Range): [SparseMultilineTokens, SparseMultilineTokens] { + // split tokens to two: + // a) all the tokens before `range` + // b) all the tokens after `range` + const startLineIndex = range.startLineNumber - this._startLineNumber; + const endLineIndex = range.endLineNumber - this._startLineNumber; + + const [a, b, bDeltaLine] = this._tokens.split(startLineIndex, range.startColumn - 1, endLineIndex, range.endColumn - 1); + return [new SparseMultilineTokens(this._startLineNumber, a), new SparseMultilineTokens(this._startLineNumber + bDeltaLine, b)]; + } + + public applyEdit(range: IRange, text: string): void { + const [eolCount, firstLineLength, lastLineLength] = countEOL(text); + this.acceptEdit(range, eolCount, firstLineLength, lastLineLength, text.length > 0 ? text.charCodeAt(0) : CharCode.Null); + } + + public acceptEdit(range: IRange, eolCount: number, firstLineLength: number, lastLineLength: number, firstCharCode: number): void { + this._acceptDeleteRange(range); + this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength, lastLineLength, firstCharCode); + this._updateEndLineNumber(); + } + + private _acceptDeleteRange(range: IRange): void { + if (range.startLineNumber === range.endLineNumber && range.startColumn === range.endColumn) { + // Nothing to delete + return; + } + + const firstLineIndex = range.startLineNumber - this._startLineNumber; + const lastLineIndex = range.endLineNumber - this._startLineNumber; + + if (lastLineIndex < 0) { + // this deletion occurs entirely before this block, so we only need to adjust line numbers + const deletedLinesCount = lastLineIndex - firstLineIndex; + this._startLineNumber -= deletedLinesCount; + return; + } + + const tokenMaxDeltaLine = this._tokens.getMaxDeltaLine(); + + if (firstLineIndex >= tokenMaxDeltaLine + 1) { + // this deletion occurs entirely after this block, so there is nothing to do + return; + } + + if (firstLineIndex < 0 && lastLineIndex >= tokenMaxDeltaLine + 1) { + // this deletion completely encompasses this block + this._startLineNumber = 0; + this._tokens.clear(); + return; + } + + if (firstLineIndex < 0) { + const deletedBefore = -firstLineIndex; + this._startLineNumber -= deletedBefore; + + this._tokens.acceptDeleteRange(range.startColumn - 1, 0, 0, lastLineIndex, range.endColumn - 1); + } else { + this._tokens.acceptDeleteRange(0, firstLineIndex, range.startColumn - 1, lastLineIndex, range.endColumn - 1); + } + } + + private _acceptInsertText(position: Position, eolCount: number, firstLineLength: number, lastLineLength: number, firstCharCode: number): void { + + if (eolCount === 0 && firstLineLength === 0) { + // Nothing to insert + return; + } + + const lineIndex = position.lineNumber - this._startLineNumber; + + if (lineIndex < 0) { + // this insertion occurs before this block, so we only need to adjust line numbers + this._startLineNumber += eolCount; + return; + } + + const tokenMaxDeltaLine = this._tokens.getMaxDeltaLine(); + + if (lineIndex >= tokenMaxDeltaLine + 1) { + // this insertion occurs after this block, so there is nothing to do + return; + } + + this._tokens.acceptInsertText(lineIndex, position.column - 1, eolCount, firstLineLength, lastLineLength, firstCharCode); + } +} + +class SparseMultilineTokensStorage { + /** + * The encoding of tokens is: + * 4*i deltaLine (from `startLineNumber`) + * 4*i+1 startCharacter (from the line start) + * 4*i+2 endCharacter (from the line start) + * 4*i+3 metadata + */ + private readonly _tokens: Uint32Array; + private _tokenCount: number; + + constructor(tokens: Uint32Array) { + this._tokens = tokens; + this._tokenCount = tokens.length / 4; + } + + public toString(startLineNumber: number): string { + const pieces: string[] = []; + for (let i = 0; i < this._tokenCount; i++) { + pieces.push(`(${this._getDeltaLine(i) + startLineNumber},${this._getStartCharacter(i)}-${this._getEndCharacter(i)})`); + } + return `[${pieces.join(',')}]`; + } + + public getMaxDeltaLine(): number { + const tokenCount = this._getTokenCount(); + if (tokenCount === 0) { + return -1; + } + return this._getDeltaLine(tokenCount - 1); + } + + public getRange(): Range | null { + const tokenCount = this._getTokenCount(); + if (tokenCount === 0) { + return null; + } + const startChar = this._getStartCharacter(0); + const maxDeltaLine = this._getDeltaLine(tokenCount - 1); + const endChar = this._getEndCharacter(tokenCount - 1); + return new Range(0, startChar + 1, maxDeltaLine, endChar + 1); + } + + private _getTokenCount(): number { + return this._tokenCount; + } + + private _getDeltaLine(tokenIndex: number): number { + return this._tokens[4 * tokenIndex]; + } + + private _getStartCharacter(tokenIndex: number): number { + return this._tokens[4 * tokenIndex + 1]; + } + + private _getEndCharacter(tokenIndex: number): number { + return this._tokens[4 * tokenIndex + 2]; + } + + public isEmpty(): boolean { + return (this._getTokenCount() === 0); + } + + public getLineTokens(deltaLine: number): SparseLineTokens | null { + let low = 0; + let high = this._getTokenCount() - 1; + + while (low < high) { + const mid = low + Math.floor((high - low) / 2); + const midDeltaLine = this._getDeltaLine(mid); + + if (midDeltaLine < deltaLine) { + low = mid + 1; + } else if (midDeltaLine > deltaLine) { + high = mid - 1; + } else { + let min = mid; + while (min > low && this._getDeltaLine(min - 1) === deltaLine) { + min--; + } + let max = mid; + while (max < high && this._getDeltaLine(max + 1) === deltaLine) { + max++; + } + return new SparseLineTokens(this._tokens.subarray(4 * min, 4 * max + 4)); + } + } + + if (this._getDeltaLine(low) === deltaLine) { + return new SparseLineTokens(this._tokens.subarray(4 * low, 4 * low + 4)); + } + + return null; + } + + public clear(): void { + this._tokenCount = 0; + } + + public removeTokens(startDeltaLine: number, startChar: number, endDeltaLine: number, endChar: number): number { + const tokens = this._tokens; + const tokenCount = this._tokenCount; + let newTokenCount = 0; + let hasDeletedTokens = false; + let firstDeltaLine = 0; + for (let i = 0; i < tokenCount; i++) { + const srcOffset = 4 * i; + const tokenDeltaLine = tokens[srcOffset]; + const tokenStartCharacter = tokens[srcOffset + 1]; + const tokenEndCharacter = tokens[srcOffset + 2]; + const tokenMetadata = tokens[srcOffset + 3]; + + if ( + (tokenDeltaLine > startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter >= startChar)) + && (tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter <= endChar)) + ) { + hasDeletedTokens = true; + } else { + if (newTokenCount === 0) { + firstDeltaLine = tokenDeltaLine; + } + if (hasDeletedTokens) { + // must move the token to the left + const destOffset = 4 * newTokenCount; + tokens[destOffset] = tokenDeltaLine - firstDeltaLine; + tokens[destOffset + 1] = tokenStartCharacter; + tokens[destOffset + 2] = tokenEndCharacter; + tokens[destOffset + 3] = tokenMetadata; + } + newTokenCount++; + } + } + + this._tokenCount = newTokenCount; + + return firstDeltaLine; + } + + public split(startDeltaLine: number, startChar: number, endDeltaLine: number, endChar: number): [SparseMultilineTokensStorage, SparseMultilineTokensStorage, number] { + const tokens = this._tokens; + const tokenCount = this._tokenCount; + const aTokens: number[] = []; + const bTokens: number[] = []; + let destTokens: number[] = aTokens; + let destOffset = 0; + let destFirstDeltaLine: number = 0; + for (let i = 0; i < tokenCount; i++) { + const srcOffset = 4 * i; + const tokenDeltaLine = tokens[srcOffset]; + const tokenStartCharacter = tokens[srcOffset + 1]; + const tokenEndCharacter = tokens[srcOffset + 2]; + const tokenMetadata = tokens[srcOffset + 3]; + + if ((tokenDeltaLine > startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter >= startChar))) { + if ((tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter <= endChar))) { + // this token is touching the range + continue; + } else { + // this token is after the range + if (destTokens !== bTokens) { + // this token is the first token after the range + destTokens = bTokens; + destOffset = 0; + destFirstDeltaLine = tokenDeltaLine; + } + } + } + + destTokens[destOffset++] = tokenDeltaLine - destFirstDeltaLine; + destTokens[destOffset++] = tokenStartCharacter; + destTokens[destOffset++] = tokenEndCharacter; + destTokens[destOffset++] = tokenMetadata; + } + + return [new SparseMultilineTokensStorage(new Uint32Array(aTokens)), new SparseMultilineTokensStorage(new Uint32Array(bTokens)), destFirstDeltaLine]; + } + + public acceptDeleteRange(horizontalShiftForFirstLineTokens: number, startDeltaLine: number, startCharacter: number, endDeltaLine: number, endCharacter: number): void { + // This is a bit complex, here are the cases I used to think about this: + // + // 1. The token starts before the deletion range + // 1a. The token is completely before the deletion range + // ----------- + // xxxxxxxxxxx + // 1b. The token starts before, the deletion range ends after the token + // ----------- + // xxxxxxxxxxx + // 1c. The token starts before, the deletion range ends precisely with the token + // --------------- + // xxxxxxxx + // 1d. The token starts before, the deletion range is inside the token + // --------------- + // xxxxx + // + // 2. The token starts at the same position with the deletion range + // 2a. The token starts at the same position, and ends inside the deletion range + // ------- + // xxxxxxxxxxx + // 2b. The token starts at the same position, and ends at the same position as the deletion range + // ---------- + // xxxxxxxxxx + // 2c. The token starts at the same position, and ends after the deletion range + // ------------- + // xxxxxxx + // + // 3. The token starts inside the deletion range + // 3a. The token is inside the deletion range + // ------- + // xxxxxxxxxxxxx + // 3b. The token starts inside the deletion range, and ends at the same position as the deletion range + // ---------- + // xxxxxxxxxxxxx + // 3c. The token starts inside the deletion range, and ends after the deletion range + // ------------ + // xxxxxxxxxxx + // + // 4. The token starts after the deletion range + // ----------- + // xxxxxxxx + // + const tokens = this._tokens; + const tokenCount = this._tokenCount; + const deletedLineCount = (endDeltaLine - startDeltaLine); + let newTokenCount = 0; + let hasDeletedTokens = false; + for (let i = 0; i < tokenCount; i++) { + const srcOffset = 4 * i; + let tokenDeltaLine = tokens[srcOffset]; + let tokenStartCharacter = tokens[srcOffset + 1]; + let tokenEndCharacter = tokens[srcOffset + 2]; + const tokenMetadata = tokens[srcOffset + 3]; + + if (tokenDeltaLine < startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter <= startCharacter)) { + // 1a. The token is completely before the deletion range + // => nothing to do + newTokenCount++; + continue; + } else if (tokenDeltaLine === startDeltaLine && tokenStartCharacter < startCharacter) { + // 1b, 1c, 1d + // => the token survives, but it needs to shrink + if (tokenDeltaLine === endDeltaLine && tokenEndCharacter > endCharacter) { + // 1d. The token starts before, the deletion range is inside the token + // => the token shrinks by the deletion character count + tokenEndCharacter -= (endCharacter - startCharacter); + } else { + // 1b. The token starts before, the deletion range ends after the token + // 1c. The token starts before, the deletion range ends precisely with the token + // => the token shrinks its ending to the deletion start + tokenEndCharacter = startCharacter; + } + } else if (tokenDeltaLine === startDeltaLine && tokenStartCharacter === startCharacter) { + // 2a, 2b, 2c + if (tokenDeltaLine === endDeltaLine && tokenEndCharacter > endCharacter) { + // 2c. The token starts at the same position, and ends after the deletion range + // => the token shrinks by the deletion character count + tokenEndCharacter -= (endCharacter - startCharacter); + } else { + // 2a. The token starts at the same position, and ends inside the deletion range + // 2b. The token starts at the same position, and ends at the same position as the deletion range + // => the token is deleted + hasDeletedTokens = true; + continue; + } + } else if (tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter < endCharacter)) { + // 3a, 3b, 3c + if (tokenDeltaLine === endDeltaLine && tokenEndCharacter > endCharacter) { + // 3c. The token starts inside the deletion range, and ends after the deletion range + // => the token moves left and shrinks + if (tokenDeltaLine === startDeltaLine) { + // the deletion started on the same line as the token + // => the token moves left and shrinks + tokenStartCharacter = startCharacter; + tokenEndCharacter = tokenStartCharacter + (tokenEndCharacter - endCharacter); + } else { + // the deletion started on a line above the token + // => the token moves to the beginning of the line + tokenStartCharacter = 0; + tokenEndCharacter = tokenStartCharacter + (tokenEndCharacter - endCharacter); + } + } else { + // 3a. The token is inside the deletion range + // 3b. The token starts inside the deletion range, and ends at the same position as the deletion range + // => the token is deleted + hasDeletedTokens = true; + continue; + } + } else if (tokenDeltaLine > endDeltaLine) { + // 4. (partial) The token starts after the deletion range, on a line below... + if (deletedLineCount === 0 && !hasDeletedTokens) { + // early stop, there is no need to walk all the tokens and do nothing... + newTokenCount = tokenCount; + break; + } + tokenDeltaLine -= deletedLineCount; + } else if (tokenDeltaLine === endDeltaLine && tokenStartCharacter >= endCharacter) { + // 4. (continued) The token starts after the deletion range, on the last line where a deletion occurs + if (horizontalShiftForFirstLineTokens && tokenDeltaLine === 0) { + tokenStartCharacter += horizontalShiftForFirstLineTokens; + tokenEndCharacter += horizontalShiftForFirstLineTokens; + } + tokenDeltaLine -= deletedLineCount; + tokenStartCharacter -= (endCharacter - startCharacter); + tokenEndCharacter -= (endCharacter - startCharacter); + } else { + throw new Error(`Not possible!`); + } + + const destOffset = 4 * newTokenCount; + tokens[destOffset] = tokenDeltaLine; + tokens[destOffset + 1] = tokenStartCharacter; + tokens[destOffset + 2] = tokenEndCharacter; + tokens[destOffset + 3] = tokenMetadata; + newTokenCount++; + } + + this._tokenCount = newTokenCount; + } + + public acceptInsertText(deltaLine: number, character: number, eolCount: number, firstLineLength: number, lastLineLength: number, firstCharCode: number): void { + // Here are the cases I used to think about this: + // + // 1. The token is completely before the insertion point + // ----------- | + // 2. The token ends precisely at the insertion point + // -----------| + // 3. The token contains the insertion point + // -----|------ + // 4. The token starts precisely at the insertion point + // |----------- + // 5. The token is completely after the insertion point + // | ----------- + // + const isInsertingPreciselyOneWordCharacter = ( + eolCount === 0 + && firstLineLength === 1 + && ( + (firstCharCode >= CharCode.Digit0 && firstCharCode <= CharCode.Digit9) + || (firstCharCode >= CharCode.A && firstCharCode <= CharCode.Z) + || (firstCharCode >= CharCode.a && firstCharCode <= CharCode.z) + ) + ); + const tokens = this._tokens; + const tokenCount = this._tokenCount; + for (let i = 0; i < tokenCount; i++) { + const offset = 4 * i; + let tokenDeltaLine = tokens[offset]; + let tokenStartCharacter = tokens[offset + 1]; + let tokenEndCharacter = tokens[offset + 2]; + + if (tokenDeltaLine < deltaLine || (tokenDeltaLine === deltaLine && tokenEndCharacter < character)) { + // 1. The token is completely before the insertion point + // => nothing to do + continue; + } else if (tokenDeltaLine === deltaLine && tokenEndCharacter === character) { + // 2. The token ends precisely at the insertion point + // => expand the end character only if inserting precisely one character that is a word character + if (isInsertingPreciselyOneWordCharacter) { + tokenEndCharacter += 1; + } else { + continue; + } + } else if (tokenDeltaLine === deltaLine && tokenStartCharacter < character && character < tokenEndCharacter) { + // 3. The token contains the insertion point + if (eolCount === 0) { + // => just expand the end character + tokenEndCharacter += firstLineLength; + } else { + // => cut off the token + tokenEndCharacter = character; + } + } else { + // 4. or 5. + if (tokenDeltaLine === deltaLine && tokenStartCharacter === character) { + // 4. The token starts precisely at the insertion point + // => grow the token (by keeping its start constant) only if inserting precisely one character that is a word character + // => otherwise behave as in case 5. + if (isInsertingPreciselyOneWordCharacter) { + continue; + } + } + // => the token must move and keep its size constant + if (tokenDeltaLine === deltaLine) { + tokenDeltaLine += eolCount; + // this token is on the line where the insertion is taking place + if (eolCount === 0) { + tokenStartCharacter += firstLineLength; + tokenEndCharacter += firstLineLength; + } else { + const tokenLength = tokenEndCharacter - tokenStartCharacter; + tokenStartCharacter = lastLineLength + (tokenStartCharacter - character); + tokenEndCharacter = tokenStartCharacter + tokenLength; + } + } else { + tokenDeltaLine += eolCount; + } + } + + tokens[offset] = tokenDeltaLine; + tokens[offset + 1] = tokenStartCharacter; + tokens[offset + 2] = tokenEndCharacter; + } + } +} + +export class SparseLineTokens { + + private readonly _tokens: Uint32Array; + + constructor(tokens: Uint32Array) { + this._tokens = tokens; + } + + public getCount(): number { + return this._tokens.length / 4; + } + + public getStartCharacter(tokenIndex: number): number { + return this._tokens[4 * tokenIndex + 1]; + } + + public getEndCharacter(tokenIndex: number): number { + return this._tokens[4 * tokenIndex + 2]; + } + + public getMetadata(tokenIndex: number): number { + return this._tokens[4 * tokenIndex + 3]; + } +} diff --git a/src/vs/editor/common/model/tokens/sparseTokensStore.ts b/src/vs/editor/common/model/tokens/sparseTokensStore.ts new file mode 100644 index 00000000000..58815779244 --- /dev/null +++ b/src/vs/editor/common/model/tokens/sparseTokensStore.ts @@ -0,0 +1,238 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as arrays from 'vs/base/common/arrays'; +import { IRange, Range } from 'vs/editor/common/core/range'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; +import { SparseMultilineTokens } from 'vs/editor/common/model/tokens/sparseMultilineTokens'; +import { ILanguageIdCodec, MetadataConsts } from 'vs/editor/common/modes'; + +/** + * Represents sparse tokens in a text model. + */ +export class SparseTokensStore { + + private _pieces: SparseMultilineTokens[]; + private _isComplete: boolean; + private readonly _languageIdCodec: ILanguageIdCodec; + + constructor(languageIdCodec: ILanguageIdCodec) { + this._pieces = []; + this._isComplete = false; + this._languageIdCodec = languageIdCodec; + } + + public flush(): void { + this._pieces = []; + this._isComplete = false; + } + + public isEmpty(): boolean { + return (this._pieces.length === 0); + } + + public set(pieces: SparseMultilineTokens[] | null, isComplete: boolean): void { + this._pieces = pieces || []; + this._isComplete = isComplete; + } + + public setPartial(_range: Range, pieces: SparseMultilineTokens[]): Range { + // console.log(`setPartial ${_range} ${pieces.map(p => p.toString()).join(', ')}`); + + let range = _range; + if (pieces.length > 0) { + const _firstRange = pieces[0].getRange(); + const _lastRange = pieces[pieces.length - 1].getRange(); + if (!_firstRange || !_lastRange) { + return _range; + } + range = _range.plusRange(_firstRange).plusRange(_lastRange); + } + + let insertPosition: { index: number; } | null = null; + for (let i = 0, len = this._pieces.length; i < len; i++) { + const piece = this._pieces[i]; + if (piece.endLineNumber < range.startLineNumber) { + // this piece is before the range + continue; + } + + if (piece.startLineNumber > range.endLineNumber) { + // this piece is after the range, so mark the spot before this piece + // as a good insertion position and stop looping + insertPosition = insertPosition || { index: i }; + break; + } + + // this piece might intersect with the range + piece.removeTokens(range); + + if (piece.isEmpty()) { + // remove the piece if it became empty + this._pieces.splice(i, 1); + i--; + len--; + continue; + } + + if (piece.endLineNumber < range.startLineNumber) { + // after removal, this piece is before the range + continue; + } + + if (piece.startLineNumber > range.endLineNumber) { + // after removal, this piece is after the range + insertPosition = insertPosition || { index: i }; + continue; + } + + // after removal, this piece contains the range + const [a, b] = piece.split(range); + if (a.isEmpty()) { + // this piece is actually after the range + insertPosition = insertPosition || { index: i }; + continue; + } + if (b.isEmpty()) { + // this piece is actually before the range + continue; + } + this._pieces.splice(i, 1, a, b); + i++; + len++; + + insertPosition = insertPosition || { index: i }; + } + + insertPosition = insertPosition || { index: this._pieces.length }; + + if (pieces.length > 0) { + this._pieces = arrays.arrayInsert(this._pieces, insertPosition.index, pieces); + } + + // console.log(`I HAVE ${this._pieces.length} pieces`); + // console.log(`${this._pieces.map(p => p.toString()).join('\n')}`); + + return range; + } + + public isComplete(): boolean { + return this._isComplete; + } + + public addSparseTokens(lineNumber: number, aTokens: LineTokens): LineTokens { + const pieces = this._pieces; + + if (pieces.length === 0) { + return aTokens; + } + + const pieceIndex = SparseTokensStore._findFirstPieceWithLine(pieces, lineNumber); + const bTokens = pieces[pieceIndex].getLineTokens(lineNumber); + + if (!bTokens) { + return aTokens; + } + + const aLen = aTokens.getCount(); + const bLen = bTokens.getCount(); + + let aIndex = 0; + const result: number[] = []; + let resultLen = 0; + let lastEndOffset = 0; + + const emitToken = (endOffset: number, metadata: number) => { + if (endOffset === lastEndOffset) { + return; + } + lastEndOffset = endOffset; + result[resultLen++] = endOffset; + result[resultLen++] = metadata; + }; + + for (let bIndex = 0; bIndex < bLen; bIndex++) { + const bStartCharacter = bTokens.getStartCharacter(bIndex); + const bEndCharacter = bTokens.getEndCharacter(bIndex); + const bMetadata = bTokens.getMetadata(bIndex); + + const bMask = ( + ((bMetadata & MetadataConsts.SEMANTIC_USE_ITALIC) ? MetadataConsts.ITALIC_MASK : 0) + | ((bMetadata & MetadataConsts.SEMANTIC_USE_BOLD) ? MetadataConsts.BOLD_MASK : 0) + | ((bMetadata & MetadataConsts.SEMANTIC_USE_UNDERLINE) ? MetadataConsts.UNDERLINE_MASK : 0) + | ((bMetadata & MetadataConsts.SEMANTIC_USE_STRIKETHROUGH) ? MetadataConsts.STRIKETHROUGH_MASK : 0) + | ((bMetadata & MetadataConsts.SEMANTIC_USE_FOREGROUND) ? MetadataConsts.FOREGROUND_MASK : 0) + | ((bMetadata & MetadataConsts.SEMANTIC_USE_BACKGROUND) ? MetadataConsts.BACKGROUND_MASK : 0) + ) >>> 0; + const aMask = (~bMask) >>> 0; + + // push any token from `a` that is before `b` + while (aIndex < aLen && aTokens.getEndOffset(aIndex) <= bStartCharacter) { + emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex)); + aIndex++; + } + + // push the token from `a` if it intersects the token from `b` + if (aIndex < aLen && aTokens.getStartOffset(aIndex) < bStartCharacter) { + emitToken(bStartCharacter, aTokens.getMetadata(aIndex)); + } + + // skip any tokens from `a` that are contained inside `b` + while (aIndex < aLen && aTokens.getEndOffset(aIndex) < bEndCharacter) { + emitToken(aTokens.getEndOffset(aIndex), (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask)); + aIndex++; + } + + if (aIndex < aLen) { + emitToken(bEndCharacter, (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask)); + if (aTokens.getEndOffset(aIndex) === bEndCharacter) { + // `a` ends exactly at the same spot as `b`! + aIndex++; + } + } else { + const aMergeIndex = Math.min(Math.max(0, aIndex - 1), aLen - 1); + + // push the token from `b` + emitToken(bEndCharacter, (aTokens.getMetadata(aMergeIndex) & aMask) | (bMetadata & bMask)); + } + } + + // push the remaining tokens from `a` + while (aIndex < aLen) { + emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex)); + aIndex++; + } + + return new LineTokens(new Uint32Array(result), aTokens.getLineContent(), this._languageIdCodec); + } + + private static _findFirstPieceWithLine(pieces: SparseMultilineTokens[], lineNumber: number): number { + let low = 0; + let high = pieces.length - 1; + + while (low < high) { + let mid = low + Math.floor((high - low) / 2); + + if (pieces[mid].endLineNumber < lineNumber) { + low = mid + 1; + } else if (pieces[mid].startLineNumber > lineNumber) { + high = mid - 1; + } else { + while (mid > low && pieces[mid - 1].startLineNumber <= lineNumber && lineNumber <= pieces[mid - 1].endLineNumber) { + mid--; + } + return mid; + } + } + + return low; + } + + public acceptEdit(range: IRange, eolCount: number, firstLineLength: number, lastLineLength: number, firstCharCode: number): void { + for (const piece of this._pieces) { + piece.acceptEdit(range, eolCount, firstLineLength, lastLineLength, firstCharCode); + } + } +} diff --git a/src/vs/editor/common/model/tokensStore.ts b/src/vs/editor/common/model/tokensStore.ts deleted file mode 100644 index fbb421e1a25..00000000000 --- a/src/vs/editor/common/model/tokensStore.ts +++ /dev/null @@ -1,1415 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import * as arrays from 'vs/base/common/arrays'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; -import { Position } from 'vs/editor/common/core/position'; -import { IRange, Range } from 'vs/editor/common/core/range'; -import { ColorId, FontStyle, ILanguageIdCodec, LanguageId, MetadataConsts, StandardTokenType, TokenMetadata } from 'vs/editor/common/modes'; -import { writeUInt32BE, readUInt32BE } from 'vs/base/common/buffer'; -import { CharCode } from 'vs/base/common/charCode'; - -export const enum StringEOL { - Unknown = 0, - Invalid = 3, - LF = 1, - CRLF = 2 -} - -export function countEOL(text: string): [number, number, number, StringEOL] { - let eolCount = 0; - let firstLineLength = 0; - let lastLineStart = 0; - let eol: StringEOL = StringEOL.Unknown; - for (let i = 0, len = text.length; i < len; i++) { - const chr = text.charCodeAt(i); - - if (chr === CharCode.CarriageReturn) { - if (eolCount === 0) { - firstLineLength = i; - } - eolCount++; - if (i + 1 < len && text.charCodeAt(i + 1) === CharCode.LineFeed) { - // \r\n... case - eol |= StringEOL.CRLF; - i++; // skip \n - } else { - // \r... case - eol |= StringEOL.Invalid; - } - lastLineStart = i + 1; - } else if (chr === CharCode.LineFeed) { - // \n... case - eol |= StringEOL.LF; - if (eolCount === 0) { - firstLineLength = i; - } - eolCount++; - lastLineStart = i + 1; - } - } - if (eolCount === 0) { - firstLineLength = text.length; - } - return [eolCount, firstLineLength, text.length - lastLineStart, eol]; -} - -function getDefaultMetadata(topLevelLanguageId: LanguageId): number { - return ( - (topLevelLanguageId << MetadataConsts.LANGUAGEID_OFFSET) - | (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET) - | (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET) - | (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET) - | (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET) - ) >>> 0; -} - -const EMPTY_LINE_TOKENS = (new Uint32Array(0)).buffer; - -export class MultilineTokensBuilder { - - public readonly tokens: MultilineTokens[]; - - constructor() { - this.tokens = []; - } - - public add(lineNumber: number, lineTokens: Uint32Array): void { - if (this.tokens.length > 0) { - const last = this.tokens[this.tokens.length - 1]; - const lastLineNumber = last.startLineNumber + last.tokens.length - 1; - if (lastLineNumber + 1 === lineNumber) { - // append - last.tokens.push(lineTokens); - return; - } - } - this.tokens.push(new MultilineTokens(lineNumber, [lineTokens])); - } - - public static deserialize(buff: Uint8Array): MultilineTokens[] { - let offset = 0; - const count = readUInt32BE(buff, offset); offset += 4; - const result: MultilineTokens[] = []; - for (let i = 0; i < count; i++) { - offset = MultilineTokens.deserialize(buff, offset, result); - } - return result; - } - - public serialize(): Uint8Array { - const size = this._serializeSize(); - const result = new Uint8Array(size); - this._serialize(result); - return result; - } - - private _serializeSize(): number { - let result = 0; - result += 4; // 4 bytes for the count - for (let i = 0; i < this.tokens.length; i++) { - result += this.tokens[i].serializeSize(); - } - return result; - } - - private _serialize(destination: Uint8Array): void { - let offset = 0; - writeUInt32BE(destination, this.tokens.length, offset); offset += 4; - for (let i = 0; i < this.tokens.length; i++) { - offset = this.tokens[i].serialize(destination, offset); - } - } -} - -export class SparseEncodedTokens { - /** - * The encoding of tokens is: - * 4*i deltaLine (from `startLineNumber`) - * 4*i+1 startCharacter (from the line start) - * 4*i+2 endCharacter (from the line start) - * 4*i+3 metadata - */ - private readonly _tokens: Uint32Array; - private _tokenCount: number; - - constructor(tokens: Uint32Array) { - this._tokens = tokens; - this._tokenCount = tokens.length / 4; - } - - public toString(startLineNumber: number): string { - const pieces: string[] = []; - for (let i = 0; i < this._tokenCount; i++) { - pieces.push(`(${this._getDeltaLine(i) + startLineNumber},${this._getStartCharacter(i)}-${this._getEndCharacter(i)})`); - } - return `[${pieces.join(',')}]`; - } - - public getMaxDeltaLine(): number { - const tokenCount = this._getTokenCount(); - if (tokenCount === 0) { - return -1; - } - return this._getDeltaLine(tokenCount - 1); - } - - public getRange(): Range | null { - const tokenCount = this._getTokenCount(); - if (tokenCount === 0) { - return null; - } - const startChar = this._getStartCharacter(0); - const maxDeltaLine = this._getDeltaLine(tokenCount - 1); - const endChar = this._getEndCharacter(tokenCount - 1); - return new Range(0, startChar + 1, maxDeltaLine, endChar + 1); - } - - private _getTokenCount(): number { - return this._tokenCount; - } - - private _getDeltaLine(tokenIndex: number): number { - return this._tokens[4 * tokenIndex]; - } - - private _getStartCharacter(tokenIndex: number): number { - return this._tokens[4 * tokenIndex + 1]; - } - - private _getEndCharacter(tokenIndex: number): number { - return this._tokens[4 * tokenIndex + 2]; - } - - public isEmpty(): boolean { - return (this._getTokenCount() === 0); - } - - public getLineTokens(deltaLine: number): LineTokens2 | null { - let low = 0; - let high = this._getTokenCount() - 1; - - while (low < high) { - const mid = low + Math.floor((high - low) / 2); - const midDeltaLine = this._getDeltaLine(mid); - - if (midDeltaLine < deltaLine) { - low = mid + 1; - } else if (midDeltaLine > deltaLine) { - high = mid - 1; - } else { - let min = mid; - while (min > low && this._getDeltaLine(min - 1) === deltaLine) { - min--; - } - let max = mid; - while (max < high && this._getDeltaLine(max + 1) === deltaLine) { - max++; - } - return new LineTokens2(this._tokens.subarray(4 * min, 4 * max + 4)); - } - } - - if (this._getDeltaLine(low) === deltaLine) { - return new LineTokens2(this._tokens.subarray(4 * low, 4 * low + 4)); - } - - return null; - } - - public clear(): void { - this._tokenCount = 0; - } - - public removeTokens(startDeltaLine: number, startChar: number, endDeltaLine: number, endChar: number): number { - const tokens = this._tokens; - const tokenCount = this._tokenCount; - let newTokenCount = 0; - let hasDeletedTokens = false; - let firstDeltaLine = 0; - for (let i = 0; i < tokenCount; i++) { - const srcOffset = 4 * i; - const tokenDeltaLine = tokens[srcOffset]; - const tokenStartCharacter = tokens[srcOffset + 1]; - const tokenEndCharacter = tokens[srcOffset + 2]; - const tokenMetadata = tokens[srcOffset + 3]; - - if ( - (tokenDeltaLine > startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter >= startChar)) - && (tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter <= endChar)) - ) { - hasDeletedTokens = true; - } else { - if (newTokenCount === 0) { - firstDeltaLine = tokenDeltaLine; - } - if (hasDeletedTokens) { - // must move the token to the left - const destOffset = 4 * newTokenCount; - tokens[destOffset] = tokenDeltaLine - firstDeltaLine; - tokens[destOffset + 1] = tokenStartCharacter; - tokens[destOffset + 2] = tokenEndCharacter; - tokens[destOffset + 3] = tokenMetadata; - } - newTokenCount++; - } - } - - this._tokenCount = newTokenCount; - - return firstDeltaLine; - } - - public split(startDeltaLine: number, startChar: number, endDeltaLine: number, endChar: number): [SparseEncodedTokens, SparseEncodedTokens, number] { - const tokens = this._tokens; - const tokenCount = this._tokenCount; - const aTokens: number[] = []; - const bTokens: number[] = []; - let destTokens: number[] = aTokens; - let destOffset = 0; - let destFirstDeltaLine: number = 0; - for (let i = 0; i < tokenCount; i++) { - const srcOffset = 4 * i; - const tokenDeltaLine = tokens[srcOffset]; - const tokenStartCharacter = tokens[srcOffset + 1]; - const tokenEndCharacter = tokens[srcOffset + 2]; - const tokenMetadata = tokens[srcOffset + 3]; - - if ((tokenDeltaLine > startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter >= startChar))) { - if ((tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter <= endChar))) { - // this token is touching the range - continue; - } else { - // this token is after the range - if (destTokens !== bTokens) { - // this token is the first token after the range - destTokens = bTokens; - destOffset = 0; - destFirstDeltaLine = tokenDeltaLine; - } - } - } - - destTokens[destOffset++] = tokenDeltaLine - destFirstDeltaLine; - destTokens[destOffset++] = tokenStartCharacter; - destTokens[destOffset++] = tokenEndCharacter; - destTokens[destOffset++] = tokenMetadata; - } - - return [new SparseEncodedTokens(new Uint32Array(aTokens)), new SparseEncodedTokens(new Uint32Array(bTokens)), destFirstDeltaLine]; - } - - public acceptDeleteRange(horizontalShiftForFirstLineTokens: number, startDeltaLine: number, startCharacter: number, endDeltaLine: number, endCharacter: number): void { - // This is a bit complex, here are the cases I used to think about this: - // - // 1. The token starts before the deletion range - // 1a. The token is completely before the deletion range - // ----------- - // xxxxxxxxxxx - // 1b. The token starts before, the deletion range ends after the token - // ----------- - // xxxxxxxxxxx - // 1c. The token starts before, the deletion range ends precisely with the token - // --------------- - // xxxxxxxx - // 1d. The token starts before, the deletion range is inside the token - // --------------- - // xxxxx - // - // 2. The token starts at the same position with the deletion range - // 2a. The token starts at the same position, and ends inside the deletion range - // ------- - // xxxxxxxxxxx - // 2b. The token starts at the same position, and ends at the same position as the deletion range - // ---------- - // xxxxxxxxxx - // 2c. The token starts at the same position, and ends after the deletion range - // ------------- - // xxxxxxx - // - // 3. The token starts inside the deletion range - // 3a. The token is inside the deletion range - // ------- - // xxxxxxxxxxxxx - // 3b. The token starts inside the deletion range, and ends at the same position as the deletion range - // ---------- - // xxxxxxxxxxxxx - // 3c. The token starts inside the deletion range, and ends after the deletion range - // ------------ - // xxxxxxxxxxx - // - // 4. The token starts after the deletion range - // ----------- - // xxxxxxxx - // - const tokens = this._tokens; - const tokenCount = this._tokenCount; - const deletedLineCount = (endDeltaLine - startDeltaLine); - let newTokenCount = 0; - let hasDeletedTokens = false; - for (let i = 0; i < tokenCount; i++) { - const srcOffset = 4 * i; - let tokenDeltaLine = tokens[srcOffset]; - let tokenStartCharacter = tokens[srcOffset + 1]; - let tokenEndCharacter = tokens[srcOffset + 2]; - const tokenMetadata = tokens[srcOffset + 3]; - - if (tokenDeltaLine < startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter <= startCharacter)) { - // 1a. The token is completely before the deletion range - // => nothing to do - newTokenCount++; - continue; - } else if (tokenDeltaLine === startDeltaLine && tokenStartCharacter < startCharacter) { - // 1b, 1c, 1d - // => the token survives, but it needs to shrink - if (tokenDeltaLine === endDeltaLine && tokenEndCharacter > endCharacter) { - // 1d. The token starts before, the deletion range is inside the token - // => the token shrinks by the deletion character count - tokenEndCharacter -= (endCharacter - startCharacter); - } else { - // 1b. The token starts before, the deletion range ends after the token - // 1c. The token starts before, the deletion range ends precisely with the token - // => the token shrinks its ending to the deletion start - tokenEndCharacter = startCharacter; - } - } else if (tokenDeltaLine === startDeltaLine && tokenStartCharacter === startCharacter) { - // 2a, 2b, 2c - if (tokenDeltaLine === endDeltaLine && tokenEndCharacter > endCharacter) { - // 2c. The token starts at the same position, and ends after the deletion range - // => the token shrinks by the deletion character count - tokenEndCharacter -= (endCharacter - startCharacter); - } else { - // 2a. The token starts at the same position, and ends inside the deletion range - // 2b. The token starts at the same position, and ends at the same position as the deletion range - // => the token is deleted - hasDeletedTokens = true; - continue; - } - } else if (tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter < endCharacter)) { - // 3a, 3b, 3c - if (tokenDeltaLine === endDeltaLine && tokenEndCharacter > endCharacter) { - // 3c. The token starts inside the deletion range, and ends after the deletion range - // => the token moves left and shrinks - if (tokenDeltaLine === startDeltaLine) { - // the deletion started on the same line as the token - // => the token moves left and shrinks - tokenStartCharacter = startCharacter; - tokenEndCharacter = tokenStartCharacter + (tokenEndCharacter - endCharacter); - } else { - // the deletion started on a line above the token - // => the token moves to the beginning of the line - tokenStartCharacter = 0; - tokenEndCharacter = tokenStartCharacter + (tokenEndCharacter - endCharacter); - } - } else { - // 3a. The token is inside the deletion range - // 3b. The token starts inside the deletion range, and ends at the same position as the deletion range - // => the token is deleted - hasDeletedTokens = true; - continue; - } - } else if (tokenDeltaLine > endDeltaLine) { - // 4. (partial) The token starts after the deletion range, on a line below... - if (deletedLineCount === 0 && !hasDeletedTokens) { - // early stop, there is no need to walk all the tokens and do nothing... - newTokenCount = tokenCount; - break; - } - tokenDeltaLine -= deletedLineCount; - } else if (tokenDeltaLine === endDeltaLine && tokenStartCharacter >= endCharacter) { - // 4. (continued) The token starts after the deletion range, on the last line where a deletion occurs - if (horizontalShiftForFirstLineTokens && tokenDeltaLine === 0) { - tokenStartCharacter += horizontalShiftForFirstLineTokens; - tokenEndCharacter += horizontalShiftForFirstLineTokens; - } - tokenDeltaLine -= deletedLineCount; - tokenStartCharacter -= (endCharacter - startCharacter); - tokenEndCharacter -= (endCharacter - startCharacter); - } else { - throw new Error(`Not possible!`); - } - - const destOffset = 4 * newTokenCount; - tokens[destOffset] = tokenDeltaLine; - tokens[destOffset + 1] = tokenStartCharacter; - tokens[destOffset + 2] = tokenEndCharacter; - tokens[destOffset + 3] = tokenMetadata; - newTokenCount++; - } - - this._tokenCount = newTokenCount; - } - - public acceptInsertText(deltaLine: number, character: number, eolCount: number, firstLineLength: number, lastLineLength: number, firstCharCode: number): void { - // Here are the cases I used to think about this: - // - // 1. The token is completely before the insertion point - // ----------- | - // 2. The token ends precisely at the insertion point - // -----------| - // 3. The token contains the insertion point - // -----|------ - // 4. The token starts precisely at the insertion point - // |----------- - // 5. The token is completely after the insertion point - // | ----------- - // - const isInsertingPreciselyOneWordCharacter = ( - eolCount === 0 - && firstLineLength === 1 - && ( - (firstCharCode >= CharCode.Digit0 && firstCharCode <= CharCode.Digit9) - || (firstCharCode >= CharCode.A && firstCharCode <= CharCode.Z) - || (firstCharCode >= CharCode.a && firstCharCode <= CharCode.z) - ) - ); - const tokens = this._tokens; - const tokenCount = this._tokenCount; - for (let i = 0; i < tokenCount; i++) { - const offset = 4 * i; - let tokenDeltaLine = tokens[offset]; - let tokenStartCharacter = tokens[offset + 1]; - let tokenEndCharacter = tokens[offset + 2]; - - if (tokenDeltaLine < deltaLine || (tokenDeltaLine === deltaLine && tokenEndCharacter < character)) { - // 1. The token is completely before the insertion point - // => nothing to do - continue; - } else if (tokenDeltaLine === deltaLine && tokenEndCharacter === character) { - // 2. The token ends precisely at the insertion point - // => expand the end character only if inserting precisely one character that is a word character - if (isInsertingPreciselyOneWordCharacter) { - tokenEndCharacter += 1; - } else { - continue; - } - } else if (tokenDeltaLine === deltaLine && tokenStartCharacter < character && character < tokenEndCharacter) { - // 3. The token contains the insertion point - if (eolCount === 0) { - // => just expand the end character - tokenEndCharacter += firstLineLength; - } else { - // => cut off the token - tokenEndCharacter = character; - } - } else { - // 4. or 5. - if (tokenDeltaLine === deltaLine && tokenStartCharacter === character) { - // 4. The token starts precisely at the insertion point - // => grow the token (by keeping its start constant) only if inserting precisely one character that is a word character - // => otherwise behave as in case 5. - if (isInsertingPreciselyOneWordCharacter) { - continue; - } - } - // => the token must move and keep its size constant - if (tokenDeltaLine === deltaLine) { - tokenDeltaLine += eolCount; - // this token is on the line where the insertion is taking place - if (eolCount === 0) { - tokenStartCharacter += firstLineLength; - tokenEndCharacter += firstLineLength; - } else { - const tokenLength = tokenEndCharacter - tokenStartCharacter; - tokenStartCharacter = lastLineLength + (tokenStartCharacter - character); - tokenEndCharacter = tokenStartCharacter + tokenLength; - } - } else { - tokenDeltaLine += eolCount; - } - } - - tokens[offset] = tokenDeltaLine; - tokens[offset + 1] = tokenStartCharacter; - tokens[offset + 2] = tokenEndCharacter; - } - } -} - -export class LineTokens2 { - - private readonly _tokens: Uint32Array; - - constructor(tokens: Uint32Array) { - this._tokens = tokens; - } - - public getCount(): number { - return this._tokens.length / 4; - } - - public getStartCharacter(tokenIndex: number): number { - return this._tokens[4 * tokenIndex + 1]; - } - - public getEndCharacter(tokenIndex: number): number { - return this._tokens[4 * tokenIndex + 2]; - } - - public getMetadata(tokenIndex: number): number { - return this._tokens[4 * tokenIndex + 3]; - } -} - -export class MultilineTokens2 { - - public startLineNumber: number; - public endLineNumber: number; - public tokens: SparseEncodedTokens; - - constructor(startLineNumber: number, tokens: SparseEncodedTokens) { - this.startLineNumber = startLineNumber; - this.tokens = tokens; - this.endLineNumber = this.startLineNumber + this.tokens.getMaxDeltaLine(); - } - - public toString(): string { - return this.tokens.toString(this.startLineNumber); - } - - private _updateEndLineNumber(): void { - this.endLineNumber = this.startLineNumber + this.tokens.getMaxDeltaLine(); - } - - public isEmpty(): boolean { - return this.tokens.isEmpty(); - } - - public getLineTokens(lineNumber: number): LineTokens2 | null { - if (this.startLineNumber <= lineNumber && lineNumber <= this.endLineNumber) { - return this.tokens.getLineTokens(lineNumber - this.startLineNumber); - } - return null; - } - - public getRange(): Range | null { - const deltaRange = this.tokens.getRange(); - if (!deltaRange) { - return deltaRange; - } - return new Range(this.startLineNumber + deltaRange.startLineNumber, deltaRange.startColumn, this.startLineNumber + deltaRange.endLineNumber, deltaRange.endColumn); - } - - public removeTokens(range: Range): void { - const startLineIndex = range.startLineNumber - this.startLineNumber; - const endLineIndex = range.endLineNumber - this.startLineNumber; - - this.startLineNumber += this.tokens.removeTokens(startLineIndex, range.startColumn - 1, endLineIndex, range.endColumn - 1); - this._updateEndLineNumber(); - } - - public split(range: Range): [MultilineTokens2, MultilineTokens2] { - // split tokens to two: - // a) all the tokens before `range` - // b) all the tokens after `range` - const startLineIndex = range.startLineNumber - this.startLineNumber; - const endLineIndex = range.endLineNumber - this.startLineNumber; - - const [a, b, bDeltaLine] = this.tokens.split(startLineIndex, range.startColumn - 1, endLineIndex, range.endColumn - 1); - return [new MultilineTokens2(this.startLineNumber, a), new MultilineTokens2(this.startLineNumber + bDeltaLine, b)]; - } - - public applyEdit(range: IRange, text: string): void { - const [eolCount, firstLineLength, lastLineLength] = countEOL(text); - this.acceptEdit(range, eolCount, firstLineLength, lastLineLength, text.length > 0 ? text.charCodeAt(0) : CharCode.Null); - } - - public acceptEdit(range: IRange, eolCount: number, firstLineLength: number, lastLineLength: number, firstCharCode: number): void { - this._acceptDeleteRange(range); - this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength, lastLineLength, firstCharCode); - this._updateEndLineNumber(); - } - - private _acceptDeleteRange(range: IRange): void { - if (range.startLineNumber === range.endLineNumber && range.startColumn === range.endColumn) { - // Nothing to delete - return; - } - - const firstLineIndex = range.startLineNumber - this.startLineNumber; - const lastLineIndex = range.endLineNumber - this.startLineNumber; - - if (lastLineIndex < 0) { - // this deletion occurs entirely before this block, so we only need to adjust line numbers - const deletedLinesCount = lastLineIndex - firstLineIndex; - this.startLineNumber -= deletedLinesCount; - return; - } - - const tokenMaxDeltaLine = this.tokens.getMaxDeltaLine(); - - if (firstLineIndex >= tokenMaxDeltaLine + 1) { - // this deletion occurs entirely after this block, so there is nothing to do - return; - } - - if (firstLineIndex < 0 && lastLineIndex >= tokenMaxDeltaLine + 1) { - // this deletion completely encompasses this block - this.startLineNumber = 0; - this.tokens.clear(); - return; - } - - if (firstLineIndex < 0) { - const deletedBefore = -firstLineIndex; - this.startLineNumber -= deletedBefore; - - this.tokens.acceptDeleteRange(range.startColumn - 1, 0, 0, lastLineIndex, range.endColumn - 1); - } else { - this.tokens.acceptDeleteRange(0, firstLineIndex, range.startColumn - 1, lastLineIndex, range.endColumn - 1); - } - } - - private _acceptInsertText(position: Position, eolCount: number, firstLineLength: number, lastLineLength: number, firstCharCode: number): void { - - if (eolCount === 0 && firstLineLength === 0) { - // Nothing to insert - return; - } - - const lineIndex = position.lineNumber - this.startLineNumber; - - if (lineIndex < 0) { - // this insertion occurs before this block, so we only need to adjust line numbers - this.startLineNumber += eolCount; - return; - } - - const tokenMaxDeltaLine = this.tokens.getMaxDeltaLine(); - - if (lineIndex >= tokenMaxDeltaLine + 1) { - // this insertion occurs after this block, so there is nothing to do - return; - } - - this.tokens.acceptInsertText(lineIndex, position.column - 1, eolCount, firstLineLength, lastLineLength, firstCharCode); - } -} - -export class MultilineTokens { - - public startLineNumber: number; - public tokens: (Uint32Array | ArrayBuffer | null)[]; - - constructor(startLineNumber: number, tokens: Uint32Array[]) { - this.startLineNumber = startLineNumber; - this.tokens = tokens; - } - - public static deserialize(buff: Uint8Array, offset: number, result: MultilineTokens[]): number { - const view32 = new Uint32Array(buff.buffer); - const startLineNumber = readUInt32BE(buff, offset); offset += 4; - const count = readUInt32BE(buff, offset); offset += 4; - const tokens: Uint32Array[] = []; - for (let i = 0; i < count; i++) { - const byteCount = readUInt32BE(buff, offset); offset += 4; - tokens.push(view32.subarray(offset / 4, offset / 4 + byteCount / 4)); - offset += byteCount; - } - result.push(new MultilineTokens(startLineNumber, tokens)); - return offset; - } - - public serializeSize(): number { - let result = 0; - result += 4; // 4 bytes for the start line number - result += 4; // 4 bytes for the line count - for (let i = 0; i < this.tokens.length; i++) { - const lineTokens = this.tokens[i]; - if (!(lineTokens instanceof Uint32Array)) { - throw new Error(`Not supported!`); - } - result += 4; // 4 bytes for the byte count - result += lineTokens.byteLength; - } - return result; - } - - public serialize(destination: Uint8Array, offset: number): number { - writeUInt32BE(destination, this.startLineNumber, offset); offset += 4; - writeUInt32BE(destination, this.tokens.length, offset); offset += 4; - for (let i = 0; i < this.tokens.length; i++) { - const lineTokens = this.tokens[i]; - if (!(lineTokens instanceof Uint32Array)) { - throw new Error(`Not supported!`); - } - writeUInt32BE(destination, lineTokens.byteLength, offset); offset += 4; - destination.set(new Uint8Array(lineTokens.buffer), offset); offset += lineTokens.byteLength; - } - return offset; - } - - public applyEdit(range: IRange, text: string): void { - const [eolCount, firstLineLength] = countEOL(text); - this._acceptDeleteRange(range); - this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength); - } - - private _acceptDeleteRange(range: IRange): void { - if (range.startLineNumber === range.endLineNumber && range.startColumn === range.endColumn) { - // Nothing to delete - return; - } - - const firstLineIndex = range.startLineNumber - this.startLineNumber; - const lastLineIndex = range.endLineNumber - this.startLineNumber; - - if (lastLineIndex < 0) { - // this deletion occurs entirely before this block, so we only need to adjust line numbers - const deletedLinesCount = lastLineIndex - firstLineIndex; - this.startLineNumber -= deletedLinesCount; - return; - } - - if (firstLineIndex >= this.tokens.length) { - // this deletion occurs entirely after this block, so there is nothing to do - return; - } - - if (firstLineIndex < 0 && lastLineIndex >= this.tokens.length) { - // this deletion completely encompasses this block - this.startLineNumber = 0; - this.tokens = []; - return; - } - - if (firstLineIndex === lastLineIndex) { - // a delete on a single line - this.tokens[firstLineIndex] = TokensStore._delete(this.tokens[firstLineIndex], range.startColumn - 1, range.endColumn - 1); - return; - } - - if (firstLineIndex >= 0) { - // The first line survives - this.tokens[firstLineIndex] = TokensStore._deleteEnding(this.tokens[firstLineIndex], range.startColumn - 1); - - if (lastLineIndex < this.tokens.length) { - // The last line survives - const lastLineTokens = TokensStore._deleteBeginning(this.tokens[lastLineIndex], range.endColumn - 1); - - // Take remaining text on last line and append it to remaining text on first line - this.tokens[firstLineIndex] = TokensStore._append(this.tokens[firstLineIndex], lastLineTokens); - - // Delete middle lines - this.tokens.splice(firstLineIndex + 1, lastLineIndex - firstLineIndex); - } else { - // The last line does not survive - - // Take remaining text on last line and append it to remaining text on first line - this.tokens[firstLineIndex] = TokensStore._append(this.tokens[firstLineIndex], null); - - // Delete lines - this.tokens = this.tokens.slice(0, firstLineIndex + 1); - } - } else { - // The first line does not survive - - const deletedBefore = -firstLineIndex; - this.startLineNumber -= deletedBefore; - - // Remove beginning from last line - this.tokens[lastLineIndex] = TokensStore._deleteBeginning(this.tokens[lastLineIndex], range.endColumn - 1); - - // Delete lines - this.tokens = this.tokens.slice(lastLineIndex); - } - } - - private _acceptInsertText(position: Position, eolCount: number, firstLineLength: number): void { - - if (eolCount === 0 && firstLineLength === 0) { - // Nothing to insert - return; - } - - const lineIndex = position.lineNumber - this.startLineNumber; - - if (lineIndex < 0) { - // this insertion occurs before this block, so we only need to adjust line numbers - this.startLineNumber += eolCount; - return; - } - - if (lineIndex >= this.tokens.length) { - // this insertion occurs after this block, so there is nothing to do - return; - } - - if (eolCount === 0) { - // Inserting text on one line - this.tokens[lineIndex] = TokensStore._insert(this.tokens[lineIndex], position.column - 1, firstLineLength); - return; - } - - this.tokens[lineIndex] = TokensStore._deleteEnding(this.tokens[lineIndex], position.column - 1); - this.tokens[lineIndex] = TokensStore._insert(this.tokens[lineIndex], position.column - 1, firstLineLength); - - this._insertLines(position.lineNumber, eolCount); - } - - private _insertLines(insertIndex: number, insertCount: number): void { - if (insertCount === 0) { - return; - } - const lineTokens: (Uint32Array | ArrayBuffer | null)[] = []; - for (let i = 0; i < insertCount; i++) { - lineTokens[i] = null; - } - this.tokens = arrays.arrayInsert(this.tokens, insertIndex, lineTokens); - } -} - -function toUint32Array(arr: Uint32Array | ArrayBuffer): Uint32Array { - if (arr instanceof Uint32Array) { - return arr; - } else { - return new Uint32Array(arr); - } -} - -export class TokensStore2 { - - private _pieces: MultilineTokens2[]; - private _isComplete: boolean; - private readonly _languageIdCodec: ILanguageIdCodec; - - constructor(languageIdCodec: ILanguageIdCodec) { - this._pieces = []; - this._isComplete = false; - this._languageIdCodec = languageIdCodec; - } - - public flush(): void { - this._pieces = []; - this._isComplete = false; - } - - public isEmpty(): boolean { - return (this._pieces.length === 0); - } - - public set(pieces: MultilineTokens2[] | null, isComplete: boolean): void { - this._pieces = pieces || []; - this._isComplete = isComplete; - } - - public setPartial(_range: Range, pieces: MultilineTokens2[]): Range { - // console.log(`setPartial ${_range} ${pieces.map(p => p.toString()).join(', ')}`); - - let range = _range; - if (pieces.length > 0) { - const _firstRange = pieces[0].getRange(); - const _lastRange = pieces[pieces.length - 1].getRange(); - if (!_firstRange || !_lastRange) { - return _range; - } - range = _range.plusRange(_firstRange).plusRange(_lastRange); - } - - let insertPosition: { index: number; } | null = null; - for (let i = 0, len = this._pieces.length; i < len; i++) { - const piece = this._pieces[i]; - if (piece.endLineNumber < range.startLineNumber) { - // this piece is before the range - continue; - } - - if (piece.startLineNumber > range.endLineNumber) { - // this piece is after the range, so mark the spot before this piece - // as a good insertion position and stop looping - insertPosition = insertPosition || { index: i }; - break; - } - - // this piece might intersect with the range - piece.removeTokens(range); - - if (piece.isEmpty()) { - // remove the piece if it became empty - this._pieces.splice(i, 1); - i--; - len--; - continue; - } - - if (piece.endLineNumber < range.startLineNumber) { - // after removal, this piece is before the range - continue; - } - - if (piece.startLineNumber > range.endLineNumber) { - // after removal, this piece is after the range - insertPosition = insertPosition || { index: i }; - continue; - } - - // after removal, this piece contains the range - const [a, b] = piece.split(range); - if (a.isEmpty()) { - // this piece is actually after the range - insertPosition = insertPosition || { index: i }; - continue; - } - if (b.isEmpty()) { - // this piece is actually before the range - continue; - } - this._pieces.splice(i, 1, a, b); - i++; - len++; - - insertPosition = insertPosition || { index: i }; - } - - insertPosition = insertPosition || { index: this._pieces.length }; - - if (pieces.length > 0) { - this._pieces = arrays.arrayInsert(this._pieces, insertPosition.index, pieces); - } - - // console.log(`I HAVE ${this._pieces.length} pieces`); - // console.log(`${this._pieces.map(p => p.toString()).join('\n')}`); - - return range; - } - - public isComplete(): boolean { - return this._isComplete; - } - - public addSemanticTokens(lineNumber: number, aTokens: LineTokens): LineTokens { - const pieces = this._pieces; - - if (pieces.length === 0) { - return aTokens; - } - - const pieceIndex = TokensStore2._findFirstPieceWithLine(pieces, lineNumber); - const bTokens = pieces[pieceIndex].getLineTokens(lineNumber); - - if (!bTokens) { - return aTokens; - } - - const aLen = aTokens.getCount(); - const bLen = bTokens.getCount(); - - let aIndex = 0; - const result: number[] = []; - let resultLen = 0; - let lastEndOffset = 0; - - const emitToken = (endOffset: number, metadata: number) => { - if (endOffset === lastEndOffset) { - return; - } - lastEndOffset = endOffset; - result[resultLen++] = endOffset; - result[resultLen++] = metadata; - }; - - for (let bIndex = 0; bIndex < bLen; bIndex++) { - const bStartCharacter = bTokens.getStartCharacter(bIndex); - const bEndCharacter = bTokens.getEndCharacter(bIndex); - const bMetadata = bTokens.getMetadata(bIndex); - - const bMask = ( - ((bMetadata & MetadataConsts.SEMANTIC_USE_ITALIC) ? MetadataConsts.ITALIC_MASK : 0) - | ((bMetadata & MetadataConsts.SEMANTIC_USE_BOLD) ? MetadataConsts.BOLD_MASK : 0) - | ((bMetadata & MetadataConsts.SEMANTIC_USE_UNDERLINE) ? MetadataConsts.UNDERLINE_MASK : 0) - | ((bMetadata & MetadataConsts.SEMANTIC_USE_STRIKETHROUGH) ? MetadataConsts.STRIKETHROUGH_MASK : 0) - | ((bMetadata & MetadataConsts.SEMANTIC_USE_FOREGROUND) ? MetadataConsts.FOREGROUND_MASK : 0) - | ((bMetadata & MetadataConsts.SEMANTIC_USE_BACKGROUND) ? MetadataConsts.BACKGROUND_MASK : 0) - ) >>> 0; - const aMask = (~bMask) >>> 0; - - // push any token from `a` that is before `b` - while (aIndex < aLen && aTokens.getEndOffset(aIndex) <= bStartCharacter) { - emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex)); - aIndex++; - } - - // push the token from `a` if it intersects the token from `b` - if (aIndex < aLen && aTokens.getStartOffset(aIndex) < bStartCharacter) { - emitToken(bStartCharacter, aTokens.getMetadata(aIndex)); - } - - // skip any tokens from `a` that are contained inside `b` - while (aIndex < aLen && aTokens.getEndOffset(aIndex) < bEndCharacter) { - emitToken(aTokens.getEndOffset(aIndex), (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask)); - aIndex++; - } - - if (aIndex < aLen) { - emitToken(bEndCharacter, (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask)); - if (aTokens.getEndOffset(aIndex) === bEndCharacter) { - // `a` ends exactly at the same spot as `b`! - aIndex++; - } - } else { - const aMergeIndex = Math.min(Math.max(0, aIndex - 1), aLen - 1); - - // push the token from `b` - emitToken(bEndCharacter, (aTokens.getMetadata(aMergeIndex) & aMask) | (bMetadata & bMask)); - } - } - - // push the remaining tokens from `a` - while (aIndex < aLen) { - emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex)); - aIndex++; - } - - return new LineTokens(new Uint32Array(result), aTokens.getLineContent(), this._languageIdCodec); - } - - private static _findFirstPieceWithLine(pieces: MultilineTokens2[], lineNumber: number): number { - let low = 0; - let high = pieces.length - 1; - - while (low < high) { - let mid = low + Math.floor((high - low) / 2); - - if (pieces[mid].endLineNumber < lineNumber) { - low = mid + 1; - } else if (pieces[mid].startLineNumber > lineNumber) { - high = mid - 1; - } else { - while (mid > low && pieces[mid - 1].startLineNumber <= lineNumber && lineNumber <= pieces[mid - 1].endLineNumber) { - mid--; - } - return mid; - } - } - - return low; - } - - //#region Editing - - public acceptEdit(range: IRange, eolCount: number, firstLineLength: number, lastLineLength: number, firstCharCode: number): void { - for (const piece of this._pieces) { - piece.acceptEdit(range, eolCount, firstLineLength, lastLineLength, firstCharCode); - } - } - - //#endregion -} - -export class TokensStore { - private _lineTokens: (Uint32Array | ArrayBuffer | null)[]; - private _len: number; - private readonly _languageIdCodec: ILanguageIdCodec; - - constructor(languageIdCodec: ILanguageIdCodec) { - this._lineTokens = []; - this._len = 0; - this._languageIdCodec = languageIdCodec; - } - - public flush(): void { - this._lineTokens = []; - this._len = 0; - } - - public getTokens(topLevelLanguageId: string, lineIndex: number, lineText: string): LineTokens { - let rawLineTokens: Uint32Array | ArrayBuffer | null = null; - if (lineIndex < this._len) { - rawLineTokens = this._lineTokens[lineIndex]; - } - - if (rawLineTokens !== null && rawLineTokens !== EMPTY_LINE_TOKENS) { - return new LineTokens(toUint32Array(rawLineTokens), lineText, this._languageIdCodec); - } - - const lineTokens = new Uint32Array(2); - lineTokens[0] = lineText.length; - lineTokens[1] = getDefaultMetadata(this._languageIdCodec.encodeLanguageId(topLevelLanguageId)); - return new LineTokens(lineTokens, lineText, this._languageIdCodec); - } - - private static _massageTokens(topLevelLanguageId: LanguageId, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null): Uint32Array | ArrayBuffer { - - const tokens = _tokens ? toUint32Array(_tokens) : null; - - if (lineTextLength === 0) { - let hasDifferentLanguageId = false; - if (tokens && tokens.length > 1) { - hasDifferentLanguageId = (TokenMetadata.getLanguageId(tokens[1]) !== topLevelLanguageId); - } - - if (!hasDifferentLanguageId) { - return EMPTY_LINE_TOKENS; - } - } - - if (!tokens || tokens.length === 0) { - const tokens = new Uint32Array(2); - tokens[0] = lineTextLength; - tokens[1] = getDefaultMetadata(topLevelLanguageId); - return tokens.buffer; - } - - // Ensure the last token covers the end of the text - tokens[tokens.length - 2] = lineTextLength; - - if (tokens.byteOffset === 0 && tokens.byteLength === tokens.buffer.byteLength) { - // Store directly the ArrayBuffer pointer to save an object - return tokens.buffer; - } - return tokens; - } - - private _ensureLine(lineIndex: number): void { - while (lineIndex >= this._len) { - this._lineTokens[this._len] = null; - this._len++; - } - } - - private _deleteLines(start: number, deleteCount: number): void { - if (deleteCount === 0) { - return; - } - if (start + deleteCount > this._len) { - deleteCount = this._len - start; - } - this._lineTokens.splice(start, deleteCount); - this._len -= deleteCount; - } - - private _insertLines(insertIndex: number, insertCount: number): void { - if (insertCount === 0) { - return; - } - const lineTokens: (Uint32Array | ArrayBuffer | null)[] = []; - for (let i = 0; i < insertCount; i++) { - lineTokens[i] = null; - } - this._lineTokens = arrays.arrayInsert(this._lineTokens, insertIndex, lineTokens); - this._len += insertCount; - } - - public setTokens(topLevelLanguageId: string, lineIndex: number, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null, checkEquality: boolean): boolean { - const tokens = TokensStore._massageTokens(this._languageIdCodec.encodeLanguageId(topLevelLanguageId), lineTextLength, _tokens); - this._ensureLine(lineIndex); - const oldTokens = this._lineTokens[lineIndex]; - this._lineTokens[lineIndex] = tokens; - - if (checkEquality) { - return !TokensStore._equals(oldTokens, tokens); - } - return false; - } - - private static _equals(_a: Uint32Array | ArrayBuffer | null, _b: Uint32Array | ArrayBuffer | null) { - if (!_a || !_b) { - return !_a && !_b; - } - - const a = toUint32Array(_a); - const b = toUint32Array(_b); - - if (a.length !== b.length) { - return false; - } - for (let i = 0, len = a.length; i < len; i++) { - if (a[i] !== b[i]) { - return false; - } - } - return true; - } - - //#region Editing - - public acceptEdit(range: IRange, eolCount: number, firstLineLength: number): void { - this._acceptDeleteRange(range); - this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength); - } - - private _acceptDeleteRange(range: IRange): void { - - const firstLineIndex = range.startLineNumber - 1; - if (firstLineIndex >= this._len) { - return; - } - - if (range.startLineNumber === range.endLineNumber) { - if (range.startColumn === range.endColumn) { - // Nothing to delete - return; - } - - this._lineTokens[firstLineIndex] = TokensStore._delete(this._lineTokens[firstLineIndex], range.startColumn - 1, range.endColumn - 1); - return; - } - - this._lineTokens[firstLineIndex] = TokensStore._deleteEnding(this._lineTokens[firstLineIndex], range.startColumn - 1); - - const lastLineIndex = range.endLineNumber - 1; - let lastLineTokens: Uint32Array | ArrayBuffer | null = null; - if (lastLineIndex < this._len) { - lastLineTokens = TokensStore._deleteBeginning(this._lineTokens[lastLineIndex], range.endColumn - 1); - } - - // Take remaining text on last line and append it to remaining text on first line - this._lineTokens[firstLineIndex] = TokensStore._append(this._lineTokens[firstLineIndex], lastLineTokens); - - // Delete middle lines - this._deleteLines(range.startLineNumber, range.endLineNumber - range.startLineNumber); - } - - private _acceptInsertText(position: Position, eolCount: number, firstLineLength: number): void { - - if (eolCount === 0 && firstLineLength === 0) { - // Nothing to insert - return; - } - - const lineIndex = position.lineNumber - 1; - if (lineIndex >= this._len) { - return; - } - - if (eolCount === 0) { - // Inserting text on one line - this._lineTokens[lineIndex] = TokensStore._insert(this._lineTokens[lineIndex], position.column - 1, firstLineLength); - return; - } - - this._lineTokens[lineIndex] = TokensStore._deleteEnding(this._lineTokens[lineIndex], position.column - 1); - this._lineTokens[lineIndex] = TokensStore._insert(this._lineTokens[lineIndex], position.column - 1, firstLineLength); - - this._insertLines(position.lineNumber, eolCount); - } - - public static _deleteBeginning(lineTokens: Uint32Array | ArrayBuffer | null, toChIndex: number): Uint32Array | ArrayBuffer | null { - if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) { - return lineTokens; - } - return TokensStore._delete(lineTokens, 0, toChIndex); - } - - public static _deleteEnding(lineTokens: Uint32Array | ArrayBuffer | null, fromChIndex: number): Uint32Array | ArrayBuffer | null { - if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) { - return lineTokens; - } - - const tokens = toUint32Array(lineTokens); - const lineTextLength = tokens[tokens.length - 2]; - return TokensStore._delete(lineTokens, fromChIndex, lineTextLength); - } - - public static _delete(lineTokens: Uint32Array | ArrayBuffer | null, fromChIndex: number, toChIndex: number): Uint32Array | ArrayBuffer | null { - if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS || fromChIndex === toChIndex) { - return lineTokens; - } - - const tokens = toUint32Array(lineTokens); - const tokensCount = (tokens.length >>> 1); - - // special case: deleting everything - if (fromChIndex === 0 && tokens[tokens.length - 2] === toChIndex) { - return EMPTY_LINE_TOKENS; - } - - const fromTokenIndex = LineTokens.findIndexInTokensArray(tokens, fromChIndex); - const fromTokenStartOffset = (fromTokenIndex > 0 ? tokens[(fromTokenIndex - 1) << 1] : 0); - const fromTokenEndOffset = tokens[fromTokenIndex << 1]; - - if (toChIndex < fromTokenEndOffset) { - // the delete range is inside a single token - const delta = (toChIndex - fromChIndex); - for (let i = fromTokenIndex; i < tokensCount; i++) { - tokens[i << 1] -= delta; - } - return lineTokens; - } - - let dest: number; - let lastEnd: number; - if (fromTokenStartOffset !== fromChIndex) { - tokens[fromTokenIndex << 1] = fromChIndex; - dest = ((fromTokenIndex + 1) << 1); - lastEnd = fromChIndex; - } else { - dest = (fromTokenIndex << 1); - lastEnd = fromTokenStartOffset; - } - - const delta = (toChIndex - fromChIndex); - for (let tokenIndex = fromTokenIndex + 1; tokenIndex < tokensCount; tokenIndex++) { - const tokenEndOffset = tokens[tokenIndex << 1] - delta; - if (tokenEndOffset > lastEnd) { - tokens[dest++] = tokenEndOffset; - tokens[dest++] = tokens[(tokenIndex << 1) + 1]; - lastEnd = tokenEndOffset; - } - } - - if (dest === tokens.length) { - // nothing to trim - return lineTokens; - } - - const tmp = new Uint32Array(dest); - tmp.set(tokens.subarray(0, dest), 0); - return tmp.buffer; - } - - public static _append(lineTokens: Uint32Array | ArrayBuffer | null, _otherTokens: Uint32Array | ArrayBuffer | null): Uint32Array | ArrayBuffer | null { - if (_otherTokens === EMPTY_LINE_TOKENS) { - return lineTokens; - } - if (lineTokens === EMPTY_LINE_TOKENS) { - return _otherTokens; - } - if (lineTokens === null) { - return lineTokens; - } - if (_otherTokens === null) { - // cannot determine combined line length... - return null; - } - const myTokens = toUint32Array(lineTokens); - const otherTokens = toUint32Array(_otherTokens); - const otherTokensCount = (otherTokens.length >>> 1); - - const result = new Uint32Array(myTokens.length + otherTokens.length); - result.set(myTokens, 0); - let dest = myTokens.length; - const delta = myTokens[myTokens.length - 2]; - for (let i = 0; i < otherTokensCount; i++) { - result[dest++] = otherTokens[(i << 1)] + delta; - result[dest++] = otherTokens[(i << 1) + 1]; - } - return result.buffer; - } - - public static _insert(lineTokens: Uint32Array | ArrayBuffer | null, chIndex: number, textLength: number): Uint32Array | ArrayBuffer | null { - if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) { - // nothing to do - return lineTokens; - } - - const tokens = toUint32Array(lineTokens); - const tokensCount = (tokens.length >>> 1); - - let fromTokenIndex = LineTokens.findIndexInTokensArray(tokens, chIndex); - if (fromTokenIndex > 0) { - const fromTokenStartOffset = tokens[(fromTokenIndex - 1) << 1]; - if (fromTokenStartOffset === chIndex) { - fromTokenIndex--; - } - } - for (let tokenIndex = fromTokenIndex; tokenIndex < tokensCount; tokenIndex++) { - tokens[tokenIndex << 1] += textLength; - } - return lineTokens; - } - - //#endregion -} diff --git a/src/vs/editor/common/modes/languageConfigurationRegistry.ts b/src/vs/editor/common/modes/languageConfigurationRegistry.ts index efe344243c7..56bb953a6cc 100644 --- a/src/vs/editor/common/modes/languageConfigurationRegistry.ts +++ b/src/vs/editor/common/modes/languageConfigurationRegistry.ts @@ -6,7 +6,7 @@ import { Emitter, Event } from 'vs/base/common/event'; import { Disposable, IDisposable, toDisposable } from 'vs/base/common/lifecycle'; import * as strings from 'vs/base/common/strings'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { Range } from 'vs/editor/common/core/range'; import { ITextModel } from 'vs/editor/common/model'; import { DEFAULT_WORD_REGEXP, ensureValidWordDefinition } from 'vs/editor/common/model/wordHelper'; diff --git a/src/vs/editor/common/modes/supports.ts b/src/vs/editor/common/modes/supports.ts index b770683a5de..47783dd80ad 100644 --- a/src/vs/editor/common/modes/supports.ts +++ b/src/vs/editor/common/modes/supports.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { StandardTokenType } from 'vs/editor/common/modes'; export function createScopedLineTokens(context: LineTokens, offset: number): ScopedLineTokens { diff --git a/src/vs/editor/common/modes/textToHtmlTokenizer.ts b/src/vs/editor/common/modes/textToHtmlTokenizer.ts index 56e4171dc53..27fb061c3f5 100644 --- a/src/vs/editor/common/modes/textToHtmlTokenizer.ts +++ b/src/vs/editor/common/modes/textToHtmlTokenizer.ts @@ -5,7 +5,7 @@ import { CharCode } from 'vs/base/common/charCode'; import * as strings from 'vs/base/common/strings'; -import { IViewLineTokens, LineTokens } from 'vs/editor/common/core/lineTokens'; +import { IViewLineTokens, LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { ILanguageIdCodec, IState, ITokenizationSupport, LanguageId, TokenizationRegistry } from 'vs/editor/common/modes'; import { NullState, nullTokenizeEncoded } from 'vs/editor/common/modes/nullMode'; import { ILanguageService } from 'vs/editor/common/services/language'; diff --git a/src/vs/editor/common/services/semanticTokensProviderStyling.ts b/src/vs/editor/common/services/semanticTokensProviderStyling.ts index 2a43495bcc5..6bd6722ed6d 100644 --- a/src/vs/editor/common/services/semanticTokensProviderStyling.ts +++ b/src/vs/editor/common/services/semanticTokensProviderStyling.ts @@ -6,7 +6,7 @@ import { SemanticTokensLegend, TokenMetadata, FontStyle, MetadataConsts, SemanticTokens } from 'vs/editor/common/modes'; import { IThemeService } from 'vs/platform/theme/common/themeService'; import { ILogService, LogLevel } from 'vs/platform/log/common/log'; -import { MultilineTokens2, SparseEncodedTokens } from 'vs/editor/common/model/tokensStore'; +import { SparseMultilineTokens } from 'vs/editor/common/model/tokens/sparseMultilineTokens'; import { ILanguageService } from 'vs/editor/common/services/language'; export const enum SemanticTokensProviderStylingConstants { @@ -123,11 +123,11 @@ const enum SemanticColoringConstants { DesiredMaxAreas = 1024, } -export function toMultilineTokens2(tokens: SemanticTokens, styling: SemanticTokensProviderStyling, languageId: string): MultilineTokens2[] { +export function toMultilineTokens2(tokens: SemanticTokens, styling: SemanticTokensProviderStyling, languageId: string): SparseMultilineTokens[] { const srcData = tokens.data; const tokenCount = (tokens.data.length / 5) | 0; const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea); - const result: MultilineTokens2[] = []; + const result: SparseMultilineTokens[] = []; let tokenIndex = 0; let lastLineNumber = 1; @@ -209,7 +209,7 @@ export function toMultilineTokens2(tokens: SemanticTokens, styling: SemanticToke destData = destData.subarray(0, destOffset); } - const tokens = new MultilineTokens2(areaLine, new SparseEncodedTokens(destData)); + const tokens = SparseMultilineTokens.create(areaLine, destData); result.push(tokens); } diff --git a/src/vs/editor/common/viewLayout/viewLineRenderer.ts b/src/vs/editor/common/viewLayout/viewLineRenderer.ts index 44ec32b8bd7..0c1c7baf26a 100644 --- a/src/vs/editor/common/viewLayout/viewLineRenderer.ts +++ b/src/vs/editor/common/viewLayout/viewLineRenderer.ts @@ -5,7 +5,7 @@ import { CharCode } from 'vs/base/common/charCode'; import * as strings from 'vs/base/common/strings'; -import { IViewLineTokens } from 'vs/editor/common/core/lineTokens'; +import { IViewLineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { IStringBuilder, createStringBuilder } from 'vs/editor/common/core/stringBuilder'; import { LineDecoration, LineDecorationsNormalizer } from 'vs/editor/common/viewLayout/lineDecorations'; import { InlineDecorationType } from 'vs/editor/common/viewModel/viewModel'; diff --git a/src/vs/editor/common/viewModel/modelLineProjection.ts b/src/vs/editor/common/viewModel/modelLineProjection.ts index 3501cdfc575..04ab3ea68b9 100644 --- a/src/vs/editor/common/viewModel/modelLineProjection.ts +++ b/src/vs/editor/common/viewModel/modelLineProjection.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { Position } from 'vs/editor/common/core/position'; import { IRange } from 'vs/editor/common/core/range'; import { EndOfLinePreference, ITextModel, PositionAffinity } from 'vs/editor/common/model'; diff --git a/src/vs/editor/common/viewModel/viewModel.ts b/src/vs/editor/common/viewModel/viewModel.ts index 657a80dde65..9cd63df18b1 100644 --- a/src/vs/editor/common/viewModel/viewModel.ts +++ b/src/vs/editor/common/viewModel/viewModel.ts @@ -7,7 +7,7 @@ import { IScrollPosition, Scrollable } from 'vs/base/common/scrollable'; import * as strings from 'vs/base/common/strings'; import { CursorConfiguration, CursorState, EditOperationType, IColumnSelectData, ICursorSimpleModel, PartialCursorState } from 'vs/editor/common/controller/cursorCommon'; import { CursorChangeReason } from 'vs/editor/common/controller/cursorEvents'; -import { IViewLineTokens } from 'vs/editor/common/core/lineTokens'; +import { IViewLineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { IPosition, Position } from 'vs/editor/common/core/position'; import { IRange, Range } from 'vs/editor/common/core/range'; import { INewScrollPosition, ScrollType } from 'vs/editor/common/editorCommon'; diff --git a/src/vs/editor/contrib/folding/hiddenRangeModel.ts b/src/vs/editor/contrib/folding/hiddenRangeModel.ts index 5d5d0698725..4f246928ff9 100644 --- a/src/vs/editor/contrib/folding/hiddenRangeModel.ts +++ b/src/vs/editor/contrib/folding/hiddenRangeModel.ts @@ -10,7 +10,7 @@ import { IDisposable } from 'vs/base/common/lifecycle'; import { IRange, Range } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; import { IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents'; -import { countEOL } from 'vs/editor/common/model/tokensStore'; +import { countEOL } from 'vs/editor/common/model/pieceTreeTextBuffer/eolCounter'; import { CollapseMemento, FoldingModel } from 'vs/editor/contrib/folding/foldingModel'; export class HiddenRangeModel { diff --git a/src/vs/editor/contrib/inlineCompletions/ghostTextWidget.ts b/src/vs/editor/contrib/inlineCompletions/ghostTextWidget.ts index c20bc73d668..f546862b3a0 100644 --- a/src/vs/editor/contrib/inlineCompletions/ghostTextWidget.ts +++ b/src/vs/editor/contrib/inlineCompletions/ghostTextWidget.ts @@ -10,7 +10,7 @@ import 'vs/css!./ghostText'; import { applyFontInfo } from 'vs/editor/browser/config/domFontInfo'; import { ContentWidgetPositionPreference, ICodeEditor, IContentWidget, IContentWidgetPosition } from 'vs/editor/browser/editorBrowser'; import { EditorFontLigatures, EditorOption, IComputedEditorOptions } from 'vs/editor/common/config/editorOptions'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; import { createStringBuilder } from 'vs/editor/common/core/stringBuilder'; diff --git a/src/vs/editor/standalone/browser/colorizer.ts b/src/vs/editor/standalone/browser/colorizer.ts index afe059a00ad..8ccbbc336ab 100644 --- a/src/vs/editor/standalone/browser/colorizer.ts +++ b/src/vs/editor/standalone/browser/colorizer.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as strings from 'vs/base/common/strings'; -import { IViewLineTokens, LineTokens } from 'vs/editor/common/core/lineTokens'; +import { IViewLineTokens, LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { ITextModel } from 'vs/editor/common/model'; import { ColorId, FontStyle, ILanguageIdCodec, ITokenizationSupport, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes'; import { ILanguageService } from 'vs/editor/common/services/language'; diff --git a/src/vs/editor/test/browser/viewModel/modelLineProjection.test.ts b/src/vs/editor/test/browser/viewModel/modelLineProjection.test.ts index 4eb7d5fe3d4..0bd14298bfb 100644 --- a/src/vs/editor/test/browser/viewModel/modelLineProjection.test.ts +++ b/src/vs/editor/test/browser/viewModel/modelLineProjection.test.ts @@ -5,7 +5,7 @@ import * as assert from 'assert'; import { IDisposable } from 'vs/base/common/lifecycle'; -import { IViewLineTokens } from 'vs/editor/common/core/lineTokens'; +import { IViewLineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { Position } from 'vs/editor/common/core/position'; import { IRange, Range } from 'vs/editor/common/core/range'; import { EncodedTokenizationResult } from 'vs/editor/common/core/token'; diff --git a/src/vs/editor/test/common/core/lineTokens.test.ts b/src/vs/editor/test/common/core/lineTokens.test.ts index 778c1686c92..94ec3d4d519 100644 --- a/src/vs/editor/test/common/core/lineTokens.test.ts +++ b/src/vs/editor/test/common/core/lineTokens.test.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as assert from 'assert'; -import { IViewLineTokens, LineTokens } from 'vs/editor/common/core/lineTokens'; +import { IViewLineTokens, LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { MetadataConsts } from 'vs/editor/common/modes'; import { LanguageIdCodec } from 'vs/editor/common/services/languagesRegistry'; diff --git a/src/vs/editor/test/common/core/viewLineToken.ts b/src/vs/editor/test/common/core/viewLineToken.ts index d3a8b8b8500..6c7fce7f644 100644 --- a/src/vs/editor/test/common/core/viewLineToken.ts +++ b/src/vs/editor/test/common/core/viewLineToken.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { IViewLineTokens } from 'vs/editor/common/core/lineTokens'; +import { IViewLineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { ColorId, TokenMetadata } from 'vs/editor/common/modes'; /** diff --git a/src/vs/editor/test/common/model/model.line.test.ts b/src/vs/editor/test/common/model/model.line.test.ts index 56b6ee60d4b..105a5afe27b 100644 --- a/src/vs/editor/test/common/model/model.line.test.ts +++ b/src/vs/editor/test/common/model/model.line.test.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import * as assert from 'assert'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { Range } from 'vs/editor/common/core/range'; import { computeIndentLevel } from 'vs/editor/common/model/utils'; import { MetadataConsts } from 'vs/editor/common/modes'; diff --git a/src/vs/editor/test/common/model/tokensStore.test.ts b/src/vs/editor/test/common/model/tokensStore.test.ts index 73ec1af76db..e9bb1db5df9 100644 --- a/src/vs/editor/test/common/model/tokensStore.test.ts +++ b/src/vs/editor/test/common/model/tokensStore.test.ts @@ -4,20 +4,21 @@ *--------------------------------------------------------------------------------------------*/ import * as assert from 'assert'; -import { MultilineTokens2, SparseEncodedTokens, TokensStore2 } from 'vs/editor/common/model/tokensStore'; +import { SparseMultilineTokens } from 'vs/editor/common/model/tokens/sparseMultilineTokens'; +import { SparseTokensStore } from 'vs/editor/common/model/tokens/sparseTokensStore'; import { Range } from 'vs/editor/common/core/range'; import { TextModel } from 'vs/editor/common/model/textModel'; import { IIdentifiedSingleEditOperation } from 'vs/editor/common/model'; import { MetadataConsts, TokenMetadata, FontStyle, ColorId } from 'vs/editor/common/modes'; import { createTextModel } from 'vs/editor/test/common/testTextModel'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { LanguageIdCodec } from 'vs/editor/common/services/languagesRegistry'; suite('TokensStore', () => { const SEMANTIC_COLOR: ColorId = 5; - function parseTokensState(state: string[]): { text: string; tokens: MultilineTokens2; } { + function parseTokensState(state: string[]): { text: string; tokens: SparseMultilineTokens; } { let text: string[] = []; let tokens: number[] = []; let baseLine = 1; @@ -66,7 +67,7 @@ suite('TokensStore', () => { return { text: text.join('\n'), - tokens: new MultilineTokens2(baseLine, new SparseEncodedTokens(new Uint32Array(tokens))) + tokens: SparseMultilineTokens.create(baseLine, new Uint32Array(tokens)) }; } @@ -174,7 +175,7 @@ suite('TokensStore', () => { test('issue #91936: Semantic token color highlighting fails on line with selected text', () => { const model = createTextModel(' else if ($s = 08) then \'\\b\''); model.setSemanticTokens([ - new MultilineTokens2(1, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(1, new Uint32Array([ 0, 20, 24, 0b0111100000000010000, 0, 25, 27, 0b0111100000000010000, 0, 28, 29, 0b0000100000000010000, @@ -184,7 +185,7 @@ suite('TokensStore', () => { 0, 36, 37, 0b0000100000000010000, 0, 38, 42, 0b0111100000000010000, 0, 43, 47, 0b0101100000000010000, - ]))) + ])) ], true); const lineTokens = model.getLineTokens(1); let decodedTokens: number[] = []; @@ -216,137 +217,137 @@ suite('TokensStore', () => { test('partial tokens 1', () => { const codec = new LanguageIdCodec(); - const store = new TokensStore2(codec); + const store = new SparseTokensStore(codec); // setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)] store.setPartial(new Range(1, 1, 31, 2), [ - new MultilineTokens2(5, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(5, new Uint32Array([ 0, 5, 10, 1, 5, 5, 10, 2, 10, 5, 10, 3, 15, 5, 10, 4, 20, 5, 10, 5, 25, 5, 10, 6, - ]))) + ])) ]); // setPartial: [18,1 -> 42,1], [(20,5-10),(25,5-10),(30,5-10),(35,5-10),(40,5-10)] store.setPartial(new Range(18, 1, 42, 1), [ - new MultilineTokens2(20, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(20, new Uint32Array([ 0, 5, 10, 4, 5, 5, 10, 5, 10, 5, 10, 6, 15, 5, 10, 7, 20, 5, 10, 8, - ]))) + ])) ]); // setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)] store.setPartial(new Range(1, 1, 31, 2), [ - new MultilineTokens2(5, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(5, new Uint32Array([ 0, 5, 10, 1, 5, 5, 10, 2, 10, 5, 10, 3, 15, 5, 10, 4, 20, 5, 10, 5, 25, 5, 10, 6, - ]))) + ])) ]); - const lineTokens = store.addSemanticTokens(10, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec)); + const lineTokens = store.addSparseTokens(10, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec)); assert.strictEqual(lineTokens.getCount(), 3); }); test('partial tokens 2', () => { const codec = new LanguageIdCodec(); - const store = new TokensStore2(codec); + const store = new SparseTokensStore(codec); // setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)] store.setPartial(new Range(1, 1, 31, 2), [ - new MultilineTokens2(5, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(5, new Uint32Array([ 0, 5, 10, 1, 5, 5, 10, 2, 10, 5, 10, 3, 15, 5, 10, 4, 20, 5, 10, 5, 25, 5, 10, 6, - ]))) + ])) ]); // setPartial: [6,1 -> 36,2], [(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10),(35,5-10)] store.setPartial(new Range(6, 1, 36, 2), [ - new MultilineTokens2(10, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(10, new Uint32Array([ 0, 5, 10, 2, 5, 5, 10, 3, 10, 5, 10, 4, 15, 5, 10, 5, 20, 5, 10, 6, - ]))) + ])) ]); // setPartial: [17,1 -> 42,1], [(20,5-10),(25,5-10),(30,5-10),(35,5-10),(40,5-10)] store.setPartial(new Range(17, 1, 42, 1), [ - new MultilineTokens2(20, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(20, new Uint32Array([ 0, 5, 10, 4, 5, 5, 10, 5, 10, 5, 10, 6, 15, 5, 10, 7, 20, 5, 10, 8, - ]))) + ])) ]); - const lineTokens = store.addSemanticTokens(20, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec)); + const lineTokens = store.addSparseTokens(20, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec)); assert.strictEqual(lineTokens.getCount(), 3); }); test('partial tokens 3', () => { const codec = new LanguageIdCodec(); - const store = new TokensStore2(codec); + const store = new SparseTokensStore(codec); // setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)] store.setPartial(new Range(1, 1, 31, 2), [ - new MultilineTokens2(5, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(5, new Uint32Array([ 0, 5, 10, 1, 5, 5, 10, 2, 10, 5, 10, 3, 15, 5, 10, 4, 20, 5, 10, 5, 25, 5, 10, 6, - ]))) + ])) ]); // setPartial: [11,1 -> 16,2], [(15,5-10),(20,5-10)] store.setPartial(new Range(11, 1, 16, 2), [ - new MultilineTokens2(10, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(10, new Uint32Array([ 0, 5, 10, 3, 5, 5, 10, 4, - ]))) + ])) ]); - const lineTokens = store.addSemanticTokens(5, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec)); + const lineTokens = store.addSparseTokens(5, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec)); assert.strictEqual(lineTokens.getCount(), 3); }); test('issue #94133: Semantic colors stick around when using (only) range provider', () => { const codec = new LanguageIdCodec(); - const store = new TokensStore2(codec); + const store = new SparseTokensStore(codec); // setPartial: [1,1 -> 1,20] [(1,9-11)] store.setPartial(new Range(1, 1, 1, 20), [ - new MultilineTokens2(1, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(1, new Uint32Array([ 0, 9, 11, 1, - ]))) + ])) ]); // setPartial: [1,1 -> 1,20], [] store.setPartial(new Range(1, 1, 1, 20), []); - const lineTokens = store.addSemanticTokens(1, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec)); + const lineTokens = store.addSparseTokens(1, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec)); assert.strictEqual(lineTokens.getCount(), 1); }); test('bug', () => { - function createTokens(str: string): MultilineTokens2 { + function createTokens(str: string): SparseMultilineTokens { str = str.replace(/^\[\(/, ''); str = str.replace(/\)\]$/, ''); const strTokens = str.split('),('); @@ -364,11 +365,11 @@ suite('TokensStore', () => { } result.push(lineNumber - firstLineNumber, startChar, endChar, (lineNumber + startChar) % 13); } - return new MultilineTokens2(firstLineNumber, new SparseEncodedTokens(new Uint32Array(result))); + return SparseMultilineTokens.create(firstLineNumber, new Uint32Array(result)); } const codec = new LanguageIdCodec(); - const store = new TokensStore2(codec); + const store = new SparseTokensStore(codec); // setPartial [36446,1 -> 36475,115] [(36448,24-29),(36448,33-46),(36448,47-54),(36450,25-35),(36450,36-50),(36451,28-33),(36451,36-49),(36451,50-57),(36452,35-53),(36452,54-62),(36454,33-38),(36454,41-54),(36454,55-60),(36455,35-53),(36455,54-62),(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62),(36466,33-71),(36466,72-76),(36467,35-53),(36467,54-62),(36469,24-29),(36469,33-46),(36469,47-54),(36470,24-35),(36470,38-46),(36473,25-35),(36473,36-51),(36474,28-33),(36474,36-49),(36474,50-58),(36475,35-53),(36475,54-62)] store.setPartial( new Range(36446, 1, 36475, 115), @@ -390,7 +391,7 @@ suite('TokensStore', () => { [createTokens('[(36442,25-35),(36442,36-50),(36443,30-39),(36443,42-46),(36443,47-53),(36443,54-58),(36443,63-73),(36443,74-84),(36443,87-91),(36443,92-98),(36443,101-105),(36443,106-112),(36443,113-119),(36444,28-37),(36444,38-42),(36444,47-57),(36444,58-75),(36444,80-95),(36444,96-105),(36445,35-53),(36445,54-62),(36448,24-29),(36448,33-46),(36448,47-54),(36450,25-35),(36450,36-50),(36451,28-33),(36451,36-49),(36451,50-57),(36452,35-53),(36452,54-62),(36454,33-38),(36454,41-54),(36454,55-60),(36455,35-53),(36455,54-62),(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62),(36466,33-71),(36466,72-76),(36467,35-53),(36467,54-62),(36469,24-29),(36469,33-46),(36469,47-54),(36470,24-35)]')] ); - const lineTokens = store.addSemanticTokens(36451, new LineTokens(new Uint32Array([60, 1]), ` if (flags & ModifierFlags.Ambient) {`, codec)); + const lineTokens = store.addSparseTokens(36451, new LineTokens(new Uint32Array([60, 1]), ` if (flags & ModifierFlags.Ambient) {`, codec)); assert.strictEqual(lineTokens.getCount(), 7); }); @@ -415,15 +416,15 @@ suite('TokensStore', () => { } const codec = new LanguageIdCodec(); - const store = new TokensStore2(codec); + const store = new SparseTokensStore(codec); store.set([ - new MultilineTokens2(1, new SparseEncodedTokens(new Uint32Array([ + SparseMultilineTokens.create(1, new Uint32Array([ 0, 6, 11, (1 << MetadataConsts.FOREGROUND_OFFSET) | MetadataConsts.SEMANTIC_USE_FOREGROUND, - ]))) + ])) ], true); - const lineTokens = store.addSemanticTokens(1, new LineTokens(new Uint32Array([ + const lineTokens = store.addSparseTokens(1, new LineTokens(new Uint32Array([ 5, createTMMetadata(5, FontStyle.Bold, 53), 14, createTMMetadata(1, FontStyle.None, 53), 17, createTMMetadata(6, FontStyle.None, 53), diff --git a/src/vs/editor/test/common/modesTestUtils.ts b/src/vs/editor/test/common/modesTestUtils.ts index 04ff5a1749a..3b043fc7d59 100644 --- a/src/vs/editor/test/common/modesTestUtils.ts +++ b/src/vs/editor/test/common/modesTestUtils.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { MetadataConsts, StandardTokenType } from 'vs/editor/common/modes'; import { ScopedLineTokens, createScopedLineTokens } from 'vs/editor/common/modes/supports'; import { LanguageIdCodec } from 'vs/editor/common/services/languagesRegistry'; diff --git a/src/vs/editor/test/common/services/semanticTokensProviderStyling.test.ts b/src/vs/editor/test/common/services/semanticTokensProviderStyling.test.ts index 0a308587856..a9778fdb250 100644 --- a/src/vs/editor/test/common/services/semanticTokensProviderStyling.test.ts +++ b/src/vs/editor/test/common/services/semanticTokensProviderStyling.test.ts @@ -5,7 +5,7 @@ import * as assert from 'assert'; import { DisposableStore } from 'vs/base/common/lifecycle'; -import { MultilineTokens2, SparseEncodedTokens } from 'vs/editor/common/model/tokensStore'; +import { SparseMultilineTokens } from 'vs/editor/common/model/tokens/sparseMultilineTokens'; import { MetadataConsts } from 'vs/editor/common/modes'; import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry'; import { SemanticTokensProviderStyling, toMultilineTokens2 } from 'vs/editor/common/services/semanticTokensProviderStyling'; @@ -64,7 +64,7 @@ suite('ModelService', () => { ]) }; const result = toMultilineTokens2(badTokens, styling, languageId); - const expected = new MultilineTokens2(1, new SparseEncodedTokens(new Uint32Array([ + const expected = SparseMultilineTokens.create(1, new Uint32Array([ 0, 13, 29, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (1 << MetadataConsts.FOREGROUND_OFFSET)), 1, 2, 8, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (2 << MetadataConsts.FOREGROUND_OFFSET)), 1, 9, 15, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (3 << MetadataConsts.FOREGROUND_OFFSET)), @@ -74,7 +74,7 @@ suite('ModelService', () => { 2, 12, 20, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (7 << MetadataConsts.FOREGROUND_OFFSET)), 2, 31, 36, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (8 << MetadataConsts.FOREGROUND_OFFSET)), 2, 36, 41, (MetadataConsts.SEMANTIC_USE_FOREGROUND | (9 << MetadataConsts.FOREGROUND_OFFSET)), - ]))); + ])); assert.deepStrictEqual(result.toString(), expected.toString()); }); }); diff --git a/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts b/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts index aa8d30b5eb5..e2efbb97b45 100644 --- a/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts +++ b/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts @@ -6,7 +6,7 @@ import * as assert from 'assert'; import { CharCode } from 'vs/base/common/charCode'; import * as strings from 'vs/base/common/strings'; -import { IViewLineTokens } from 'vs/editor/common/core/lineTokens'; +import { IViewLineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { MetadataConsts } from 'vs/editor/common/modes'; import { LineDecoration } from 'vs/editor/common/viewLayout/lineDecorations'; import { CharacterMapping, RenderLineInput, renderViewLine2 as renderViewLine, LineRange, DomPosition } from 'vs/editor/common/viewLayout/viewLineRenderer'; diff --git a/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts b/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts index a38928849c5..26559508c78 100644 --- a/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts +++ b/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts @@ -23,7 +23,7 @@ import { editorHoverBackground, editorHoverBorder } from 'vs/platform/theme/comm import { registerThemingParticipant } from 'vs/platform/theme/common/themeService'; import { findMatchingThemeRule } from 'vs/workbench/services/textMate/common/TMHelper'; import { ITextMateService } from 'vs/workbench/services/textMate/browser/textMate'; -import { IGrammar, IToken, StackElement } from 'vscode-textmate'; +import type { IGrammar, IToken, StackElement } from 'vscode-textmate'; import { IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService'; import { CancellationTokenSource } from 'vs/base/common/cancellation'; import { ColorThemeData, TokenStyleDefinitions, TokenStyleDefinition, TextMateThemingRuleDefinitions } from 'vs/workbench/services/themes/common/colorThemeData'; diff --git a/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts b/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts index 96fc80d817c..973a6643b0e 100644 --- a/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts +++ b/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts @@ -11,7 +11,7 @@ import { IWorkbenchThemeService, IWorkbenchColorTheme } from 'vs/workbench/servi import { IEditorService } from 'vs/workbench/services/editor/common/editorService'; import { EditorResourceAccessor } from 'vs/workbench/common/editor'; import { ITextMateService } from 'vs/workbench/services/textMate/browser/textMate'; -import { IGrammar, StackElement } from 'vscode-textmate'; +import type { IGrammar, StackElement } from 'vscode-textmate'; import { TokenizationRegistry, TokenMetadata } from 'vs/editor/common/modes'; import { ThemeRule, findMatchingThemeRule } from 'vs/workbench/services/textMate/common/TMHelper'; import { Color } from 'vs/base/common/color'; diff --git a/src/vs/workbench/services/textMate/browser/nativeTextMateService.ts b/src/vs/workbench/services/textMate/browser/nativeTextMateService.ts index 53659b2e2a5..d33d72eba49 100644 --- a/src/vs/workbench/services/textMate/browser/nativeTextMateService.ts +++ b/src/vs/workbench/services/textMate/browser/nativeTextMateService.ts @@ -19,7 +19,7 @@ import { TextMateWorker } from 'vs/workbench/services/textMate/browser/textMateW import { ITextModel } from 'vs/editor/common/model'; import { Disposable } from 'vs/base/common/lifecycle'; import { UriComponents, URI } from 'vs/base/common/uri'; -import { MultilineTokensBuilder } from 'vs/editor/common/model/tokensStore'; +import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/model/tokens/contiguousMultilineTokensBuilder'; import { TMGrammarFactory } from 'vs/workbench/services/textMate/common/TMGrammarFactory'; import { IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents'; import { IExtensionResourceLoaderService } from 'vs/workbench/services/extensionResourceLoader/common/extensionResourceLoader'; @@ -109,7 +109,7 @@ class ModelWorkerTextMateTokenizer extends Disposable { public setTokens(versionId: number, rawTokens: ArrayBuffer): void { this._confirm(versionId); - const tokens = MultilineTokensBuilder.deserialize(new Uint8Array(rawTokens)); + const tokens = ContiguousMultilineTokensBuilder.deserialize(new Uint8Array(rawTokens)); for (let i = 0; i < this._pendingChanges.length; i++) { const change = this._pendingChanges[i]; diff --git a/src/vs/workbench/services/textMate/browser/textMate.ts b/src/vs/workbench/services/textMate/browser/textMate.ts index 41963c90199..22808248af0 100644 --- a/src/vs/workbench/services/textMate/browser/textMate.ts +++ b/src/vs/workbench/services/textMate/browser/textMate.ts @@ -5,7 +5,7 @@ import { Event } from 'vs/base/common/event'; import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; -import { IGrammar } from 'vscode-textmate'; +import type { IGrammar } from 'vscode-textmate'; export const ITextMateService = createDecorator('textMateService'); diff --git a/src/vs/workbench/services/textMate/browser/textMateWorker.ts b/src/vs/workbench/services/textMate/browser/textMateWorker.ts index aaef23b3768..61e0471b685 100644 --- a/src/vs/workbench/services/textMate/browser/textMateWorker.ts +++ b/src/vs/workbench/services/textMate/browser/textMateWorker.ts @@ -12,8 +12,9 @@ import { IModelChangedEvent, MirrorTextModel } from 'vs/editor/common/model/mirr import { TextMateWorkerHost } from 'vs/workbench/services/textMate/browser/nativeTextMateService'; import { TokenizationStateStore } from 'vs/editor/common/model/textModelTokens'; import type { IGrammar, StackElement, IRawTheme, IOnigLib } from 'vscode-textmate'; -import { MultilineTokensBuilder, countEOL } from 'vs/editor/common/model/tokensStore'; -import { LineTokens } from 'vs/editor/common/core/lineTokens'; +import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/model/tokens/contiguousMultilineTokensBuilder'; +import { countEOL } from 'vs/editor/common/model/pieceTreeTextBuffer/eolCounter'; +import { LineTokens } from 'vs/editor/common/model/tokens/lineTokens'; import { FileAccess } from 'vs/base/common/network'; export interface IValidGrammarDefinitionDTO { @@ -100,7 +101,7 @@ class TextMateWorkerModel extends MirrorTextModel { if (!this._grammar) { return; } - const builder = new MultilineTokensBuilder(); + const builder = new ContiguousMultilineTokensBuilder(); const lineCount = this._lines.length; // Validate all states up to and including endLineIndex