From 525509f7c6b0d7acbd5016dbcc4b4cf2eae7cfca Mon Sep 17 00:00:00 2001 From: Henning Dieterichs Date: Mon, 14 Nov 2022 15:28:27 +0100 Subject: [PATCH] Fixes tests. --- .../beforeEditPositionMapper.ts | 9 ++++++--- .../bracketPairsTree/bracketPairsTree.ts | 2 +- .../bracketPairsTree/parser.ts | 8 +++++--- .../bracketPairsTree/tokenizer.ts | 2 +- .../getBracketPairsInRange.test.ts | 18 +++++++++++++++++- .../bracketPairColorizer/tokenizer.test.ts | 4 ++-- 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/beforeEditPositionMapper.ts b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/beforeEditPositionMapper.ts index 7d9e19c05d8..162509bed20 100644 --- a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/beforeEditPositionMapper.ts +++ b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/beforeEditPositionMapper.ts @@ -26,7 +26,6 @@ export class BeforeEditPositionMapper { */ constructor( edits: readonly TextEditInfo[], - private readonly documentLength: Length, ) { this.edits = edits.map(edit => TextEditInfoCache.from(edit)); } @@ -41,12 +40,16 @@ export class BeforeEditPositionMapper { /** * @param offset Must be equal to or greater than the last offset this method has been called with. + * Returns null if there is no edit anymore. */ - getDistanceToNextChange(offset: Length): Length { + getDistanceToNextChange(offset: Length): Length | null { this.adjustNextEdit(offset); const nextEdit = this.edits[this.nextEditIdx]; - const nextChangeOffset = nextEdit ? this.translateOldToCur(nextEdit.offsetObj) : this.documentLength; + const nextChangeOffset = nextEdit ? this.translateOldToCur(nextEdit.offsetObj) : null; + if (nextChangeOffset === null) { + return null; + } return lengthDiffNonNegative(offset, nextChangeOffset); } diff --git a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/bracketPairsTree.ts b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/bracketPairsTree.ts index 7fa3b29f4e9..1a85bec9ec6 100644 --- a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/bracketPairsTree.ts +++ b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/bracketPairsTree.ts @@ -131,7 +131,7 @@ export class BracketPairsTree extends Disposable { this.astWithTokens = this.parseDocumentFromTextBuffer(this.queuedTextEdits, this.astWithTokens, false); this.queuedTextEdits = []; } - if (this.queuedTextEditsForInitialAstWithoutTokens) { + if (this.queuedTextEditsForInitialAstWithoutTokens.length > 0) { if (this.initialAstWithoutTokens) { this.initialAstWithoutTokens = this.parseDocumentFromTextBuffer(this.queuedTextEditsForInitialAstWithoutTokens, this.initialAstWithoutTokens, false); } diff --git a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/parser.ts b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/parser.ts index 7aebcb52520..8758dfb3ea0 100644 --- a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/parser.ts +++ b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/parser.ts @@ -53,7 +53,7 @@ class Parser { } this.oldNodeReader = oldNode ? new NodeReader(oldNode) : undefined; - this.positionMapper = new BeforeEditPositionMapper(edits, tokenizer.length); + this.positionMapper = new BeforeEditPositionMapper(edits); } parseDocument(): AstNode { @@ -104,9 +104,11 @@ class Parser { private tryReadChildFromCache(openedBracketIds: SmallImmutableSet): AstNode | undefined { if (this.oldNodeReader) { const maxCacheableLength = this.positionMapper.getDistanceToNextChange(this.tokenizer.offset); - if (!lengthIsZero(maxCacheableLength)) { + if (maxCacheableLength === null || !lengthIsZero(maxCacheableLength)) { const cachedNode = this.oldNodeReader.readLongestNodeAt(this.positionMapper.getOffsetBeforeChange(this.tokenizer.offset), curNode => { - if (!lengthLessThan(curNode.length, maxCacheableLength)) { + // The edit could extend the ending token, thus we cannot re-use nodes that touch the edit. + // If there is no edit anymore, we can re-use the node in any case. + if (maxCacheableLength !== null && !lengthLessThan(curNode.length, maxCacheableLength)) { // Either the node contains edited text or touches edited text. // In the latter case, brackets might have been extended (`end` -> `ending`), so even touching nodes cannot be reused. return false; diff --git a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts index af1d10112ce..c5d8898ded7 100644 --- a/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts +++ b/src/vs/editor/common/model/bracketPairsTextModelPart/bracketPairsTree/tokenizer.ts @@ -81,7 +81,7 @@ export class TextBufferTokenizer implements Tokenizer { } get length() { - return toLength(this.textBufferLineCount, this.textBufferLastLineLength); + return toLength(this.textBufferLineCount - 1, this.textBufferLastLineLength); } getText() { diff --git a/src/vs/editor/test/common/model/bracketPairColorizer/getBracketPairsInRange.test.ts b/src/vs/editor/test/common/model/bracketPairColorizer/getBracketPairsInRange.test.ts index c938b730a46..e44874f2b7e 100644 --- a/src/vs/editor/test/common/model/bracketPairColorizer/getBracketPairsInRange.test.ts +++ b/src/vs/editor/test/common/model/bracketPairColorizer/getBracketPairsInRange.test.ts @@ -11,6 +11,10 @@ import { BracketPairInfo } from 'vs/editor/common/textModelBracketPairs'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; import { createModelServices, instantiateTextModel } from 'vs/editor/test/common/testTextModel'; import { TextModel } from 'vs/editor/common/model/textModel'; +import { TokenInfo, TokenizedDocument } from 'vs/editor/test/common/model/bracketPairColorizer/tokenizer.test'; +import { ILanguageService } from 'vs/editor/common/languages/language'; +import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { TokenizationRegistry } from 'vs/editor/common/languages'; suite('Bracket Pair Colorizer - getBracketPairsInRange', () => { @@ -18,6 +22,16 @@ suite('Bracket Pair Colorizer - getBracketPairsInRange', () => { const languageId = 'testLanguage'; const instantiationService = createModelServices(store); const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const languageService = instantiationService.get(ILanguageService); + languageService.registerLanguage({ + id: languageId, + }); + + const encodedMode1 = languageService.languageIdCodec.encodeLanguageId(languageId); + const document = new TokenizedDocument([ + new TokenInfo(text, encodedMode1, StandardTokenType.Other, true) + ]); + store.add(TokenizationRegistry.register(languageId, document.getTokenizationSupport())); store.add(languageConfigurationService.register(languageId, { colorizedBracketPairs: [ @@ -26,13 +40,15 @@ suite('Bracket Pair Colorizer - getBracketPairsInRange', () => { ['(', ')'], ] })); - return store.add(instantiateTextModel(instantiationService, text, languageId)); + const textModel = store.add(instantiateTextModel(instantiationService, text, languageId)); + return textModel; } test('Basic 1', () => { disposeOnReturn(store => { const doc = new AnnotatedDocument(`{ ( [] ¹ ) [ ² { } ] () } []`); const model = createTextModelWithColorizedBracketPairs(store, doc.text); + model.tokenization.getLineTokens(1).getLanguageId(0); assert.deepStrictEqual( model.bracketPairs .getBracketPairsInRange(doc.range(1, 2)) diff --git a/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts b/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts index d1800b09f0d..ca5535c0eb5 100644 --- a/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts +++ b/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts @@ -124,7 +124,7 @@ function tokenToObj(token: Token, offset: Length, model: TextModel, keyProvider: }; } -class TokenizedDocument { +export class TokenizedDocument { private readonly tokensByLine: readonly TokenInfo[][]; constructor(tokens: TokenInfo[]) { const tokensByLine = new Array(); @@ -189,7 +189,7 @@ class TokenizedDocument { } } -class TokenInfo { +export class TokenInfo { constructor( public readonly text: string, public readonly languageId: LanguageId,