From 17f797bcd174e5130abe51ef96fd32116cd7678a Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Tue, 3 Mar 2020 22:23:31 +0100 Subject: [PATCH] Fixes #91936: Do not emit empty tokens in the model, add workaround for rendering empty tokens in the view --- src/vs/editor/common/model/tokensStore.ts | 29 ++++--- .../common/viewLayout/viewLineRenderer.ts | 2 +- .../test/common/model/tokensStore.test.ts | 43 ++++++++++ .../viewLayout/viewLineRenderer.test.ts | 86 +++++++++++++++++++ 4 files changed, 147 insertions(+), 13 deletions(-) diff --git a/src/vs/editor/common/model/tokensStore.ts b/src/vs/editor/common/model/tokensStore.ts index c662f820c9c..047ca4be628 100644 --- a/src/vs/editor/common/model/tokensStore.ts +++ b/src/vs/editor/common/model/tokensStore.ts @@ -781,6 +781,17 @@ export class TokensStore2 { let aIndex = 0; let result: number[] = [], resultLen = 0; + let lastEndOffset = 0; + + const emitToken = (endOffset: number, metadata: number) => { + if (endOffset === lastEndOffset) { + return; + } + lastEndOffset = endOffset; + result[resultLen++] = endOffset; + result[resultLen++] = metadata; + }; + for (let bIndex = 0; bIndex < bLen; bIndex++) { const bStartCharacter = bTokens.getStartCharacter(bIndex); const bEndCharacter = bTokens.getEndCharacter(bIndex); @@ -797,42 +808,36 @@ export class TokensStore2 { // push any token from `a` that is before `b` while (aIndex < aLen && aTokens.getEndOffset(aIndex) <= bStartCharacter) { - result[resultLen++] = aTokens.getEndOffset(aIndex); - result[resultLen++] = aTokens.getMetadata(aIndex); + emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex)); aIndex++; } // push the token from `a` if it intersects the token from `b` if (aIndex < aLen && aTokens.getStartOffset(aIndex) < bStartCharacter) { - result[resultLen++] = bStartCharacter; - result[resultLen++] = aTokens.getMetadata(aIndex); + emitToken(bStartCharacter, aTokens.getMetadata(aIndex)); } // skip any tokens from `a` that are contained inside `b` while (aIndex < aLen && aTokens.getEndOffset(aIndex) < bEndCharacter) { - result[resultLen++] = aTokens.getEndOffset(aIndex); - result[resultLen++] = (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask); + emitToken(aTokens.getEndOffset(aIndex), (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask)); aIndex++; } if (aIndex < aLen && aTokens.getEndOffset(aIndex) === bEndCharacter) { // `a` ends exactly at the same spot as `b`! - result[resultLen++] = aTokens.getEndOffset(aIndex); - result[resultLen++] = (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask); + emitToken(aTokens.getEndOffset(aIndex), (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask)); aIndex++; } else { const aMergeIndex = Math.min(Math.max(0, aIndex - 1), aLen - 1); // push the token from `b` - result[resultLen++] = bEndCharacter; - result[resultLen++] = (aTokens.getMetadata(aMergeIndex) & aMask) | (bMetadata & bMask); + emitToken(bEndCharacter, (aTokens.getMetadata(aMergeIndex) & aMask) | (bMetadata & bMask)); } } // push the remaining tokens from `a` while (aIndex < aLen) { - result[resultLen++] = aTokens.getEndOffset(aIndex); - result[resultLen++] = aTokens.getMetadata(aIndex); + emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex)); aIndex++; } diff --git a/src/vs/editor/common/viewLayout/viewLineRenderer.ts b/src/vs/editor/common/viewLayout/viewLineRenderer.ts index c032724b8ba..e551e63929d 100644 --- a/src/vs/editor/common/viewLayout/viewLineRenderer.ts +++ b/src/vs/editor/common/viewLayout/viewLineRenderer.ts @@ -683,7 +683,7 @@ function _applyRenderWhitespace(input: RenderLineInput, lineContent: string, len wasInWhitespace = isInWhitespace; - if (charIndex === tokenEndIndex) { + while (charIndex === tokenEndIndex) { tokenIndex++; if (tokenIndex < tokensLength) { tokenType = tokens[tokenIndex].type; diff --git a/src/vs/editor/test/common/model/tokensStore.test.ts b/src/vs/editor/test/common/model/tokensStore.test.ts index 51d85e382f8..05d8c435b28 100644 --- a/src/vs/editor/test/common/model/tokensStore.test.ts +++ b/src/vs/editor/test/common/model/tokensStore.test.ts @@ -169,4 +169,47 @@ suite('TokensStore', () => { ); }); + test('issue #91936: Semantic token color highlighting fails on line with selected text', () => { + const model = createTextModel(' else if ($s = 08) then \'\\b\''); + model.setSemanticTokens([ + new MultilineTokens2(1, new SparseEncodedTokens(new Uint32Array([ + 0, 20, 24, 245768, + 0, 25, 27, 245768, + 0, 28, 29, 16392, + 0, 29, 31, 262152, + 0, 32, 33, 16392, + 0, 34, 36, 98312, + 0, 36, 37, 16392, + 0, 38, 42, 245768, + 0, 43, 47, 180232, + ]))) + ]); + const lineTokens = model.getLineTokens(1); + let decodedTokens: number[] = []; + for (let i = 0, len = lineTokens.getCount(); i < len; i++) { + decodedTokens.push(lineTokens.getEndOffset(i), lineTokens.getMetadata(i)); + } + + assert.deepEqual(decodedTokens, [ + 20, 16793600, + 24, 17022976, + 25, 16793600, + 27, 17022976, + 28, 16793600, + 29, 16793600, + 31, 17039360, + 32, 16793600, + 33, 16793600, + 34, 16793600, + 36, 16875520, + 37, 16793600, + 38, 16793600, + 42, 17022976, + 43, 16793600, + 47, 16957440 + ]); + + model.dispose(); + }); + }); diff --git a/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts b/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts index f5e4acbc157..7c94bf13248 100644 --- a/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts +++ b/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts @@ -1861,6 +1861,92 @@ suite('viewLineRenderer.renderLine 2', () => { assert.deepEqual(actual.html, expected); }); + test('issue #91936: Semantic token color highlighting fails on line with selected text', () => { + let actual = renderViewLine(new RenderLineInput( + false, + true, + ' else if ($s = 08) then \'\\b\'', + false, + true, + false, + 0, + createViewLineTokens([ + createPart(20, 1), + createPart(24, 15), + createPart(25, 1), + createPart(27, 15), + createPart(28, 1), + createPart(29, 1), + createPart(29, 1), + createPart(31, 16), + createPart(32, 1), + createPart(33, 1), + createPart(34, 1), + createPart(36, 6), + createPart(36, 1), + createPart(37, 1), + createPart(38, 1), + createPart(42, 15), + createPart(43, 1), + createPart(47, 11) + ]), + [], + 4, + 0, + 10, + 11, + 11, + 10000, + 'selection', + false, + false, + [new LineRange(0, 47)] + )); + + let expected = [ + '', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + '\u00b7', + 'else', + '\u00b7', + 'if', + '\u00b7', + '(', + '$s', + '\u00b7', + '=', + '\u00b7', + '08', + ')', + '\u00b7', + 'then', + '\u00b7', + '\'\\b\'', + '' + ].join(''); + + assert.deepEqual(actual.html, expected); + }); + + function createTestGetColumnOfLinePartOffset(lineContent: string, tabSize: number, parts: ViewLineToken[], expectedPartLengths: number[]): (partIndex: number, partLength: number, offset: number, expected: number) => void { let renderLineOutput = renderViewLine(new RenderLineInput( false,