mirror of
https://github.com/microsoft/vscode.git
synced 2026-07-13 23:44:09 +01:00
Fixes #91936: Do not emit empty tokens in the model, add workaround for rendering empty tokens in the view
This commit is contained in:
Alex Dima
@@ -781,6 +781,17 @@ export class TokensStore2 {
|
||||
|
||||
let aIndex = 0;
|
||||
let result: number[] = [], resultLen = 0;
|
||||
let lastEndOffset = 0;
|
||||
|
||||
const emitToken = (endOffset: number, metadata: number) => {
|
||||
if (endOffset === lastEndOffset) {
|
||||
return;
|
||||
}
|
||||
lastEndOffset = endOffset;
|
||||
result[resultLen++] = endOffset;
|
||||
result[resultLen++] = metadata;
|
||||
};
|
||||
|
||||
for (let bIndex = 0; bIndex < bLen; bIndex++) {
|
||||
const bStartCharacter = bTokens.getStartCharacter(bIndex);
|
||||
const bEndCharacter = bTokens.getEndCharacter(bIndex);
|
||||
@@ -797,42 +808,36 @@ export class TokensStore2 {
|
||||
|
||||
// push any token from `a` that is before `b`
|
||||
while (aIndex < aLen && aTokens.getEndOffset(aIndex) <= bStartCharacter) {
|
||||
result[resultLen++] = aTokens.getEndOffset(aIndex);
|
||||
result[resultLen++] = aTokens.getMetadata(aIndex);
|
||||
emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex));
|
||||
aIndex++;
|
||||
}
|
||||
|
||||
// push the token from `a` if it intersects the token from `b`
|
||||
if (aIndex < aLen && aTokens.getStartOffset(aIndex) < bStartCharacter) {
|
||||
result[resultLen++] = bStartCharacter;
|
||||
result[resultLen++] = aTokens.getMetadata(aIndex);
|
||||
emitToken(bStartCharacter, aTokens.getMetadata(aIndex));
|
||||
}
|
||||
|
||||
// skip any tokens from `a` that are contained inside `b`
|
||||
while (aIndex < aLen && aTokens.getEndOffset(aIndex) < bEndCharacter) {
|
||||
result[resultLen++] = aTokens.getEndOffset(aIndex);
|
||||
result[resultLen++] = (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask);
|
||||
emitToken(aTokens.getEndOffset(aIndex), (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask));
|
||||
aIndex++;
|
||||
}
|
||||
|
||||
if (aIndex < aLen && aTokens.getEndOffset(aIndex) === bEndCharacter) {
|
||||
// `a` ends exactly at the same spot as `b`!
|
||||
result[resultLen++] = aTokens.getEndOffset(aIndex);
|
||||
result[resultLen++] = (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask);
|
||||
emitToken(aTokens.getEndOffset(aIndex), (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask));
|
||||
aIndex++;
|
||||
} else {
|
||||
const aMergeIndex = Math.min(Math.max(0, aIndex - 1), aLen - 1);
|
||||
|
||||
// push the token from `b`
|
||||
result[resultLen++] = bEndCharacter;
|
||||
result[resultLen++] = (aTokens.getMetadata(aMergeIndex) & aMask) | (bMetadata & bMask);
|
||||
emitToken(bEndCharacter, (aTokens.getMetadata(aMergeIndex) & aMask) | (bMetadata & bMask));
|
||||
}
|
||||
}
|
||||
|
||||
// push the remaining tokens from `a`
|
||||
while (aIndex < aLen) {
|
||||
result[resultLen++] = aTokens.getEndOffset(aIndex);
|
||||
result[resultLen++] = aTokens.getMetadata(aIndex);
|
||||
emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex));
|
||||
aIndex++;
|
||||
}
|
||||
|
||||
|
||||
@@ -683,7 +683,7 @@ function _applyRenderWhitespace(input: RenderLineInput, lineContent: string, len
|
||||
|
||||
wasInWhitespace = isInWhitespace;
|
||||
|
||||
if (charIndex === tokenEndIndex) {
|
||||
while (charIndex === tokenEndIndex) {
|
||||
tokenIndex++;
|
||||
if (tokenIndex < tokensLength) {
|
||||
tokenType = tokens[tokenIndex].type;
|
||||
|
||||
@@ -169,4 +169,47 @@ suite('TokensStore', () => {
|
||||
);
|
||||
});
|
||||
|
||||
test('issue #91936: Semantic token color highlighting fails on line with selected text', () => {
|
||||
const model = createTextModel(' else if ($s = 08) then \'\\b\'');
|
||||
model.setSemanticTokens([
|
||||
new MultilineTokens2(1, new SparseEncodedTokens(new Uint32Array([
|
||||
0, 20, 24, 245768,
|
||||
0, 25, 27, 245768,
|
||||
0, 28, 29, 16392,
|
||||
0, 29, 31, 262152,
|
||||
0, 32, 33, 16392,
|
||||
0, 34, 36, 98312,
|
||||
0, 36, 37, 16392,
|
||||
0, 38, 42, 245768,
|
||||
0, 43, 47, 180232,
|
||||
])))
|
||||
]);
|
||||
const lineTokens = model.getLineTokens(1);
|
||||
let decodedTokens: number[] = [];
|
||||
for (let i = 0, len = lineTokens.getCount(); i < len; i++) {
|
||||
decodedTokens.push(lineTokens.getEndOffset(i), lineTokens.getMetadata(i));
|
||||
}
|
||||
|
||||
assert.deepEqual(decodedTokens, [
|
||||
20, 16793600,
|
||||
24, 17022976,
|
||||
25, 16793600,
|
||||
27, 17022976,
|
||||
28, 16793600,
|
||||
29, 16793600,
|
||||
31, 17039360,
|
||||
32, 16793600,
|
||||
33, 16793600,
|
||||
34, 16793600,
|
||||
36, 16875520,
|
||||
37, 16793600,
|
||||
38, 16793600,
|
||||
42, 17022976,
|
||||
43, 16793600,
|
||||
47, 16957440
|
||||
]);
|
||||
|
||||
model.dispose();
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
@@ -1861,6 +1861,92 @@ suite('viewLineRenderer.renderLine 2', () => {
|
||||
assert.deepEqual(actual.html, expected);
|
||||
});
|
||||
|
||||
test('issue #91936: Semantic token color highlighting fails on line with selected text', () => {
|
||||
let actual = renderViewLine(new RenderLineInput(
|
||||
false,
|
||||
true,
|
||||
' else if ($s = 08) then \'\\b\'',
|
||||
false,
|
||||
true,
|
||||
false,
|
||||
0,
|
||||
createViewLineTokens([
|
||||
createPart(20, 1),
|
||||
createPart(24, 15),
|
||||
createPart(25, 1),
|
||||
createPart(27, 15),
|
||||
createPart(28, 1),
|
||||
createPart(29, 1),
|
||||
createPart(29, 1),
|
||||
createPart(31, 16),
|
||||
createPart(32, 1),
|
||||
createPart(33, 1),
|
||||
createPart(34, 1),
|
||||
createPart(36, 6),
|
||||
createPart(36, 1),
|
||||
createPart(37, 1),
|
||||
createPart(38, 1),
|
||||
createPart(42, 15),
|
||||
createPart(43, 1),
|
||||
createPart(47, 11)
|
||||
]),
|
||||
[],
|
||||
4,
|
||||
0,
|
||||
10,
|
||||
11,
|
||||
11,
|
||||
10000,
|
||||
'selection',
|
||||
false,
|
||||
false,
|
||||
[new LineRange(0, 47)]
|
||||
));
|
||||
|
||||
let expected = [
|
||||
'<span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtk15">else</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtk15">if</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtk1">(</span>',
|
||||
'<span class="mtk16">$s</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtk1">=</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtk6">08</span>',
|
||||
'<span class="mtk1">)</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtk15">then</span>',
|
||||
'<span class="mtkz" style="width:10px">\u00b7</span>',
|
||||
'<span class="mtk11">\'\\b\'</span>',
|
||||
'</span>'
|
||||
].join('');
|
||||
|
||||
assert.deepEqual(actual.html, expected);
|
||||
});
|
||||
|
||||
|
||||
function createTestGetColumnOfLinePartOffset(lineContent: string, tabSize: number, parts: ViewLineToken[], expectedPartLengths: number[]): (partIndex: number, partLength: number, offset: number, expected: number) => void {
|
||||
let renderLineOutput = renderViewLine(new RenderLineInput(
|
||||
false,
|
||||
|
||||
Reference in New Issue
Block a user