From 9fe7dabbdbfefa21fe35123d1bc9919578a364ba Mon Sep 17 00:00:00 2001 From: Alex Dima Date: Thu, 7 Apr 2022 16:38:21 +0200 Subject: [PATCH] Adopt `vscode-textmate@7.0.0` --- package.json | 2 +- remote/package.json | 2 +- remote/web/package.json | 2 +- remote/web/yarn.lock | 8 +-- remote/yarn.lock | 8 +-- src/vs/editor/common/languages.ts | 29 +++++++---- .../test/common/core/lineTokens.test.ts | 12 ++--- .../test/common/model/tokensStore.test.ts | 50 +++++++++---------- yarn.lock | 8 +-- 9 files changed, 64 insertions(+), 57 deletions(-) diff --git a/package.json b/package.json index 46ef28a3183..14ae62a2566 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,7 @@ "vscode-oniguruma": "1.6.1", "vscode-proxy-agent": "^0.12.0", "vscode-regexpp": "^3.1.0", - "vscode-textmate": "6.0.0", + "vscode-textmate": "7.0.0", "xterm": "4.19.0-beta.20", "xterm-addon-search": "0.9.0-beta.18", "xterm-addon-serialize": "0.7.0-beta.12", diff --git a/remote/package.json b/remote/package.json index 203c90916b6..b517ff7d59a 100644 --- a/remote/package.json +++ b/remote/package.json @@ -23,7 +23,7 @@ "vscode-oniguruma": "1.6.1", "vscode-proxy-agent": "^0.12.0", "vscode-regexpp": "^3.1.0", - "vscode-textmate": "6.0.0", + "vscode-textmate": "7.0.0", "xterm": "4.19.0-beta.20", "xterm-addon-search": "0.9.0-beta.18", "xterm-addon-serialize": "0.7.0-beta.12", diff --git a/remote/web/package.json b/remote/web/package.json index 1d3e920919e..4bf0db5224c 100644 --- a/remote/web/package.json +++ b/remote/web/package.json @@ -9,7 +9,7 @@ "jschardet": "3.0.0", "tas-client-umd": "0.1.4", "vscode-oniguruma": "1.6.1", - "vscode-textmate": "6.0.0", + "vscode-textmate": "7.0.0", "xterm": "4.19.0-beta.20", "xterm-addon-search": "0.9.0-beta.18", "xterm-addon-unicode11": "0.4.0-beta.3", diff --git a/remote/web/yarn.lock b/remote/web/yarn.lock index 0fbfec54b2c..628b59fc2b5 100644 --- a/remote/web/yarn.lock +++ b/remote/web/yarn.lock @@ -108,10 +108,10 @@ vscode-oniguruma@1.6.1: resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.6.1.tgz#2bf4dfcfe3dd2e56eb549a3068c8ee39e6c30ce5" integrity sha512-vc4WhSIaVpgJ0jJIejjYxPvURJavX6QG41vu0mGhqywMkQqulezEqEQ3cO3gc8GvcOpX6ycmKGqRoROEMBNXTQ== -vscode-textmate@6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-6.0.0.tgz#a3777197235036814ac9a92451492f2748589210" - integrity sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ== +vscode-textmate@7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-7.0.0.tgz#6c90a591c1291e80c6783bddf43e841ad6f8ccc3" + integrity sha512-27bFXGG5JtLrq/reWnkCxkfNq7gnFko7xORba2Gznokx2LmPpHfu880a44B1jPyoFiGNjprZ883qiYTvR/P+pA== xterm-addon-search@0.9.0-beta.18: version "0.9.0-beta.18" diff --git a/remote/yarn.lock b/remote/yarn.lock index 381610e5a8d..49022cbe3ad 100644 --- a/remote/yarn.lock +++ b/remote/yarn.lock @@ -883,10 +883,10 @@ vscode-regexpp@^3.1.0: resolved "https://registry.yarnpkg.com/vscode-regexpp/-/vscode-regexpp-3.1.0.tgz#42d059b6fffe99bd42939c0d013f632f0cad823f" integrity sha512-pqtN65VC1jRLawfluX4Y80MMG0DHJydWhe5ZwMHewZD6sys4LbU6lHwFAHxeuaVE6Y6+xZOtAw+9hvq7/0ejkg== -vscode-textmate@6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-6.0.0.tgz#a3777197235036814ac9a92451492f2748589210" - integrity sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ== +vscode-textmate@7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-7.0.0.tgz#6c90a591c1291e80c6783bddf43e841ad6f8ccc3" + integrity sha512-27bFXGG5JtLrq/reWnkCxkfNq7gnFko7xORba2Gznokx2LmPpHfu880a44B1jPyoFiGNjprZ883qiYTvR/P+pA== vscode-windows-ca-certs@^0.3.0: version "0.3.0" diff --git a/src/vs/editor/common/languages.ts b/src/vs/editor/common/languages.ts index de092a4d1be..570e85863ad 100644 --- a/src/vs/editor/common/languages.ts +++ b/src/vs/editor/common/languages.ts @@ -75,10 +75,11 @@ export const enum StandardTokenType { * 1098 7654 3210 9876 5432 1098 7654 3210 * - ------------------------------------------- * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx - * bbbb bbbb bfff ffff ffFF FFTT LLLL LLLL + * bbbb bbbb ffff ffff fFFF FBTT LLLL LLLL * - ------------------------------------------- * - L = LanguageId (8 bits) * - T = StandardTokenType (2 bits) + * - B = Balanced bracket (1 bit) * - F = FontStyle (4 bits) * - f = foreground color (9 bits) * - b = background color (9 bits) @@ -88,14 +89,15 @@ export const enum StandardTokenType { export const enum MetadataConsts { LANGUAGEID_MASK = 0b00000000000000000000000011111111, TOKEN_TYPE_MASK = 0b00000000000000000000001100000000, - FONT_STYLE_MASK = 0b00000000000000000011110000000000, - FOREGROUND_MASK = 0b00000000011111111100000000000000, - BACKGROUND_MASK = 0b11111111100000000000000000000000, + BALANCED_BRACKETS_MASK = 0b00000000000000000000010000000000, + FONT_STYLE_MASK = 0b00000000000000000111100000000000, + FOREGROUND_MASK = 0b00000000111111111000000000000000, + BACKGROUND_MASK = 0b11111111000000000000000000000000, - ITALIC_MASK = 0b00000000000000000000010000000000, - BOLD_MASK = 0b00000000000000000000100000000000, - UNDERLINE_MASK = 0b00000000000000000001000000000000, - STRIKETHROUGH_MASK = 0b00000000000000000010000000000000, + ITALIC_MASK = 0b00000000000000000000100000000000, + BOLD_MASK = 0b00000000000000000001000000000000, + UNDERLINE_MASK = 0b00000000000000000010000000000000, + STRIKETHROUGH_MASK = 0b00000000000000000100000000000000, // Semantic tokens cannot set the language id, so we can // use the first 8 bits for control purposes @@ -108,9 +110,10 @@ export const enum MetadataConsts { LANGUAGEID_OFFSET = 0, TOKEN_TYPE_OFFSET = 8, - FONT_STYLE_OFFSET = 10, - FOREGROUND_OFFSET = 14, - BACKGROUND_OFFSET = 23 + BALANCED_BRACKETS_OFFSET = 10, + FONT_STYLE_OFFSET = 11, + FOREGROUND_OFFSET = 15, + BACKGROUND_OFFSET = 24 } /** @@ -126,6 +129,10 @@ export class TokenMetadata { return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET; } + public static containsBalancedBrackets(metadata: number): boolean { + return (metadata & MetadataConsts.BALANCED_BRACKETS_MASK) !== 0; + } + public static getFontStyle(metadata: number): FontStyle { return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET; } diff --git a/src/vs/editor/test/common/core/lineTokens.test.ts b/src/vs/editor/test/common/core/lineTokens.test.ts index ac74d86cf8f..f26d8930a56 100644 --- a/src/vs/editor/test/common/core/lineTokens.test.ts +++ b/src/vs/editor/test/common/core/lineTokens.test.ts @@ -57,7 +57,7 @@ suite('LineTokens', () => { test('withInserted 1', () => { const lineTokens = createTestLineTokens(); - assert.strictEqual(renderLineTokens(lineTokens), 'Hello (16384)world, (32768)this (49152)is (65536)a (81920)lovely (98304)day(114688)'); + assert.strictEqual(renderLineTokens(lineTokens), 'Hello (32768)world, (65536)this (98304)is (131072)a (163840)lovely (196608)day(229376)'); const lineTokens2 = lineTokens.withInserted([ { offset: 0, text: '1', tokenMetadata: 0, }, @@ -65,12 +65,12 @@ suite('LineTokens', () => { { offset: 9, text: '3', tokenMetadata: 0, }, ]); - assert.strictEqual(renderLineTokens(lineTokens2), '1(0)Hello (16384)2(0)wor(32768)3(0)ld, (32768)this (49152)is (65536)a (81920)lovely (98304)day(114688)'); + assert.strictEqual(renderLineTokens(lineTokens2), '1(0)Hello (32768)2(0)wor(65536)3(0)ld, (65536)this (98304)is (131072)a (163840)lovely (196608)day(229376)'); }); test('withInserted (tokens at the same position)', () => { const lineTokens = createTestLineTokens(); - assert.strictEqual(renderLineTokens(lineTokens), 'Hello (16384)world, (32768)this (49152)is (65536)a (81920)lovely (98304)day(114688)'); + assert.strictEqual(renderLineTokens(lineTokens), 'Hello (32768)world, (65536)this (98304)is (131072)a (163840)lovely (196608)day(229376)'); const lineTokens2 = lineTokens.withInserted([ { offset: 0, text: '1', tokenMetadata: 0, }, @@ -78,19 +78,19 @@ suite('LineTokens', () => { { offset: 0, text: '3', tokenMetadata: 0, }, ]); - assert.strictEqual(renderLineTokens(lineTokens2), '1(0)2(0)3(0)Hello (16384)world, (32768)this (49152)is (65536)a (81920)lovely (98304)day(114688)'); + assert.strictEqual(renderLineTokens(lineTokens2), '1(0)2(0)3(0)Hello (32768)world, (65536)this (98304)is (131072)a (163840)lovely (196608)day(229376)'); }); test('withInserted (tokens at the end)', () => { const lineTokens = createTestLineTokens(); - assert.strictEqual(renderLineTokens(lineTokens), 'Hello (16384)world, (32768)this (49152)is (65536)a (81920)lovely (98304)day(114688)'); + assert.strictEqual(renderLineTokens(lineTokens), 'Hello (32768)world, (65536)this (98304)is (131072)a (163840)lovely (196608)day(229376)'); const lineTokens2 = lineTokens.withInserted([ { offset: 'Hello world, this is a lovely day'.length - 1, text: '1', tokenMetadata: 0, }, { offset: 'Hello world, this is a lovely day'.length, text: '2', tokenMetadata: 0, }, ]); - assert.strictEqual(renderLineTokens(lineTokens2), 'Hello (16384)world, (32768)this (49152)is (65536)a (81920)lovely (98304)da(114688)1(0)y(114688)2(0)'); + assert.strictEqual(renderLineTokens(lineTokens2), 'Hello (32768)world, (65536)this (98304)is (131072)a (163840)lovely (196608)da(229376)1(0)y(229376)2(0)'); }); test('basics', () => { diff --git a/src/vs/editor/test/common/model/tokensStore.test.ts b/src/vs/editor/test/common/model/tokensStore.test.ts index 680503c31c2..540c4afd84b 100644 --- a/src/vs/editor/test/common/model/tokensStore.test.ts +++ b/src/vs/editor/test/common/model/tokensStore.test.ts @@ -176,15 +176,15 @@ suite('TokensStore', () => { const model = createTextModel(' else if ($s = 08) then \'\\b\''); model.tokenization.setSemanticTokens([ SparseMultilineTokens.create(1, new Uint32Array([ - 0, 20, 24, 0b0111100000000010000, - 0, 25, 27, 0b0111100000000010000, - 0, 28, 29, 0b0000100000000010000, - 0, 29, 31, 0b1000000000000010000, - 0, 32, 33, 0b0000100000000010000, - 0, 34, 36, 0b0011000000000010000, - 0, 36, 37, 0b0000100000000010000, - 0, 38, 42, 0b0111100000000010000, - 0, 43, 47, 0b0101100000000010000, + 0, 20, 24, 0b01111000000000010000, + 0, 25, 27, 0b01111000000000010000, + 0, 28, 29, 0b00001000000000010000, + 0, 29, 31, 0b10000000000000010000, + 0, 32, 33, 0b00001000000000010000, + 0, 34, 36, 0b00110000000000010000, + 0, 36, 37, 0b00001000000000010000, + 0, 38, 42, 0b01111000000000010000, + 0, 43, 47, 0b01011000000000010000, ])) ], true); const lineTokens = model.tokenization.getLineTokens(1); @@ -194,22 +194,22 @@ suite('TokensStore', () => { } assert.deepStrictEqual(decodedTokens, [ - 20, 0b1000000000100000000000001, - 24, 0b1000000111100000000000001, - 25, 0b1000000000100000000000001, - 27, 0b1000000111100000000000001, - 28, 0b1000000000100000000000001, - 29, 0b1000000000100000000000001, - 31, 0b1000001000000000000000001, - 32, 0b1000000000100000000000001, - 33, 0b1000000000100000000000001, - 34, 0b1000000000100000000000001, - 36, 0b1000000011000000000000001, - 37, 0b1000000000100000000000001, - 38, 0b1000000000100000000000001, - 42, 0b1000000111100000000000001, - 43, 0b1000000000100000000000001, - 47, 0b1000000101100000000000001 + 20, 0b10000000001000000000000001, + 24, 0b10000001111000000000000001, + 25, 0b10000000001000000000000001, + 27, 0b10000001111000000000000001, + 28, 0b10000000001000000000000001, + 29, 0b10000000001000000000000001, + 31, 0b10000010000000000000000001, + 32, 0b10000000001000000000000001, + 33, 0b10000000001000000000000001, + 34, 0b10000000001000000000000001, + 36, 0b10000000110000000000000001, + 37, 0b10000000001000000000000001, + 38, 0b10000000001000000000000001, + 42, 0b10000001111000000000000001, + 43, 0b10000000001000000000000001, + 47, 0b10000001011000000000000001 ]); model.dispose(); diff --git a/yarn.lock b/yarn.lock index c4babf584c3..37dd13d40d2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11903,10 +11903,10 @@ vscode-regexpp@^3.1.0: resolved "https://registry.yarnpkg.com/vscode-regexpp/-/vscode-regexpp-3.1.0.tgz#42d059b6fffe99bd42939c0d013f632f0cad823f" integrity sha512-pqtN65VC1jRLawfluX4Y80MMG0DHJydWhe5ZwMHewZD6sys4LbU6lHwFAHxeuaVE6Y6+xZOtAw+9hvq7/0ejkg== -vscode-textmate@6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-6.0.0.tgz#a3777197235036814ac9a92451492f2748589210" - integrity sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ== +vscode-textmate@7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-7.0.0.tgz#6c90a591c1291e80c6783bddf43e841ad6f8ccc3" + integrity sha512-27bFXGG5JtLrq/reWnkCxkfNq7gnFko7xORba2Gznokx2LmPpHfu880a44B1jPyoFiGNjprZ883qiYTvR/P+pA== vscode-uri@^3.0.3: version "3.0.3"