From cedad6e8fcbab7ceeb82cd74a56ebe82d50632b8 Mon Sep 17 00:00:00 2001 From: Alex Ross Date: Wed, 25 Sep 2024 16:18:15 +0200 Subject: [PATCH] Support targetting multiple scopes in a theme (#229717) and fix font style Fixes #229557 Fixes #229553 --- .../test-issue5431_ts.json | 4 ++ .../test-strings_ts.json | 10 +++ .../languages/highlights/typescript.scm | 2 +- .../browser/themes.test.contribution.ts | 2 +- .../services/themes/common/colorThemeData.ts | 61 ++++++++++++------- .../browser/treeSitterTokenizationFeature.ts | 42 ++++++------- 6 files changed, 76 insertions(+), 45 deletions(-) diff --git a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5431_ts.json b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5431_ts.json index fca4ee7894b..2dd0da1b0aa 100644 --- a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5431_ts.json +++ b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-issue5431_ts.json @@ -224,6 +224,8 @@ "r": { "dark_plus": "variable: #9CDCFE", "light_plus": "variable: #001080", + "dark_vs": "string: #CE9178", + "light_vs": "string: #A31515", "hc_black": "variable: #9CDCFE", "dark_modern": "variable: #9CDCFE", "hc_light": "variable: #001080", @@ -278,6 +280,8 @@ "r": { "dark_plus": "variable: #9CDCFE", "light_plus": "variable: #001080", + "dark_vs": "string: #CE9178", + "light_vs": "string: #A31515", "hc_black": "variable: #9CDCFE", "dark_modern": "variable: #9CDCFE", "hc_light": "variable: #001080", diff --git a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-strings_ts.json b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-strings_ts.json index bd845f74e07..7f9d901343e 100644 --- a/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-strings_ts.json +++ b/extensions/vscode-colorize-tests/test/colorize-tree-sitter-results/test-strings_ts.json @@ -87,6 +87,8 @@ "r": { "dark_plus": "variable: #9CDCFE", "light_plus": "variable: #001080", + "dark_vs": "string: #CE9178", + "light_vs": "string: #A31515", "hc_black": "variable: #9CDCFE", "dark_modern": "variable: #9CDCFE", "hc_light": "variable: #001080", @@ -312,6 +314,8 @@ "r": { "dark_plus": "variable: #9CDCFE", "light_plus": "variable: #001080", + "dark_vs": "string: #CE9178", + "light_vs": "string: #A31515", "hc_black": "variable: #9CDCFE", "dark_modern": "variable: #9CDCFE", "hc_light": "variable: #001080", @@ -338,6 +342,8 @@ "r": { "dark_plus": "variable: #9CDCFE", "light_plus": "variable: #001080", + "dark_vs": "string: #CE9178", + "light_vs": "string: #A31515", "hc_black": "variable: #9CDCFE", "dark_modern": "variable: #9CDCFE", "hc_light": "variable: #001080", @@ -392,6 +398,8 @@ "r": { "dark_plus": "variable: #9CDCFE", "light_plus": "variable: #001080", + "dark_vs": "string: #CE9178", + "light_vs": "string: #A31515", "hc_black": "variable: #9CDCFE", "dark_modern": "variable: #9CDCFE", "hc_light": "variable: #001080", @@ -418,6 +426,8 @@ "r": { "dark_plus": "variable: #9CDCFE", "light_plus": "variable: #001080", + "dark_vs": "string: #CE9178", + "light_vs": "string: #A31515", "hc_black": "variable: #9CDCFE", "dark_modern": "variable: #9CDCFE", "hc_light": "variable: #001080", diff --git a/src/vs/editor/common/languages/highlights/typescript.scm b/src/vs/editor/common/languages/highlights/typescript.scm index 7f9eee1b6d5..85d72b42d93 100644 --- a/src/vs/editor/common/languages/highlights/typescript.scm +++ b/src/vs/editor/common/languages/highlights/typescript.scm @@ -66,7 +66,7 @@ name: (property_identifier) @storage.type (#eq? @storage.type "constructor")) (method_signature - name: (property_identifier) @entity.name.function) + name: (property_identifier) @meta.definition.method @entity.name.function) (pair key: (property_identifier) @entity.name.function diff --git a/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts b/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts index 15d6fa9e618..57e6c944465 100644 --- a/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts +++ b/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts @@ -137,7 +137,7 @@ class Snapper { for (let i = 0, len = tokens.length; i < len; i++) { const token = tokens[i]; const scopes = token.t.split(' '); - const metadata = findMetadata(colorThemeData, scopes[scopes.length - 1], this.languageService.languageIdCodec.encodeLanguageId(languageId)); + const metadata = findMetadata(colorThemeData, scopes, this.languageService.languageIdCodec.encodeLanguageId(languageId)); const color = TokenMetadata.getForeground(metadata); result[i] = { diff --git a/src/vs/workbench/services/themes/common/colorThemeData.ts b/src/vs/workbench/services/themes/common/colorThemeData.ts index be9f39e969d..4497c76c034 100644 --- a/src/vs/workbench/services/themes/common/colorThemeData.ts +++ b/src/vs/workbench/services/themes/common/colorThemeData.ts @@ -26,6 +26,7 @@ import { ThemeConfiguration } from './themeConfiguration.js'; import { ColorScheme } from '../../../../platform/theme/common/theme.js'; import { FontStyle, MetadataConsts } from '../../../../editor/common/encodedTokenAttributes.js'; import { toStandardTokenType } from '../../../../editor/common/languages/supports/tokenization.js'; +import { findMatchingThemeRule } from '../../textMate/common/TMHelper.js'; const colorRegistry = Registry.as(ColorRegistryExtensions.ColorContribution); @@ -900,36 +901,52 @@ function isSemanticTokenColorizationSetting(style: any): style is ISemanticToken || types.isBoolean(style.underline) || types.isBoolean(style.strikethrough) || types.isBoolean(style.bold)); } -export function findMetadata(colorThemeData: ColorThemeData, captureName: string, languageId: number): number { +export function findMetadata(colorThemeData: ColorThemeData, captureNames: string[], languageId: number): number { let metadata = 0; - const standardToken = toStandardTokenType(captureName); - metadata |= (standardToken << MetadataConsts.TOKEN_TYPE_OFFSET); metadata |= (languageId << MetadataConsts.LANGUAGEID_OFFSET); - const tokenStyle: TokenStyle | undefined = colorThemeData.resolveScopes([[captureName]]); - if (!tokenStyle) { + const themeRule = findMatchingThemeRule(colorThemeData, captureNames); + let tokenStyle: TokenStyle | undefined; + if (!themeRule) { + tokenStyle = colorThemeData.resolveScopes(captureNames.map(name => [name]).reverse()); + } + if (!themeRule && !tokenStyle) { return metadata; } - if (typeof tokenStyle.italic !== 'undefined') { - const italicBit = (tokenStyle.italic ? FontStyle.Italic : 0); - metadata |= italicBit | MetadataConsts.ITALIC_MASK; + const standardToken = toStandardTokenType(captureNames[captureNames.length - 1]); + metadata |= (standardToken << MetadataConsts.TOKEN_TYPE_OFFSET); + + if (themeRule?.settings.fontStyle === 'italic') { + metadata |= FontStyle.Italic | MetadataConsts.ITALIC_MASK; + } else if (themeRule?.settings.fontStyle !== 'bold') { + metadata |= FontStyle.Bold | MetadataConsts.BOLD_MASK; + } else if (typeof tokenStyle?.underline !== 'undefined') { + metadata |= FontStyle.Underline | MetadataConsts.UNDERLINE_MASK; + } else if (typeof tokenStyle?.strikethrough !== 'undefined') { + metadata |= FontStyle.Strikethrough | MetadataConsts.STRIKETHROUGH_MASK; + } else { + if (typeof tokenStyle?.italic !== 'undefined') { + const italicbit = (tokenStyle?.italic ? FontStyle.Italic : 0); + metadata |= italicbit | MetadataConsts.ITALIC_MASK; + } + if (typeof tokenStyle?.bold !== 'undefined') { + const boldBit = (tokenStyle?.bold ? FontStyle.Bold : 0); + metadata |= boldBit | MetadataConsts.BOLD_MASK; + } + if (typeof tokenStyle?.underline !== 'undefined') { + const underlineBit = (tokenStyle?.underline ? FontStyle.Underline : 0); + metadata |= underlineBit | MetadataConsts.UNDERLINE_MASK; + } + if (typeof tokenStyle?.strikethrough !== 'undefined') { + const strikethroughBit = (tokenStyle?.strikethrough ? FontStyle.Strikethrough : 0); + metadata |= strikethroughBit | MetadataConsts.STRIKETHROUGH_MASK; + } } - if (typeof tokenStyle.bold !== 'undefined') { - const boldBit = (tokenStyle.bold ? FontStyle.Bold : 0); - metadata |= boldBit | MetadataConsts.BOLD_MASK; - } - if (typeof tokenStyle.underline !== 'undefined') { - const underlineBit = (tokenStyle.underline ? FontStyle.Underline : 0); - metadata |= underlineBit | MetadataConsts.UNDERLINE_MASK; - } - if (typeof tokenStyle.strikethrough !== 'undefined') { - const strikethroughBit = (tokenStyle.strikethrough ? FontStyle.Strikethrough : 0); - metadata |= strikethroughBit | MetadataConsts.STRIKETHROUGH_MASK; - } - if (tokenStyle.foreground) { - const tokenStyleForeground = colorThemeData.getTokenColorIndex().get(tokenStyle?.foreground); + const foreground = themeRule ? themeRule.settings.foreground : tokenStyle?.foreground; + if (foreground) { + const tokenStyleForeground = colorThemeData.getTokenColorIndex().get(foreground); const foregroundBits = tokenStyleForeground << MetadataConsts.FOREGROUND_OFFSET; metadata |= foregroundBits; } diff --git a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts index ea4f77821e0..33cad30d477 100644 --- a/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts +++ b/src/vs/workbench/services/treeSitter/browser/treeSitterTokenizationFeature.ts @@ -174,21 +174,19 @@ class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTok return undefined; } - let tokens: Uint32Array = new Uint32Array(captures.length * 2); + const endOffsetsAndScopes: { endOffset: number; scopes: string[] }[] = Array(captures.length); + endOffsetsAndScopes.fill({ endOffset: 0, scopes: [] }); let tokenIndex = 0; const lineStartOffset = textModel.getOffsetAt({ lineNumber: lineNumber, column: 1 }); const increaseSizeOfTokensByOneToken = () => { - const newTokens = new Uint32Array(tokens.length + 2); - newTokens.set(tokens); - tokens = newTokens; + endOffsetsAndScopes.push({ endOffset: 0, scopes: [] }); }; const encodedLanguageId = this._languageIdCodec.encodeLanguageId(this._languageId); for (let captureIndex = 0; captureIndex < captures.length; captureIndex++) { const capture = captures[captureIndex]; - const metadata = findMetadata(this._colorThemeData, capture.name, encodedLanguageId); const tokenEndIndex = capture.node.endIndex < lineStartOffset + lineLength ? capture.node.endIndex : lineStartOffset + lineLength; const tokenStartIndex = capture.node.startIndex < lineStartOffset ? lineStartOffset : capture.node.startIndex; @@ -198,43 +196,41 @@ class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTok let previousTokenEnd: number; const currentTokenLength = tokenEndIndex - tokenStartIndex; if (captureIndex > 0) { - previousTokenEnd = tokens[(tokenIndex - 1) * 2]; + previousTokenEnd = endOffsetsAndScopes[(tokenIndex - 1)].endOffset; } else { previousTokenEnd = tokenStartIndex - lineStartOffset - 1; } const intermediateTokenOffset = lineRelativeOffset - currentTokenLength; if ((previousTokenEnd >= 0) && (previousTokenEnd < intermediateTokenOffset)) { // Add en empty token to cover the space where there were no captures - tokens[tokenIndex * 2] = intermediateTokenOffset; - tokens[tokenIndex * 2 + 1] = findMetadata(this._colorThemeData, '', encodedLanguageId); + endOffsetsAndScopes[tokenIndex] = { endOffset: intermediateTokenOffset, scopes: [] }; tokenIndex++; increaseSizeOfTokensByOneToken(); } const addCurrentTokenToArray = () => { - tokens[tokenIndex * 2] = lineRelativeOffset; - tokens[tokenIndex * 2 + 1] = metadata; + endOffsetsAndScopes[tokenIndex] = { endOffset: lineRelativeOffset, scopes: [capture.name] }; tokenIndex++; }; if (previousTokenEnd >= lineRelativeOffset) { - const previousTokenStartOffset = tokens[(tokenIndex - 2) * 2]; - const originalPreviousTokenEndOffset = tokens[(tokenIndex - 1) * 2]; + const previousTokenStartOffset = endOffsetsAndScopes[tokenIndex - 2].endOffset; + const originalPreviousTokenEndOffset = endOffsetsAndScopes[tokenIndex - 1].endOffset; // Check that the current token doesn't just replace the last token if ((previousTokenStartOffset + currentTokenLength) === originalPreviousTokenEndOffset) { // Current token and previous token span the exact same characters - tokens[(tokenIndex - 1) * 2 + 1] = metadata; + endOffsetsAndScopes[tokenIndex - 1].scopes.push(capture.name); } else { // The current token is within the previous token. Adjust the end of the previous token. - tokens[(tokenIndex - 1) * 2] = intermediateTokenOffset; + endOffsetsAndScopes[tokenIndex - 1].endOffset = intermediateTokenOffset; addCurrentTokenToArray(); // Add the rest of the previous token after the current token increaseSizeOfTokensByOneToken(); - tokens[tokenIndex * 2] = originalPreviousTokenEndOffset; - tokens[tokenIndex * 2 + 1] = tokens[(tokenIndex - 2) * 2 + 1]; + endOffsetsAndScopes[tokenIndex].endOffset = originalPreviousTokenEndOffset; + endOffsetsAndScopes[tokenIndex].scopes = endOffsetsAndScopes[tokenIndex - 2].scopes; tokenIndex++; } } else { @@ -244,12 +240,16 @@ class TreeSitterTokenizationSupport extends Disposable implements ITreeSitterTok } if (captures[captures.length - 1].node.endPosition.column + 1 < lineLength) { - const newTokens = new Uint32Array(tokens.length + 2); - newTokens.set(tokens); - tokens = newTokens; - tokens[tokenIndex * 2] = lineLength; - tokens[tokenIndex * 2 + 1] = 0; + increaseSizeOfTokensByOneToken(); + endOffsetsAndScopes[tokenIndex].endOffset = lineLength; } + + const tokens: Uint32Array = new Uint32Array(endOffsetsAndScopes.length * 2); + for (let i = 0; i < endOffsetsAndScopes.length; i++) { + tokens[i * 2] = endOffsetsAndScopes[i].endOffset; + tokens[i * 2 + 1] = findMetadata(this._colorThemeData, endOffsetsAndScopes[i].scopes, encodedLanguageId); + } + return tokens; }