Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2025-12-22 11:19:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adopt official semantic tokens api for JS/TS in html (#144223)

Browse Source 
* Adopt official semantic tokens api for JS/TS in html

Fixes #114477

This switches the semantic tokens implementation for js/ts inside html to use the finalized `getEncodedSemanticClassifications` api instead of our custom impl

* Update tests
This commit is contained in:
Matt Bierner
2022-03-02 13:02:43 -08:00
committed by GitHub
parent 98da73d8aa
commit afa70ebba0
3 changed files with 77 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/html-language-features/server/src/modes/javascriptMode.ts
View File

@@ -360,7 +360,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
async getSemanticTokens(document: TextDocument): Promise<SemanticTokenData[]> {
const jsDocument = jsDocuments.get(document);
const jsLanguageService = await host.getLanguageService(jsDocument);
return getSemanticTokens(jsLanguageService, jsDocument, jsDocument.uri);
return [...getSemanticTokens(jsLanguageService, jsDocument, jsDocument.uri)];
},
getSemanticTokenLegend(): { types: string[]; modifiers: string[] } {
return getSemanticTokenLegend();

159
extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts
View File

@@ -16,94 +16,74 @@ export function getSemanticTokenLegend() {
return { types: tokenTypes, modifiers: tokenModifiers };
}
export function getSemanticTokens(jsLanguageService: ts.LanguageService, currentTextDocument: TextDocument, fileName: string): SemanticTokenData[] {
//https://ts-ast-viewer.com/#code/AQ0g2CmAuwGbALzAJwG4BQZQGNwEMBnQ4AQQEYBmYAb2C22zgEtJwATJVTRxgcwD27AQAp8AGmAAjAJS0A9POB8+7NQ168oscAJz5wANXwAnLug2bsJmAFcTAO2XAA1MHyvgu-UdOeWbOw8ViAAvpagocBAA
export function* getSemanticTokens(jsLanguageService: ts.LanguageService, document: TextDocument, fileName: string): Iterable<SemanticTokenData> {
const { spans } = jsLanguageService.getEncodedSemanticClassifications(fileName, { start: 0, length: document.getText().length }, '2020' as ts.SemanticClassificationFormat);
let resultTokens: SemanticTokenData[] = [];
const collector = (node: ts.Node, typeIdx: number, modifierSet: number) => {
resultTokens.push({ start: currentTextDocument.positionAt(node.getStart()), length: node.getWidth(), typeIdx, modifierSet });
};
collectTokens(jsLanguageService, fileName, { start: 0, length: currentTextDocument.getText().length }, collector);
for (let i = 0; i < spans.length;) {
const offset = spans[i++];
const length = spans[i++];
const tsClassification = spans[i++];
return resultTokens;
}
function collectTokens(jsLanguageService: ts.LanguageService, fileName: string, span: ts.TextSpan, collector: (node: ts.Node, tokenType: number, tokenModifier: number) => void) {
const program = jsLanguageService.getProgram();
if (program) {
const typeChecker = program.getTypeChecker();
function visit(node: ts.Node) {
if (!node || !ts.textSpanIntersectsWith(span, node.pos, node.getFullWidth())) {
return;
}
if (ts.isIdentifier(node)) {
let symbol = typeChecker.getSymbolAtLocation(node);
if (symbol) {
if (symbol.flags & ts.SymbolFlags.Alias) {
symbol = typeChecker.getAliasedSymbol(symbol);
}
let typeIdx = classifySymbol(symbol);
if (typeIdx !== undefined) {
let modifierSet = 0;
if (node.parent) {
const parentTypeIdx = tokenFromDeclarationMapping[node.parent.kind];
if (parentTypeIdx === typeIdx && (<ts.NamedDeclaration>node.parent).name === node) {
modifierSet = 1 << TokenModifier.declaration;
}
}
const decl = symbol.valueDeclaration;
const modifiers = decl ? ts.getCombinedModifierFlags(decl) : 0;
const nodeFlags = decl ? ts.getCombinedNodeFlags(decl) : 0;
if (modifiers & ts.ModifierFlags.Static) {
modifierSet |= 1 << TokenModifier.static;
}
if (modifiers & ts.ModifierFlags.Async) {
modifierSet |= 1 << TokenModifier.async;
}
if ((modifiers & ts.ModifierFlags.Readonly) || (nodeFlags & ts.NodeFlags.Const) || (symbol.getFlags() & ts.SymbolFlags.EnumMember)) {
modifierSet |= 1 << TokenModifier.readonly;
}
collector(node, typeIdx, modifierSet);
}
}
}
ts.forEachChild(node, visit);
}
const sourceFile = program.getSourceFile(fileName);
if (sourceFile) {
visit(sourceFile);
const tokenType = getTokenTypeFromClassification(tsClassification);
if (tokenType === undefined) {
continue;
}
const tokenModifiers = getTokenModifierFromClassification(tsClassification);
const startPos = document.positionAt(offset);
yield {
start: startPos,
length: length,
typeIdx: tokenType,
modifierSet: tokenModifiers
};
}
}
function classifySymbol(symbol: ts.Symbol) {
const flags = symbol.getFlags();
if (flags & ts.SymbolFlags.Class) {
return TokenType.class;
} else if (flags & ts.SymbolFlags.Enum) {
return TokenType.enum;
} else if (flags & ts.SymbolFlags.TypeAlias) {
return TokenType.type;
} else if (flags & ts.SymbolFlags.Type) {
if (flags & ts.SymbolFlags.Interface) {
return TokenType.interface;
} if (flags & ts.SymbolFlags.TypeParameter) {
return TokenType.typeParameter;
}
// typescript encodes type and modifiers in the classification:
// TSClassification = (TokenType + 1) << 8 + TokenModifier
const enum TokenType {
class = 0,
enum = 1,
interface = 2,
namespace = 3,
typeParameter = 4,
type = 5,
parameter = 6,
variable = 7,
enumMember = 8,
property = 9,
function = 10,
method = 11,
_ = 12
}
const enum TokenModifier {
declaration = 0,
static = 1,
async = 2,
readonly = 3,
defaultLibrary = 4,
local = 5,
_ = 6
}
const enum TokenEncodingConsts {
typeOffset = 8,
modifierMask = 255
}
function getTokenTypeFromClassification(tsClassification: number): number | undefined {
if (tsClassification > TokenEncodingConsts.modifierMask) {
return (tsClassification >> TokenEncodingConsts.typeOffset) - 1;
}
const decl = symbol.valueDeclaration || symbol.declarations && symbol.declarations[0];
return decl && tokenFromDeclarationMapping[decl.kind];
return undefined;
}
export const enum TokenType {
class, enum, interface, namespace, typeParameter, type, parameter, variable, property, function, method, _
}
export const enum TokenModifier {
declaration, static, async, readonly, _
function getTokenModifierFromClassification(tsClassification: number) {
return tsClassification & TokenEncodingConsts.modifierMask;
}
const tokenTypes: string[] = [];
@@ -115,6 +95,7 @@ tokenTypes[TokenType.typeParameter] = 'typeParameter';
tokenTypes[TokenType.type] = 'type';
tokenTypes[TokenType.parameter] = 'parameter';
tokenTypes[TokenType.variable] = 'variable';
tokenTypes[TokenType.enumMember] = 'enumMember';
tokenTypes[TokenType.property] = 'property';
tokenTypes[TokenType.function] = 'function';
tokenTypes[TokenType.method] = 'method';
@@ -124,21 +105,5 @@ tokenModifiers[TokenModifier.async] = 'async';
tokenModifiers[TokenModifier.declaration] = 'declaration';
tokenModifiers[TokenModifier.readonly] = 'readonly';
tokenModifiers[TokenModifier.static] = 'static';
const tokenFromDeclarationMapping: { [name: string]: TokenType } = {
[ts.SyntaxKind.VariableDeclaration]: TokenType.variable,
[ts.SyntaxKind.Parameter]: TokenType.parameter,
[ts.SyntaxKind.PropertyDeclaration]: TokenType.property,
[ts.SyntaxKind.ModuleDeclaration]: TokenType.namespace,
[ts.SyntaxKind.EnumDeclaration]: TokenType.enum,
[ts.SyntaxKind.EnumMember]: TokenType.property,
[ts.SyntaxKind.ClassDeclaration]: TokenType.class,
[ts.SyntaxKind.MethodDeclaration]: TokenType.method,
[ts.SyntaxKind.FunctionDeclaration]: TokenType.function,
[ts.SyntaxKind.MethodSignature]: TokenType.method,
[ts.SyntaxKind.GetAccessor]: TokenType.property,
[ts.SyntaxKind.PropertySignature]: TokenType.property,
[ts.SyntaxKind.InterfaceDeclaration]: TokenType.interface,
[ts.SyntaxKind.TypeAliasDeclaration]: TokenType.type,
[ts.SyntaxKind.TypeParameter]: TokenType.typeParameter
};
tokenModifiers[TokenModifier.local] = 'local';
tokenModifiers[TokenModifier.defaultLibrary] = 'defaultLibrary';