From afa70ebba04f998a97cfae40f955e6573bb2ba1d Mon Sep 17 00:00:00 2001 From: Matt Bierner Date: Wed, 2 Mar 2022 13:02:43 -0800 Subject: [PATCH] Adopt official semantic tokens api for JS/TS in html (#144223) * Adopt official semantic tokens api for JS/TS in html Fixes #114477 This switches the semantic tokens implementation for js/ts inside html to use the finalized `getEncodedSemanticClassifications` api instead of our custom impl * Update tests --- .../server/src/modes/javascriptMode.ts | 2 +- .../src/modes/javascriptSemanticTokens.ts | 159 +++++++----------- .../server/src/test/semanticTokens.test.ts | 28 +-- 3 files changed, 77 insertions(+), 112 deletions(-) diff --git a/extensions/html-language-features/server/src/modes/javascriptMode.ts b/extensions/html-language-features/server/src/modes/javascriptMode.ts index 301af0e58c5..36a41173b30 100644 --- a/extensions/html-language-features/server/src/modes/javascriptMode.ts +++ b/extensions/html-language-features/server/src/modes/javascriptMode.ts @@ -360,7 +360,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache { const jsDocument = jsDocuments.get(document); const jsLanguageService = await host.getLanguageService(jsDocument); - return getSemanticTokens(jsLanguageService, jsDocument, jsDocument.uri); + return [...getSemanticTokens(jsLanguageService, jsDocument, jsDocument.uri)]; }, getSemanticTokenLegend(): { types: string[]; modifiers: string[] } { return getSemanticTokenLegend(); diff --git a/extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts b/extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts index f560559727a..cbcf1b45082 100644 --- a/extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts +++ b/extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts @@ -16,94 +16,74 @@ export function getSemanticTokenLegend() { return { types: tokenTypes, modifiers: tokenModifiers }; } -export function getSemanticTokens(jsLanguageService: ts.LanguageService, currentTextDocument: TextDocument, fileName: string): SemanticTokenData[] { - //https://ts-ast-viewer.com/#code/AQ0g2CmAuwGbALzAJwG4BQZQGNwEMBnQ4AQQEYBmYAb2C22zgEtJwATJVTRxgcwD27AQAp8AGmAAjAJS0A9POB8+7NQ168oscAJz5wANXwAnLug2bsJmAFcTAO2XAA1MHyvgu-UdOeWbOw8ViAAvpagocBAA +export function* getSemanticTokens(jsLanguageService: ts.LanguageService, document: TextDocument, fileName: string): Iterable { + const { spans } = jsLanguageService.getEncodedSemanticClassifications(fileName, { start: 0, length: document.getText().length }, '2020' as ts.SemanticClassificationFormat); - let resultTokens: SemanticTokenData[] = []; - const collector = (node: ts.Node, typeIdx: number, modifierSet: number) => { - resultTokens.push({ start: currentTextDocument.positionAt(node.getStart()), length: node.getWidth(), typeIdx, modifierSet }); - }; - collectTokens(jsLanguageService, fileName, { start: 0, length: currentTextDocument.getText().length }, collector); + for (let i = 0; i < spans.length;) { + const offset = spans[i++]; + const length = spans[i++]; + const tsClassification = spans[i++]; - return resultTokens; -} - -function collectTokens(jsLanguageService: ts.LanguageService, fileName: string, span: ts.TextSpan, collector: (node: ts.Node, tokenType: number, tokenModifier: number) => void) { - - const program = jsLanguageService.getProgram(); - if (program) { - const typeChecker = program.getTypeChecker(); - - function visit(node: ts.Node) { - if (!node || !ts.textSpanIntersectsWith(span, node.pos, node.getFullWidth())) { - return; - } - if (ts.isIdentifier(node)) { - let symbol = typeChecker.getSymbolAtLocation(node); - if (symbol) { - if (symbol.flags & ts.SymbolFlags.Alias) { - symbol = typeChecker.getAliasedSymbol(symbol); - } - let typeIdx = classifySymbol(symbol); - if (typeIdx !== undefined) { - let modifierSet = 0; - if (node.parent) { - const parentTypeIdx = tokenFromDeclarationMapping[node.parent.kind]; - if (parentTypeIdx === typeIdx && (node.parent).name === node) { - modifierSet = 1 << TokenModifier.declaration; - } - } - const decl = symbol.valueDeclaration; - const modifiers = decl ? ts.getCombinedModifierFlags(decl) : 0; - const nodeFlags = decl ? ts.getCombinedNodeFlags(decl) : 0; - if (modifiers & ts.ModifierFlags.Static) { - modifierSet |= 1 << TokenModifier.static; - } - if (modifiers & ts.ModifierFlags.Async) { - modifierSet |= 1 << TokenModifier.async; - } - if ((modifiers & ts.ModifierFlags.Readonly) || (nodeFlags & ts.NodeFlags.Const) || (symbol.getFlags() & ts.SymbolFlags.EnumMember)) { - modifierSet |= 1 << TokenModifier.readonly; - } - collector(node, typeIdx, modifierSet); - } - } - } - - ts.forEachChild(node, visit); - } - const sourceFile = program.getSourceFile(fileName); - if (sourceFile) { - visit(sourceFile); + const tokenType = getTokenTypeFromClassification(tsClassification); + if (tokenType === undefined) { + continue; } + + const tokenModifiers = getTokenModifierFromClassification(tsClassification); + const startPos = document.positionAt(offset); + yield { + start: startPos, + length: length, + typeIdx: tokenType, + modifierSet: tokenModifiers + }; } } -function classifySymbol(symbol: ts.Symbol) { - const flags = symbol.getFlags(); - if (flags & ts.SymbolFlags.Class) { - return TokenType.class; - } else if (flags & ts.SymbolFlags.Enum) { - return TokenType.enum; - } else if (flags & ts.SymbolFlags.TypeAlias) { - return TokenType.type; - } else if (flags & ts.SymbolFlags.Type) { - if (flags & ts.SymbolFlags.Interface) { - return TokenType.interface; - } if (flags & ts.SymbolFlags.TypeParameter) { - return TokenType.typeParameter; - } + +// typescript encodes type and modifiers in the classification: +// TSClassification = (TokenType + 1) << 8 + TokenModifier + +const enum TokenType { + class = 0, + enum = 1, + interface = 2, + namespace = 3, + typeParameter = 4, + type = 5, + parameter = 6, + variable = 7, + enumMember = 8, + property = 9, + function = 10, + method = 11, + _ = 12 +} + +const enum TokenModifier { + declaration = 0, + static = 1, + async = 2, + readonly = 3, + defaultLibrary = 4, + local = 5, + _ = 6 +} + +const enum TokenEncodingConsts { + typeOffset = 8, + modifierMask = 255 +} + +function getTokenTypeFromClassification(tsClassification: number): number | undefined { + if (tsClassification > TokenEncodingConsts.modifierMask) { + return (tsClassification >> TokenEncodingConsts.typeOffset) - 1; } - const decl = symbol.valueDeclaration || symbol.declarations && symbol.declarations[0]; - return decl && tokenFromDeclarationMapping[decl.kind]; + return undefined; } -export const enum TokenType { - class, enum, interface, namespace, typeParameter, type, parameter, variable, property, function, method, _ -} - -export const enum TokenModifier { - declaration, static, async, readonly, _ +function getTokenModifierFromClassification(tsClassification: number) { + return tsClassification & TokenEncodingConsts.modifierMask; } const tokenTypes: string[] = []; @@ -115,6 +95,7 @@ tokenTypes[TokenType.typeParameter] = 'typeParameter'; tokenTypes[TokenType.type] = 'type'; tokenTypes[TokenType.parameter] = 'parameter'; tokenTypes[TokenType.variable] = 'variable'; +tokenTypes[TokenType.enumMember] = 'enumMember'; tokenTypes[TokenType.property] = 'property'; tokenTypes[TokenType.function] = 'function'; tokenTypes[TokenType.method] = 'method'; @@ -124,21 +105,5 @@ tokenModifiers[TokenModifier.async] = 'async'; tokenModifiers[TokenModifier.declaration] = 'declaration'; tokenModifiers[TokenModifier.readonly] = 'readonly'; tokenModifiers[TokenModifier.static] = 'static'; - -const tokenFromDeclarationMapping: { [name: string]: TokenType } = { - [ts.SyntaxKind.VariableDeclaration]: TokenType.variable, - [ts.SyntaxKind.Parameter]: TokenType.parameter, - [ts.SyntaxKind.PropertyDeclaration]: TokenType.property, - [ts.SyntaxKind.ModuleDeclaration]: TokenType.namespace, - [ts.SyntaxKind.EnumDeclaration]: TokenType.enum, - [ts.SyntaxKind.EnumMember]: TokenType.property, - [ts.SyntaxKind.ClassDeclaration]: TokenType.class, - [ts.SyntaxKind.MethodDeclaration]: TokenType.method, - [ts.SyntaxKind.FunctionDeclaration]: TokenType.function, - [ts.SyntaxKind.MethodSignature]: TokenType.method, - [ts.SyntaxKind.GetAccessor]: TokenType.property, - [ts.SyntaxKind.PropertySignature]: TokenType.property, - [ts.SyntaxKind.InterfaceDeclaration]: TokenType.interface, - [ts.SyntaxKind.TypeAliasDeclaration]: TokenType.type, - [ts.SyntaxKind.TypeParameter]: TokenType.typeParameter -}; +tokenModifiers[TokenModifier.local] = 'local'; +tokenModifiers[TokenModifier.defaultLibrary] = 'defaultLibrary'; diff --git a/extensions/html-language-features/server/src/test/semanticTokens.test.ts b/extensions/html-language-features/server/src/test/semanticTokens.test.ts index 2d7677d79a5..2db27dd9f17 100644 --- a/extensions/html-language-features/server/src/test/semanticTokens.test.ts +++ b/extensions/html-language-features/server/src/test/semanticTokens.test.ts @@ -66,8 +66,8 @@ suite('HTML Semantic Tokens', () => { ]; await assertTokens(input, [ t(3, 6, 1, 'variable.declaration'), t(3, 13, 2, 'variable.declaration'), t(3, 19, 1, 'variable'), - t(5, 15, 1, 'variable.declaration.readonly'), t(5, 20, 2, 'variable'), t(5, 26, 1, 'variable'), t(5, 30, 1, 'variable.readonly'), - t(6, 11, 1, 'variable.declaration'), + t(5, 15, 1, 'variable.declaration.readonly.local'), t(5, 20, 2, 'variable'), t(5, 26, 1, 'variable'), t(5, 30, 1, 'variable.readonly.local'), + t(6, 11, 1, 'variable.declaration.local'), t(7, 10, 2, 'variable') ]); }); @@ -87,8 +87,8 @@ suite('HTML Semantic Tokens', () => { ]; await assertTokens(input, [ t(3, 11, 3, 'function.declaration'), t(3, 15, 2, 'parameter.declaration'), - t(4, 11, 3, 'function'), t(4, 15, 4, 'interface'), t(4, 20, 3, 'method'), t(4, 24, 2, 'parameter'), - t(6, 6, 6, 'variable'), t(6, 13, 8, 'property'), t(6, 24, 5, 'method'), t(6, 35, 7, 'method'), t(6, 43, 1, 'parameter.declaration'), t(6, 48, 3, 'function'), t(6, 52, 1, 'parameter') + t(4, 11, 3, 'function'), t(4, 15, 4, 'variable.defaultLibrary'), t(4, 20, 3, 'method.defaultLibrary'), t(4, 24, 2, 'parameter'), + t(6, 6, 6, 'variable.defaultLibrary'), t(6, 13, 8, 'property.defaultLibrary'), t(6, 24, 5, 'method.defaultLibrary'), t(6, 35, 7, 'method.defaultLibrary'), t(6, 43, 1, 'parameter.declaration'), t(6, 48, 3, 'function'), t(6, 52, 1, 'parameter') ]); }); @@ -135,8 +135,8 @@ suite('HTML Semantic Tokens', () => { ]; await assertTokens(input, [ t(3, 12, 8, 'interface.declaration'), t(3, 23, 1, 'property.declaration'), t(3, 34, 1, 'property.declaration'), - t(4, 8, 1, 'variable.declaration.readonly'), t(4, 30, 8, 'interface'), - t(5, 8, 3, 'variable.declaration.readonly'), t(5, 15, 1, 'parameter.declaration'), t(5, 18, 8, 'interface'), t(5, 31, 1, 'parameter'), t(5, 33, 1, 'property'), t(5, 37, 1, 'parameter'), t(5, 39, 1, 'property') + t(4, 8, 1, 'variable.declaration.readonly'), t(4, 14, 1, 'property.declaration'), t(4, 20, 1, 'property.declaration'), t(4, 30, 8, 'interface'), + t(5, 8, 3, 'function.declaration.readonly'), t(5, 15, 1, 'parameter.declaration'), t(5, 18, 8, 'interface'), t(5, 31, 1, 'parameter'), t(5, 33, 1, 'property'), t(5, 37, 1, 'parameter'), t(5, 39, 1, 'property') ]); }); @@ -155,9 +155,9 @@ suite('HTML Semantic Tokens', () => { ]; await assertTokens(input, [ t(3, 8, 1, 'variable.declaration.readonly'), - t(4, 8, 1, 'class.declaration'), t(4, 28, 1, 'property.declaration.static.readonly'), t(4, 42, 3, 'property.declaration.static'), t(4, 47, 3, 'interface'), - t(5, 13, 1, 'enum.declaration'), t(5, 17, 1, 'property.declaration.readonly'), t(5, 24, 1, 'property.declaration.readonly'), t(5, 28, 1, 'property.readonly'), - t(6, 2, 7, 'variable'), t(6, 10, 3, 'method'), t(6, 14, 1, 'variable.readonly'), t(6, 18, 1, 'class'), t(6, 20, 1, 'property.static.readonly'), t(6, 24, 1, 'class'), t(6, 26, 3, 'property.static'), t(6, 30, 6, 'property.readonly'), + t(4, 8, 1, 'class.declaration'), t(4, 28, 1, 'property.declaration.static.readonly'), t(4, 42, 3, 'property.declaration.static'), t(4, 47, 3, 'interface.defaultLibrary'), + t(5, 13, 1, 'enum.declaration'), t(5, 17, 1, 'enumMember.declaration.readonly'), t(5, 24, 1, 'enumMember.declaration.readonly'), t(5, 28, 1, 'enumMember.readonly'), + t(6, 2, 7, 'variable.defaultLibrary'), t(6, 10, 3, 'method.defaultLibrary'), t(6, 14, 1, 'variable.readonly'), t(6, 18, 1, 'class'), t(6, 20, 1, 'property.static.readonly'), t(6, 24, 1, 'class'), t(6, 26, 3, 'property.static'), t(6, 30, 6, 'property.readonly.defaultLibrary'), ]); }); @@ -176,9 +176,9 @@ suite('HTML Semantic Tokens', () => { /*9*/'', ]; await assertTokens(input, [ - t(3, 7, 5, 'type.declaration'), t(3, 15, 3, 'interface') /* to investiagte */, + t(3, 7, 5, 'type.declaration'), t(3, 15, 3, 'interface.defaultLibrary') /* to investiagte */, t(4, 11, 1, 'function.declaration'), t(4, 13, 1, 'typeParameter.declaration'), t(4, 23, 5, 'type'), t(4, 30, 1, 'parameter.declaration'), t(4, 33, 1, 'typeParameter'), t(4, 47, 1, 'typeParameter'), - t(5, 12, 1, 'typeParameter'), t(5, 29, 3, 'interface'), t(5, 41, 5, 'type'), + t(5, 12, 1, 'typeParameter'), t(5, 29, 3, 'class.defaultLibrary'), t(5, 41, 5, 'type'), ]); }); @@ -197,7 +197,7 @@ suite('HTML Semantic Tokens', () => { ]; await assertTokens(input, [ t(3, 11, 1, 'function.declaration'), t(3, 13, 1, 'typeParameter.declaration'), t(3, 16, 2, 'parameter.declaration'), t(3, 20, 1, 'typeParameter'), t(3, 24, 1, 'typeParameter'), t(3, 39, 2, 'parameter'), - t(6, 2, 6, 'variable'), t(6, 9, 5, 'method') + t(6, 2, 6, 'variable.defaultLibrary'), t(6, 9, 5, 'method.defaultLibrary') ]); }); @@ -215,11 +215,11 @@ suite('HTML Semantic Tokens', () => { /*9*/'', ]; await assertTokens(input, [ - t(3, 2, 6, 'variable'), t(3, 9, 5, 'method') + t(3, 2, 6, 'variable.defaultLibrary'), t(3, 9, 5, 'method.defaultLibrary') ], [Range.create(Position.create(2, 0), Position.create(4, 0))]); await assertTokens(input, [ - t(6, 2, 6, 'variable'), + t(6, 2, 6, 'variable.defaultLibrary'), ], [Range.create(Position.create(6, 2), Position.create(6, 8))]); });