diff --git a/extensions/html-language-features/server/src/modes/javascriptMode.ts b/extensions/html-language-features/server/src/modes/javascriptMode.ts
index 301af0e58c5..36a41173b30 100644
--- a/extensions/html-language-features/server/src/modes/javascriptMode.ts
+++ b/extensions/html-language-features/server/src/modes/javascriptMode.ts
@@ -360,7 +360,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache {
const jsDocument = jsDocuments.get(document);
const jsLanguageService = await host.getLanguageService(jsDocument);
- return getSemanticTokens(jsLanguageService, jsDocument, jsDocument.uri);
+ return [...getSemanticTokens(jsLanguageService, jsDocument, jsDocument.uri)];
},
getSemanticTokenLegend(): { types: string[]; modifiers: string[] } {
return getSemanticTokenLegend();
diff --git a/extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts b/extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts
index f560559727a..cbcf1b45082 100644
--- a/extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts
+++ b/extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts
@@ -16,94 +16,74 @@ export function getSemanticTokenLegend() {
return { types: tokenTypes, modifiers: tokenModifiers };
}
-export function getSemanticTokens(jsLanguageService: ts.LanguageService, currentTextDocument: TextDocument, fileName: string): SemanticTokenData[] {
- //https://ts-ast-viewer.com/#code/AQ0g2CmAuwGbALzAJwG4BQZQGNwEMBnQ4AQQEYBmYAb2C22zgEtJwATJVTRxgcwD27AQAp8AGmAAjAJS0A9POB8+7NQ168oscAJz5wANXwAnLug2bsJmAFcTAO2XAA1MHyvgu-UdOeWbOw8ViAAvpagocBAA
+export function* getSemanticTokens(jsLanguageService: ts.LanguageService, document: TextDocument, fileName: string): Iterable {
+ const { spans } = jsLanguageService.getEncodedSemanticClassifications(fileName, { start: 0, length: document.getText().length }, '2020' as ts.SemanticClassificationFormat);
- let resultTokens: SemanticTokenData[] = [];
- const collector = (node: ts.Node, typeIdx: number, modifierSet: number) => {
- resultTokens.push({ start: currentTextDocument.positionAt(node.getStart()), length: node.getWidth(), typeIdx, modifierSet });
- };
- collectTokens(jsLanguageService, fileName, { start: 0, length: currentTextDocument.getText().length }, collector);
+ for (let i = 0; i < spans.length;) {
+ const offset = spans[i++];
+ const length = spans[i++];
+ const tsClassification = spans[i++];
- return resultTokens;
-}
-
-function collectTokens(jsLanguageService: ts.LanguageService, fileName: string, span: ts.TextSpan, collector: (node: ts.Node, tokenType: number, tokenModifier: number) => void) {
-
- const program = jsLanguageService.getProgram();
- if (program) {
- const typeChecker = program.getTypeChecker();
-
- function visit(node: ts.Node) {
- if (!node || !ts.textSpanIntersectsWith(span, node.pos, node.getFullWidth())) {
- return;
- }
- if (ts.isIdentifier(node)) {
- let symbol = typeChecker.getSymbolAtLocation(node);
- if (symbol) {
- if (symbol.flags & ts.SymbolFlags.Alias) {
- symbol = typeChecker.getAliasedSymbol(symbol);
- }
- let typeIdx = classifySymbol(symbol);
- if (typeIdx !== undefined) {
- let modifierSet = 0;
- if (node.parent) {
- const parentTypeIdx = tokenFromDeclarationMapping[node.parent.kind];
- if (parentTypeIdx === typeIdx && (node.parent).name === node) {
- modifierSet = 1 << TokenModifier.declaration;
- }
- }
- const decl = symbol.valueDeclaration;
- const modifiers = decl ? ts.getCombinedModifierFlags(decl) : 0;
- const nodeFlags = decl ? ts.getCombinedNodeFlags(decl) : 0;
- if (modifiers & ts.ModifierFlags.Static) {
- modifierSet |= 1 << TokenModifier.static;
- }
- if (modifiers & ts.ModifierFlags.Async) {
- modifierSet |= 1 << TokenModifier.async;
- }
- if ((modifiers & ts.ModifierFlags.Readonly) || (nodeFlags & ts.NodeFlags.Const) || (symbol.getFlags() & ts.SymbolFlags.EnumMember)) {
- modifierSet |= 1 << TokenModifier.readonly;
- }
- collector(node, typeIdx, modifierSet);
- }
- }
- }
-
- ts.forEachChild(node, visit);
- }
- const sourceFile = program.getSourceFile(fileName);
- if (sourceFile) {
- visit(sourceFile);
+ const tokenType = getTokenTypeFromClassification(tsClassification);
+ if (tokenType === undefined) {
+ continue;
}
+
+ const tokenModifiers = getTokenModifierFromClassification(tsClassification);
+ const startPos = document.positionAt(offset);
+ yield {
+ start: startPos,
+ length: length,
+ typeIdx: tokenType,
+ modifierSet: tokenModifiers
+ };
}
}
-function classifySymbol(symbol: ts.Symbol) {
- const flags = symbol.getFlags();
- if (flags & ts.SymbolFlags.Class) {
- return TokenType.class;
- } else if (flags & ts.SymbolFlags.Enum) {
- return TokenType.enum;
- } else if (flags & ts.SymbolFlags.TypeAlias) {
- return TokenType.type;
- } else if (flags & ts.SymbolFlags.Type) {
- if (flags & ts.SymbolFlags.Interface) {
- return TokenType.interface;
- } if (flags & ts.SymbolFlags.TypeParameter) {
- return TokenType.typeParameter;
- }
+
+// typescript encodes type and modifiers in the classification:
+// TSClassification = (TokenType + 1) << 8 + TokenModifier
+
+const enum TokenType {
+ class = 0,
+ enum = 1,
+ interface = 2,
+ namespace = 3,
+ typeParameter = 4,
+ type = 5,
+ parameter = 6,
+ variable = 7,
+ enumMember = 8,
+ property = 9,
+ function = 10,
+ method = 11,
+ _ = 12
+}
+
+const enum TokenModifier {
+ declaration = 0,
+ static = 1,
+ async = 2,
+ readonly = 3,
+ defaultLibrary = 4,
+ local = 5,
+ _ = 6
+}
+
+const enum TokenEncodingConsts {
+ typeOffset = 8,
+ modifierMask = 255
+}
+
+function getTokenTypeFromClassification(tsClassification: number): number | undefined {
+ if (tsClassification > TokenEncodingConsts.modifierMask) {
+ return (tsClassification >> TokenEncodingConsts.typeOffset) - 1;
}
- const decl = symbol.valueDeclaration || symbol.declarations && symbol.declarations[0];
- return decl && tokenFromDeclarationMapping[decl.kind];
+ return undefined;
}
-export const enum TokenType {
- class, enum, interface, namespace, typeParameter, type, parameter, variable, property, function, method, _
-}
-
-export const enum TokenModifier {
- declaration, static, async, readonly, _
+function getTokenModifierFromClassification(tsClassification: number) {
+ return tsClassification & TokenEncodingConsts.modifierMask;
}
const tokenTypes: string[] = [];
@@ -115,6 +95,7 @@ tokenTypes[TokenType.typeParameter] = 'typeParameter';
tokenTypes[TokenType.type] = 'type';
tokenTypes[TokenType.parameter] = 'parameter';
tokenTypes[TokenType.variable] = 'variable';
+tokenTypes[TokenType.enumMember] = 'enumMember';
tokenTypes[TokenType.property] = 'property';
tokenTypes[TokenType.function] = 'function';
tokenTypes[TokenType.method] = 'method';
@@ -124,21 +105,5 @@ tokenModifiers[TokenModifier.async] = 'async';
tokenModifiers[TokenModifier.declaration] = 'declaration';
tokenModifiers[TokenModifier.readonly] = 'readonly';
tokenModifiers[TokenModifier.static] = 'static';
-
-const tokenFromDeclarationMapping: { [name: string]: TokenType } = {
- [ts.SyntaxKind.VariableDeclaration]: TokenType.variable,
- [ts.SyntaxKind.Parameter]: TokenType.parameter,
- [ts.SyntaxKind.PropertyDeclaration]: TokenType.property,
- [ts.SyntaxKind.ModuleDeclaration]: TokenType.namespace,
- [ts.SyntaxKind.EnumDeclaration]: TokenType.enum,
- [ts.SyntaxKind.EnumMember]: TokenType.property,
- [ts.SyntaxKind.ClassDeclaration]: TokenType.class,
- [ts.SyntaxKind.MethodDeclaration]: TokenType.method,
- [ts.SyntaxKind.FunctionDeclaration]: TokenType.function,
- [ts.SyntaxKind.MethodSignature]: TokenType.method,
- [ts.SyntaxKind.GetAccessor]: TokenType.property,
- [ts.SyntaxKind.PropertySignature]: TokenType.property,
- [ts.SyntaxKind.InterfaceDeclaration]: TokenType.interface,
- [ts.SyntaxKind.TypeAliasDeclaration]: TokenType.type,
- [ts.SyntaxKind.TypeParameter]: TokenType.typeParameter
-};
+tokenModifiers[TokenModifier.local] = 'local';
+tokenModifiers[TokenModifier.defaultLibrary] = 'defaultLibrary';
diff --git a/extensions/html-language-features/server/src/test/semanticTokens.test.ts b/extensions/html-language-features/server/src/test/semanticTokens.test.ts
index 2d7677d79a5..2db27dd9f17 100644
--- a/extensions/html-language-features/server/src/test/semanticTokens.test.ts
+++ b/extensions/html-language-features/server/src/test/semanticTokens.test.ts
@@ -66,8 +66,8 @@ suite('HTML Semantic Tokens', () => {
];
await assertTokens(input, [
t(3, 6, 1, 'variable.declaration'), t(3, 13, 2, 'variable.declaration'), t(3, 19, 1, 'variable'),
- t(5, 15, 1, 'variable.declaration.readonly'), t(5, 20, 2, 'variable'), t(5, 26, 1, 'variable'), t(5, 30, 1, 'variable.readonly'),
- t(6, 11, 1, 'variable.declaration'),
+ t(5, 15, 1, 'variable.declaration.readonly.local'), t(5, 20, 2, 'variable'), t(5, 26, 1, 'variable'), t(5, 30, 1, 'variable.readonly.local'),
+ t(6, 11, 1, 'variable.declaration.local'),
t(7, 10, 2, 'variable')
]);
});
@@ -87,8 +87,8 @@ suite('HTML Semantic Tokens', () => {
];
await assertTokens(input, [
t(3, 11, 3, 'function.declaration'), t(3, 15, 2, 'parameter.declaration'),
- t(4, 11, 3, 'function'), t(4, 15, 4, 'interface'), t(4, 20, 3, 'method'), t(4, 24, 2, 'parameter'),
- t(6, 6, 6, 'variable'), t(6, 13, 8, 'property'), t(6, 24, 5, 'method'), t(6, 35, 7, 'method'), t(6, 43, 1, 'parameter.declaration'), t(6, 48, 3, 'function'), t(6, 52, 1, 'parameter')
+ t(4, 11, 3, 'function'), t(4, 15, 4, 'variable.defaultLibrary'), t(4, 20, 3, 'method.defaultLibrary'), t(4, 24, 2, 'parameter'),
+ t(6, 6, 6, 'variable.defaultLibrary'), t(6, 13, 8, 'property.defaultLibrary'), t(6, 24, 5, 'method.defaultLibrary'), t(6, 35, 7, 'method.defaultLibrary'), t(6, 43, 1, 'parameter.declaration'), t(6, 48, 3, 'function'), t(6, 52, 1, 'parameter')
]);
});
@@ -135,8 +135,8 @@ suite('HTML Semantic Tokens', () => {
];
await assertTokens(input, [
t(3, 12, 8, 'interface.declaration'), t(3, 23, 1, 'property.declaration'), t(3, 34, 1, 'property.declaration'),
- t(4, 8, 1, 'variable.declaration.readonly'), t(4, 30, 8, 'interface'),
- t(5, 8, 3, 'variable.declaration.readonly'), t(5, 15, 1, 'parameter.declaration'), t(5, 18, 8, 'interface'), t(5, 31, 1, 'parameter'), t(5, 33, 1, 'property'), t(5, 37, 1, 'parameter'), t(5, 39, 1, 'property')
+ t(4, 8, 1, 'variable.declaration.readonly'), t(4, 14, 1, 'property.declaration'), t(4, 20, 1, 'property.declaration'), t(4, 30, 8, 'interface'),
+ t(5, 8, 3, 'function.declaration.readonly'), t(5, 15, 1, 'parameter.declaration'), t(5, 18, 8, 'interface'), t(5, 31, 1, 'parameter'), t(5, 33, 1, 'property'), t(5, 37, 1, 'parameter'), t(5, 39, 1, 'property')
]);
});
@@ -155,9 +155,9 @@ suite('HTML Semantic Tokens', () => {
];
await assertTokens(input, [
t(3, 8, 1, 'variable.declaration.readonly'),
- t(4, 8, 1, 'class.declaration'), t(4, 28, 1, 'property.declaration.static.readonly'), t(4, 42, 3, 'property.declaration.static'), t(4, 47, 3, 'interface'),
- t(5, 13, 1, 'enum.declaration'), t(5, 17, 1, 'property.declaration.readonly'), t(5, 24, 1, 'property.declaration.readonly'), t(5, 28, 1, 'property.readonly'),
- t(6, 2, 7, 'variable'), t(6, 10, 3, 'method'), t(6, 14, 1, 'variable.readonly'), t(6, 18, 1, 'class'), t(6, 20, 1, 'property.static.readonly'), t(6, 24, 1, 'class'), t(6, 26, 3, 'property.static'), t(6, 30, 6, 'property.readonly'),
+ t(4, 8, 1, 'class.declaration'), t(4, 28, 1, 'property.declaration.static.readonly'), t(4, 42, 3, 'property.declaration.static'), t(4, 47, 3, 'interface.defaultLibrary'),
+ t(5, 13, 1, 'enum.declaration'), t(5, 17, 1, 'enumMember.declaration.readonly'), t(5, 24, 1, 'enumMember.declaration.readonly'), t(5, 28, 1, 'enumMember.readonly'),
+ t(6, 2, 7, 'variable.defaultLibrary'), t(6, 10, 3, 'method.defaultLibrary'), t(6, 14, 1, 'variable.readonly'), t(6, 18, 1, 'class'), t(6, 20, 1, 'property.static.readonly'), t(6, 24, 1, 'class'), t(6, 26, 3, 'property.static'), t(6, 30, 6, 'property.readonly.defaultLibrary'),
]);
});
@@ -176,9 +176,9 @@ suite('HTML Semantic Tokens', () => {
/*9*/'