mirror of
https://github.com/Microsoft/vscode
synced 2024-08-27 13:00:58 +00:00
Adopt official semantic tokens api for JS/TS in html (#144223)
* Adopt official semantic tokens api for JS/TS in html Fixes #114477 This switches the semantic tokens implementation for js/ts inside html to use the finalized `getEncodedSemanticClassifications` api instead of our custom impl * Update tests
This commit is contained in:
parent
98da73d8aa
commit
afa70ebba0
|
@ -360,7 +360,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
|
|||
async getSemanticTokens(document: TextDocument): Promise<SemanticTokenData[]> {
|
||||
const jsDocument = jsDocuments.get(document);
|
||||
const jsLanguageService = await host.getLanguageService(jsDocument);
|
||||
return getSemanticTokens(jsLanguageService, jsDocument, jsDocument.uri);
|
||||
return [...getSemanticTokens(jsLanguageService, jsDocument, jsDocument.uri)];
|
||||
},
|
||||
getSemanticTokenLegend(): { types: string[]; modifiers: string[] } {
|
||||
return getSemanticTokenLegend();
|
||||
|
|
|
@ -16,94 +16,74 @@ export function getSemanticTokenLegend() {
|
|||
return { types: tokenTypes, modifiers: tokenModifiers };
|
||||
}
|
||||
|
||||
export function getSemanticTokens(jsLanguageService: ts.LanguageService, currentTextDocument: TextDocument, fileName: string): SemanticTokenData[] {
|
||||
//https://ts-ast-viewer.com/#code/AQ0g2CmAuwGbALzAJwG4BQZQGNwEMBnQ4AQQEYBmYAb2C22zgEtJwATJVTRxgcwD27AQAp8AGmAAjAJS0A9POB8+7NQ168oscAJz5wANXwAnLug2bsJmAFcTAO2XAA1MHyvgu-UdOeWbOw8ViAAvpagocBAA
|
||||
export function* getSemanticTokens(jsLanguageService: ts.LanguageService, document: TextDocument, fileName: string): Iterable<SemanticTokenData> {
|
||||
const { spans } = jsLanguageService.getEncodedSemanticClassifications(fileName, { start: 0, length: document.getText().length }, '2020' as ts.SemanticClassificationFormat);
|
||||
|
||||
let resultTokens: SemanticTokenData[] = [];
|
||||
const collector = (node: ts.Node, typeIdx: number, modifierSet: number) => {
|
||||
resultTokens.push({ start: currentTextDocument.positionAt(node.getStart()), length: node.getWidth(), typeIdx, modifierSet });
|
||||
for (let i = 0; i < spans.length;) {
|
||||
const offset = spans[i++];
|
||||
const length = spans[i++];
|
||||
const tsClassification = spans[i++];
|
||||
|
||||
const tokenType = getTokenTypeFromClassification(tsClassification);
|
||||
if (tokenType === undefined) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const tokenModifiers = getTokenModifierFromClassification(tsClassification);
|
||||
const startPos = document.positionAt(offset);
|
||||
yield {
|
||||
start: startPos,
|
||||
length: length,
|
||||
typeIdx: tokenType,
|
||||
modifierSet: tokenModifiers
|
||||
};
|
||||
collectTokens(jsLanguageService, fileName, { start: 0, length: currentTextDocument.getText().length }, collector);
|
||||
|
||||
return resultTokens;
|
||||
}
|
||||
|
||||
function collectTokens(jsLanguageService: ts.LanguageService, fileName: string, span: ts.TextSpan, collector: (node: ts.Node, tokenType: number, tokenModifier: number) => void) {
|
||||
|
||||
const program = jsLanguageService.getProgram();
|
||||
if (program) {
|
||||
const typeChecker = program.getTypeChecker();
|
||||
|
||||
function visit(node: ts.Node) {
|
||||
if (!node || !ts.textSpanIntersectsWith(span, node.pos, node.getFullWidth())) {
|
||||
return;
|
||||
}
|
||||
if (ts.isIdentifier(node)) {
|
||||
let symbol = typeChecker.getSymbolAtLocation(node);
|
||||
if (symbol) {
|
||||
if (symbol.flags & ts.SymbolFlags.Alias) {
|
||||
symbol = typeChecker.getAliasedSymbol(symbol);
|
||||
}
|
||||
let typeIdx = classifySymbol(symbol);
|
||||
if (typeIdx !== undefined) {
|
||||
let modifierSet = 0;
|
||||
if (node.parent) {
|
||||
const parentTypeIdx = tokenFromDeclarationMapping[node.parent.kind];
|
||||
if (parentTypeIdx === typeIdx && (<ts.NamedDeclaration>node.parent).name === node) {
|
||||
modifierSet = 1 << TokenModifier.declaration;
|
||||
}
|
||||
}
|
||||
const decl = symbol.valueDeclaration;
|
||||
const modifiers = decl ? ts.getCombinedModifierFlags(decl) : 0;
|
||||
const nodeFlags = decl ? ts.getCombinedNodeFlags(decl) : 0;
|
||||
if (modifiers & ts.ModifierFlags.Static) {
|
||||
modifierSet |= 1 << TokenModifier.static;
|
||||
}
|
||||
if (modifiers & ts.ModifierFlags.Async) {
|
||||
modifierSet |= 1 << TokenModifier.async;
|
||||
}
|
||||
if ((modifiers & ts.ModifierFlags.Readonly) || (nodeFlags & ts.NodeFlags.Const) || (symbol.getFlags() & ts.SymbolFlags.EnumMember)) {
|
||||
modifierSet |= 1 << TokenModifier.readonly;
|
||||
}
|
||||
collector(node, typeIdx, modifierSet);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ts.forEachChild(node, visit);
|
||||
}
|
||||
const sourceFile = program.getSourceFile(fileName);
|
||||
if (sourceFile) {
|
||||
visit(sourceFile);
|
||||
}
|
||||
}
|
||||
|
||||
// typescript encodes type and modifiers in the classification:
|
||||
// TSClassification = (TokenType + 1) << 8 + TokenModifier
|
||||
|
||||
const enum TokenType {
|
||||
class = 0,
|
||||
enum = 1,
|
||||
interface = 2,
|
||||
namespace = 3,
|
||||
typeParameter = 4,
|
||||
type = 5,
|
||||
parameter = 6,
|
||||
variable = 7,
|
||||
enumMember = 8,
|
||||
property = 9,
|
||||
function = 10,
|
||||
method = 11,
|
||||
_ = 12
|
||||
}
|
||||
|
||||
function classifySymbol(symbol: ts.Symbol) {
|
||||
const flags = symbol.getFlags();
|
||||
if (flags & ts.SymbolFlags.Class) {
|
||||
return TokenType.class;
|
||||
} else if (flags & ts.SymbolFlags.Enum) {
|
||||
return TokenType.enum;
|
||||
} else if (flags & ts.SymbolFlags.TypeAlias) {
|
||||
return TokenType.type;
|
||||
} else if (flags & ts.SymbolFlags.Type) {
|
||||
if (flags & ts.SymbolFlags.Interface) {
|
||||
return TokenType.interface;
|
||||
} if (flags & ts.SymbolFlags.TypeParameter) {
|
||||
return TokenType.typeParameter;
|
||||
}
|
||||
}
|
||||
const decl = symbol.valueDeclaration || symbol.declarations && symbol.declarations[0];
|
||||
return decl && tokenFromDeclarationMapping[decl.kind];
|
||||
const enum TokenModifier {
|
||||
declaration = 0,
|
||||
static = 1,
|
||||
async = 2,
|
||||
readonly = 3,
|
||||
defaultLibrary = 4,
|
||||
local = 5,
|
||||
_ = 6
|
||||
}
|
||||
|
||||
export const enum TokenType {
|
||||
class, enum, interface, namespace, typeParameter, type, parameter, variable, property, function, method, _
|
||||
const enum TokenEncodingConsts {
|
||||
typeOffset = 8,
|
||||
modifierMask = 255
|
||||
}
|
||||
|
||||
export const enum TokenModifier {
|
||||
declaration, static, async, readonly, _
|
||||
function getTokenTypeFromClassification(tsClassification: number): number | undefined {
|
||||
if (tsClassification > TokenEncodingConsts.modifierMask) {
|
||||
return (tsClassification >> TokenEncodingConsts.typeOffset) - 1;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function getTokenModifierFromClassification(tsClassification: number) {
|
||||
return tsClassification & TokenEncodingConsts.modifierMask;
|
||||
}
|
||||
|
||||
const tokenTypes: string[] = [];
|
||||
|
@ -115,6 +95,7 @@ tokenTypes[TokenType.typeParameter] = 'typeParameter';
|
|||
tokenTypes[TokenType.type] = 'type';
|
||||
tokenTypes[TokenType.parameter] = 'parameter';
|
||||
tokenTypes[TokenType.variable] = 'variable';
|
||||
tokenTypes[TokenType.enumMember] = 'enumMember';
|
||||
tokenTypes[TokenType.property] = 'property';
|
||||
tokenTypes[TokenType.function] = 'function';
|
||||
tokenTypes[TokenType.method] = 'method';
|
||||
|
@ -124,21 +105,5 @@ tokenModifiers[TokenModifier.async] = 'async';
|
|||
tokenModifiers[TokenModifier.declaration] = 'declaration';
|
||||
tokenModifiers[TokenModifier.readonly] = 'readonly';
|
||||
tokenModifiers[TokenModifier.static] = 'static';
|
||||
|
||||
const tokenFromDeclarationMapping: { [name: string]: TokenType } = {
|
||||
[ts.SyntaxKind.VariableDeclaration]: TokenType.variable,
|
||||
[ts.SyntaxKind.Parameter]: TokenType.parameter,
|
||||
[ts.SyntaxKind.PropertyDeclaration]: TokenType.property,
|
||||
[ts.SyntaxKind.ModuleDeclaration]: TokenType.namespace,
|
||||
[ts.SyntaxKind.EnumDeclaration]: TokenType.enum,
|
||||
[ts.SyntaxKind.EnumMember]: TokenType.property,
|
||||
[ts.SyntaxKind.ClassDeclaration]: TokenType.class,
|
||||
[ts.SyntaxKind.MethodDeclaration]: TokenType.method,
|
||||
[ts.SyntaxKind.FunctionDeclaration]: TokenType.function,
|
||||
[ts.SyntaxKind.MethodSignature]: TokenType.method,
|
||||
[ts.SyntaxKind.GetAccessor]: TokenType.property,
|
||||
[ts.SyntaxKind.PropertySignature]: TokenType.property,
|
||||
[ts.SyntaxKind.InterfaceDeclaration]: TokenType.interface,
|
||||
[ts.SyntaxKind.TypeAliasDeclaration]: TokenType.type,
|
||||
[ts.SyntaxKind.TypeParameter]: TokenType.typeParameter
|
||||
};
|
||||
tokenModifiers[TokenModifier.local] = 'local';
|
||||
tokenModifiers[TokenModifier.defaultLibrary] = 'defaultLibrary';
|
||||
|
|
|
@ -66,8 +66,8 @@ suite('HTML Semantic Tokens', () => {
|
|||
];
|
||||
await assertTokens(input, [
|
||||
t(3, 6, 1, 'variable.declaration'), t(3, 13, 2, 'variable.declaration'), t(3, 19, 1, 'variable'),
|
||||
t(5, 15, 1, 'variable.declaration.readonly'), t(5, 20, 2, 'variable'), t(5, 26, 1, 'variable'), t(5, 30, 1, 'variable.readonly'),
|
||||
t(6, 11, 1, 'variable.declaration'),
|
||||
t(5, 15, 1, 'variable.declaration.readonly.local'), t(5, 20, 2, 'variable'), t(5, 26, 1, 'variable'), t(5, 30, 1, 'variable.readonly.local'),
|
||||
t(6, 11, 1, 'variable.declaration.local'),
|
||||
t(7, 10, 2, 'variable')
|
||||
]);
|
||||
});
|
||||
|
@ -87,8 +87,8 @@ suite('HTML Semantic Tokens', () => {
|
|||
];
|
||||
await assertTokens(input, [
|
||||
t(3, 11, 3, 'function.declaration'), t(3, 15, 2, 'parameter.declaration'),
|
||||
t(4, 11, 3, 'function'), t(4, 15, 4, 'interface'), t(4, 20, 3, 'method'), t(4, 24, 2, 'parameter'),
|
||||
t(6, 6, 6, 'variable'), t(6, 13, 8, 'property'), t(6, 24, 5, 'method'), t(6, 35, 7, 'method'), t(6, 43, 1, 'parameter.declaration'), t(6, 48, 3, 'function'), t(6, 52, 1, 'parameter')
|
||||
t(4, 11, 3, 'function'), t(4, 15, 4, 'variable.defaultLibrary'), t(4, 20, 3, 'method.defaultLibrary'), t(4, 24, 2, 'parameter'),
|
||||
t(6, 6, 6, 'variable.defaultLibrary'), t(6, 13, 8, 'property.defaultLibrary'), t(6, 24, 5, 'method.defaultLibrary'), t(6, 35, 7, 'method.defaultLibrary'), t(6, 43, 1, 'parameter.declaration'), t(6, 48, 3, 'function'), t(6, 52, 1, 'parameter')
|
||||
]);
|
||||
});
|
||||
|
||||
|
@ -135,8 +135,8 @@ suite('HTML Semantic Tokens', () => {
|
|||
];
|
||||
await assertTokens(input, [
|
||||
t(3, 12, 8, 'interface.declaration'), t(3, 23, 1, 'property.declaration'), t(3, 34, 1, 'property.declaration'),
|
||||
t(4, 8, 1, 'variable.declaration.readonly'), t(4, 30, 8, 'interface'),
|
||||
t(5, 8, 3, 'variable.declaration.readonly'), t(5, 15, 1, 'parameter.declaration'), t(5, 18, 8, 'interface'), t(5, 31, 1, 'parameter'), t(5, 33, 1, 'property'), t(5, 37, 1, 'parameter'), t(5, 39, 1, 'property')
|
||||
t(4, 8, 1, 'variable.declaration.readonly'), t(4, 14, 1, 'property.declaration'), t(4, 20, 1, 'property.declaration'), t(4, 30, 8, 'interface'),
|
||||
t(5, 8, 3, 'function.declaration.readonly'), t(5, 15, 1, 'parameter.declaration'), t(5, 18, 8, 'interface'), t(5, 31, 1, 'parameter'), t(5, 33, 1, 'property'), t(5, 37, 1, 'parameter'), t(5, 39, 1, 'property')
|
||||
]);
|
||||
});
|
||||
|
||||
|
@ -155,9 +155,9 @@ suite('HTML Semantic Tokens', () => {
|
|||
];
|
||||
await assertTokens(input, [
|
||||
t(3, 8, 1, 'variable.declaration.readonly'),
|
||||
t(4, 8, 1, 'class.declaration'), t(4, 28, 1, 'property.declaration.static.readonly'), t(4, 42, 3, 'property.declaration.static'), t(4, 47, 3, 'interface'),
|
||||
t(5, 13, 1, 'enum.declaration'), t(5, 17, 1, 'property.declaration.readonly'), t(5, 24, 1, 'property.declaration.readonly'), t(5, 28, 1, 'property.readonly'),
|
||||
t(6, 2, 7, 'variable'), t(6, 10, 3, 'method'), t(6, 14, 1, 'variable.readonly'), t(6, 18, 1, 'class'), t(6, 20, 1, 'property.static.readonly'), t(6, 24, 1, 'class'), t(6, 26, 3, 'property.static'), t(6, 30, 6, 'property.readonly'),
|
||||
t(4, 8, 1, 'class.declaration'), t(4, 28, 1, 'property.declaration.static.readonly'), t(4, 42, 3, 'property.declaration.static'), t(4, 47, 3, 'interface.defaultLibrary'),
|
||||
t(5, 13, 1, 'enum.declaration'), t(5, 17, 1, 'enumMember.declaration.readonly'), t(5, 24, 1, 'enumMember.declaration.readonly'), t(5, 28, 1, 'enumMember.readonly'),
|
||||
t(6, 2, 7, 'variable.defaultLibrary'), t(6, 10, 3, 'method.defaultLibrary'), t(6, 14, 1, 'variable.readonly'), t(6, 18, 1, 'class'), t(6, 20, 1, 'property.static.readonly'), t(6, 24, 1, 'class'), t(6, 26, 3, 'property.static'), t(6, 30, 6, 'property.readonly.defaultLibrary'),
|
||||
]);
|
||||
});
|
||||
|
||||
|
@ -176,9 +176,9 @@ suite('HTML Semantic Tokens', () => {
|
|||
/*9*/'</html>',
|
||||
];
|
||||
await assertTokens(input, [
|
||||
t(3, 7, 5, 'type.declaration'), t(3, 15, 3, 'interface') /* to investiagte */,
|
||||
t(3, 7, 5, 'type.declaration'), t(3, 15, 3, 'interface.defaultLibrary') /* to investiagte */,
|
||||
t(4, 11, 1, 'function.declaration'), t(4, 13, 1, 'typeParameter.declaration'), t(4, 23, 5, 'type'), t(4, 30, 1, 'parameter.declaration'), t(4, 33, 1, 'typeParameter'), t(4, 47, 1, 'typeParameter'),
|
||||
t(5, 12, 1, 'typeParameter'), t(5, 29, 3, 'interface'), t(5, 41, 5, 'type'),
|
||||
t(5, 12, 1, 'typeParameter'), t(5, 29, 3, 'class.defaultLibrary'), t(5, 41, 5, 'type'),
|
||||
]);
|
||||
});
|
||||
|
||||
|
@ -197,7 +197,7 @@ suite('HTML Semantic Tokens', () => {
|
|||
];
|
||||
await assertTokens(input, [
|
||||
t(3, 11, 1, 'function.declaration'), t(3, 13, 1, 'typeParameter.declaration'), t(3, 16, 2, 'parameter.declaration'), t(3, 20, 1, 'typeParameter'), t(3, 24, 1, 'typeParameter'), t(3, 39, 2, 'parameter'),
|
||||
t(6, 2, 6, 'variable'), t(6, 9, 5, 'method')
|
||||
t(6, 2, 6, 'variable.defaultLibrary'), t(6, 9, 5, 'method.defaultLibrary')
|
||||
]);
|
||||
});
|
||||
|
||||
|
@ -215,11 +215,11 @@ suite('HTML Semantic Tokens', () => {
|
|||
/*9*/'</html>',
|
||||
];
|
||||
await assertTokens(input, [
|
||||
t(3, 2, 6, 'variable'), t(3, 9, 5, 'method')
|
||||
t(3, 2, 6, 'variable.defaultLibrary'), t(3, 9, 5, 'method.defaultLibrary')
|
||||
], [Range.create(Position.create(2, 0), Position.create(4, 0))]);
|
||||
|
||||
await assertTokens(input, [
|
||||
t(6, 2, 6, 'variable'),
|
||||
t(6, 2, 6, 'variable.defaultLibrary'),
|
||||
], [Range.create(Position.create(6, 2), Position.create(6, 8))]);
|
||||
});
|
||||
|
||||
|
|
Loading…
Reference in a new issue