development/vscode
development/vscode
1
0
Fork 0
mirror of https://github.com/Microsoft/vscode synced 2024-07-17 02:57:19 +00:00
Code Issues Actions 9 Packages Projects Releases Wiki Activity

Adopt official semantic tokens api for JS/TS in html (#144223)

Browse source 
* Adopt official semantic tokens api for JS/TS in html

Fixes #114477

This switches the semantic tokens implementation for js/ts inside html to use the finalized `getEncodedSemanticClassifications` api instead of our custom impl

* Update tests
This commit is contained in:
Matt Bierner 2022-03-02 13:02:43 -08:00 committed by GitHub
parent 98da73d8aa
commit afa70ebba0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 77 additions and 112 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
extensions/html-language-features/server/src/modes/javascriptMode.ts
View file

@ -360,7 +360,7 @@ export function getJavaScriptMode(documentRegions: LanguageModelCache<HTMLDocume
async getSemanticTokens(document: TextDocument): Promise<SemanticTokenData[]> {
const jsDocument = jsDocuments.get(document);
const jsLanguageService = await host.getLanguageService(jsDocument);
return getSemanticTokens(jsLanguageService, jsDocument, jsDocument.uri);
return [...getSemanticTokens(jsLanguageService, jsDocument, jsDocument.uri)];
},
getSemanticTokenLegend(): { types: string[]; modifiers: string[] } {
return getSemanticTokenLegend();

159
extensions/html-language-features/server/src/modes/javascriptSemanticTokens.ts
View file

@ -16,94 +16,74 @@ export function getSemanticTokenLegend() {
return { types: tokenTypes, modifiers: tokenModifiers };
}
export function getSemanticTokens(jsLanguageService: ts.LanguageService, currentTextDocument: TextDocument, fileName: string): SemanticTokenData[] {
//https://ts-ast-viewer.com/#code/AQ0g2CmAuwGbALzAJwG4BQZQGNwEMBnQ4AQQEYBmYAb2C22zgEtJwATJVTRxgcwD27AQAp8AGmAAjAJS0A9POB8+7NQ168oscAJz5wANXwAnLug2bsJmAFcTAO2XAA1MHyvgu-UdOeWbOw8ViAAvpagocBAA
export function* getSemanticTokens(jsLanguageService: ts.LanguageService, document: TextDocument, fileName: string): Iterable<SemanticTokenData> {
const { spans } = jsLanguageService.getEncodedSemanticClassifications(fileName, { start: 0, length: document.getText().length }, '2020' as ts.SemanticClassificationFormat);
let resultTokens: SemanticTokenData[] = [];
const collector = (node: ts.Node, typeIdx: number, modifierSet: number) => {
resultTokens.push({ start: currentTextDocument.positionAt(node.getStart()), length: node.getWidth(), typeIdx, modifierSet });
};
collectTokens(jsLanguageService, fileName, { start: 0, length: currentTextDocument.getText().length }, collector);
for (let i = 0; i < spans.length;) {
const offset = spans[i++];
const length = spans[i++];
const tsClassification = spans[i++];
return resultTokens;
}
function collectTokens(jsLanguageService: ts.LanguageService, fileName: string, span: ts.TextSpan, collector: (node: ts.Node, tokenType: number, tokenModifier: number) => void) {
const program = jsLanguageService.getProgram();
if (program) {
const typeChecker = program.getTypeChecker();
function visit(node: ts.Node) {
if (!node || !ts.textSpanIntersectsWith(span, node.pos, node.getFullWidth())) {
return;
}
if (ts.isIdentifier(node)) {
let symbol = typeChecker.getSymbolAtLocation(node);
if (symbol) {
if (symbol.flags & ts.SymbolFlags.Alias) {
symbol = typeChecker.getAliasedSymbol(symbol);
}
let typeIdx = classifySymbol(symbol);
if (typeIdx !== undefined) {
let modifierSet = 0;
if (node.parent) {
const parentTypeIdx = tokenFromDeclarationMapping[node.parent.kind];
if (parentTypeIdx === typeIdx && (<ts.NamedDeclaration>node.parent).name === node) {
modifierSet = 1 << TokenModifier.declaration;
}
}
const decl = symbol.valueDeclaration;
const modifiers = decl ? ts.getCombinedModifierFlags(decl) : 0;
const nodeFlags = decl ? ts.getCombinedNodeFlags(decl) : 0;
if (modifiers & ts.ModifierFlags.Static) {
modifierSet |= 1 << TokenModifier.static;
}
if (modifiers & ts.ModifierFlags.Async) {
modifierSet |= 1 << TokenModifier.async;
}
if ((modifiers & ts.ModifierFlags.Readonly) || (nodeFlags & ts.NodeFlags.Const) || (symbol.getFlags() & ts.SymbolFlags.EnumMember)) {
modifierSet |= 1 << TokenModifier.readonly;
}
collector(node, typeIdx, modifierSet);
}
}
}
ts.forEachChild(node, visit);
}
const sourceFile = program.getSourceFile(fileName);
if (sourceFile) {
visit(sourceFile);
const tokenType = getTokenTypeFromClassification(tsClassification);
if (tokenType === undefined) {
continue;
}
const tokenModifiers = getTokenModifierFromClassification(tsClassification);
const startPos = document.positionAt(offset);
yield {
start: startPos,
length: length,
typeIdx: tokenType,
modifierSet: tokenModifiers
};
}
}
function classifySymbol(symbol: ts.Symbol) {
const flags = symbol.getFlags();
if (flags & ts.SymbolFlags.Class) {
return TokenType.class;
} else if (flags & ts.SymbolFlags.Enum) {
return TokenType.enum;
} else if (flags & ts.SymbolFlags.TypeAlias) {
return TokenType.type;
} else if (flags & ts.SymbolFlags.Type) {
if (flags & ts.SymbolFlags.Interface) {
return TokenType.interface;
} if (flags & ts.SymbolFlags.TypeParameter) {
return TokenType.typeParameter;
}
// typescript encodes type and modifiers in the classification:
// TSClassification = (TokenType + 1) << 8 + TokenModifier
const enum TokenType {
class = 0,
enum = 1,
interface = 2,
namespace = 3,
typeParameter = 4,
type = 5,
parameter = 6,
variable = 7,
enumMember = 8,
property = 9,
function = 10,
method = 11,
_ = 12
}
const enum TokenModifier {
declaration = 0,
static = 1,
async = 2,
readonly = 3,
defaultLibrary = 4,
local = 5,
_ = 6
}
const enum TokenEncodingConsts {
typeOffset = 8,
modifierMask = 255
}
function getTokenTypeFromClassification(tsClassification: number): number | undefined {
if (tsClassification > TokenEncodingConsts.modifierMask) {
return (tsClassification >> TokenEncodingConsts.typeOffset) - 1;
}
const decl = symbol.valueDeclaration || symbol.declarations && symbol.declarations[0];
return decl && tokenFromDeclarationMapping[decl.kind];
return undefined;
}
export const enum TokenType {
class, enum, interface, namespace, typeParameter, type, parameter, variable, property, function, method, _
}
export const enum TokenModifier {
declaration, static, async, readonly, _
function getTokenModifierFromClassification(tsClassification: number) {
return tsClassification & TokenEncodingConsts.modifierMask;
}
const tokenTypes: string[] = [];
@ -115,6 +95,7 @@ tokenTypes[TokenType.typeParameter] = 'typeParameter';
tokenTypes[TokenType.type] = 'type';
tokenTypes[TokenType.parameter] = 'parameter';
tokenTypes[TokenType.variable] = 'variable';
tokenTypes[TokenType.enumMember] = 'enumMember';
tokenTypes[TokenType.property] = 'property';
tokenTypes[TokenType.function] = 'function';
tokenTypes[TokenType.method] = 'method';
@ -124,21 +105,5 @@ tokenModifiers[TokenModifier.async] = 'async';
tokenModifiers[TokenModifier.declaration] = 'declaration';
tokenModifiers[TokenModifier.readonly] = 'readonly';
tokenModifiers[TokenModifier.static] = 'static';
const tokenFromDeclarationMapping: { [name: string]: TokenType } = {
[ts.SyntaxKind.VariableDeclaration]: TokenType.variable,
[ts.SyntaxKind.Parameter]: TokenType.parameter,
[ts.SyntaxKind.PropertyDeclaration]: TokenType.property,
[ts.SyntaxKind.ModuleDeclaration]: TokenType.namespace,
[ts.SyntaxKind.EnumDeclaration]: TokenType.enum,
[ts.SyntaxKind.EnumMember]: TokenType.property,
[ts.SyntaxKind.ClassDeclaration]: TokenType.class,
[ts.SyntaxKind.MethodDeclaration]: TokenType.method,
[ts.SyntaxKind.FunctionDeclaration]: TokenType.function,
[ts.SyntaxKind.MethodSignature]: TokenType.method,
[ts.SyntaxKind.GetAccessor]: TokenType.property,
[ts.SyntaxKind.PropertySignature]: TokenType.property,
[ts.SyntaxKind.InterfaceDeclaration]: TokenType.interface,
[ts.SyntaxKind.TypeAliasDeclaration]: TokenType.type,
[ts.SyntaxKind.TypeParameter]: TokenType.typeParameter
};
tokenModifiers[TokenModifier.local] = 'local';
tokenModifiers[TokenModifier.defaultLibrary] = 'defaultLibrary';

28
extensions/html-language-features/server/src/test/semanticTokens.test.ts
View file

@ -66,8 +66,8 @@ suite('HTML Semantic Tokens', () => {
];
await assertTokens(input, [
t(3, 6, 1, 'variable.declaration'), t(3, 13, 2, 'variable.declaration'), t(3, 19, 1, 'variable'),
t(5, 15, 1, 'variable.declaration.readonly'), t(5, 20, 2, 'variable'), t(5, 26, 1, 'variable'), t(5, 30, 1, 'variable.readonly'),
t(6, 11, 1, 'variable.declaration'),
t(5, 15, 1, 'variable.declaration.readonly.local'), t(5, 20, 2, 'variable'), t(5, 26, 1, 'variable'), t(5, 30, 1, 'variable.readonly.local'),
t(6, 11, 1, 'variable.declaration.local'),
t(7, 10, 2, 'variable')
]);
});
@ -87,8 +87,8 @@ suite('HTML Semantic Tokens', () => {
];
await assertTokens(input, [
t(3, 11, 3, 'function.declaration'), t(3, 15, 2, 'parameter.declaration'),
t(4, 11, 3, 'function'), t(4, 15, 4, 'interface'), t(4, 20, 3, 'method'), t(4, 24, 2, 'parameter'),
t(6, 6, 6, 'variable'), t(6, 13, 8, 'property'), t(6, 24, 5, 'method'), t(6, 35, 7, 'method'), t(6, 43, 1, 'parameter.declaration'), t(6, 48, 3, 'function'), t(6, 52, 1, 'parameter')
t(4, 11, 3, 'function'), t(4, 15, 4, 'variable.defaultLibrary'), t(4, 20, 3, 'method.defaultLibrary'), t(4, 24, 2, 'parameter'),
t(6, 6, 6, 'variable.defaultLibrary'), t(6, 13, 8, 'property.defaultLibrary'), t(6, 24, 5, 'method.defaultLibrary'), t(6, 35, 7, 'method.defaultLibrary'), t(6, 43, 1, 'parameter.declaration'), t(6, 48, 3, 'function'), t(6, 52, 1, 'parameter')
]);
});
@ -135,8 +135,8 @@ suite('HTML Semantic Tokens', () => {
];
await assertTokens(input, [
t(3, 12, 8, 'interface.declaration'), t(3, 23, 1, 'property.declaration'), t(3, 34, 1, 'property.declaration'),
t(4, 8, 1, 'variable.declaration.readonly'), t(4, 30, 8, 'interface'),
t(5, 8, 3, 'variable.declaration.readonly'), t(5, 15, 1, 'parameter.declaration'), t(5, 18, 8, 'interface'), t(5, 31, 1, 'parameter'), t(5, 33, 1, 'property'), t(5, 37, 1, 'parameter'), t(5, 39, 1, 'property')
t(4, 8, 1, 'variable.declaration.readonly'), t(4, 14, 1, 'property.declaration'), t(4, 20, 1, 'property.declaration'), t(4, 30, 8, 'interface'),
t(5, 8, 3, 'function.declaration.readonly'), t(5, 15, 1, 'parameter.declaration'), t(5, 18, 8, 'interface'), t(5, 31, 1, 'parameter'), t(5, 33, 1, 'property'), t(5, 37, 1, 'parameter'), t(5, 39, 1, 'property')
]);
});
@ -155,9 +155,9 @@ suite('HTML Semantic Tokens', () => {
];
await assertTokens(input, [
t(3, 8, 1, 'variable.declaration.readonly'),
t(4, 8, 1, 'class.declaration'), t(4, 28, 1, 'property.declaration.static.readonly'), t(4, 42, 3, 'property.declaration.static'), t(4, 47, 3, 'interface'),
t(5, 13, 1, 'enum.declaration'), t(5, 17, 1, 'property.declaration.readonly'), t(5, 24, 1, 'property.declaration.readonly'), t(5, 28, 1, 'property.readonly'),
t(6, 2, 7, 'variable'), t(6, 10, 3, 'method'), t(6, 14, 1, 'variable.readonly'), t(6, 18, 1, 'class'), t(6, 20, 1, 'property.static.readonly'), t(6, 24, 1, 'class'), t(6, 26, 3, 'property.static'), t(6, 30, 6, 'property.readonly'),
t(4, 8, 1, 'class.declaration'), t(4, 28, 1, 'property.declaration.static.readonly'), t(4, 42, 3, 'property.declaration.static'), t(4, 47, 3, 'interface.defaultLibrary'),
t(5, 13, 1, 'enum.declaration'), t(5, 17, 1, 'enumMember.declaration.readonly'), t(5, 24, 1, 'enumMember.declaration.readonly'), t(5, 28, 1, 'enumMember.readonly'),
t(6, 2, 7, 'variable.defaultLibrary'), t(6, 10, 3, 'method.defaultLibrary'), t(6, 14, 1, 'variable.readonly'), t(6, 18, 1, 'class'), t(6, 20, 1, 'property.static.readonly'), t(6, 24, 1, 'class'), t(6, 26, 3, 'property.static'), t(6, 30, 6, 'property.readonly.defaultLibrary'),
]);
});
@ -176,9 +176,9 @@ suite('HTML Semantic Tokens', () => {
/*9*/'</html>',
];
await assertTokens(input, [
t(3, 7, 5, 'type.declaration'), t(3, 15, 3, 'interface') /* to investiagte */,
t(3, 7, 5, 'type.declaration'), t(3, 15, 3, 'interface.defaultLibrary') /* to investiagte */,
t(4, 11, 1, 'function.declaration'), t(4, 13, 1, 'typeParameter.declaration'), t(4, 23, 5, 'type'), t(4, 30, 1, 'parameter.declaration'), t(4, 33, 1, 'typeParameter'), t(4, 47, 1, 'typeParameter'),
t(5, 12, 1, 'typeParameter'), t(5, 29, 3, 'interface'), t(5, 41, 5, 'type'),
t(5, 12, 1, 'typeParameter'), t(5, 29, 3, 'class.defaultLibrary'), t(5, 41, 5, 'type'),
]);
});
@ -197,7 +197,7 @@ suite('HTML Semantic Tokens', () => {
];
await assertTokens(input, [
t(3, 11, 1, 'function.declaration'), t(3, 13, 1, 'typeParameter.declaration'), t(3, 16, 2, 'parameter.declaration'), t(3, 20, 1, 'typeParameter'), t(3, 24, 1, 'typeParameter'), t(3, 39, 2, 'parameter'),
t(6, 2, 6, 'variable'), t(6, 9, 5, 'method')
t(6, 2, 6, 'variable.defaultLibrary'), t(6, 9, 5, 'method.defaultLibrary')
]);
});
@ -215,11 +215,11 @@ suite('HTML Semantic Tokens', () => {
/*9*/'</html>',
];
await assertTokens(input, [
t(3, 2, 6, 'variable'), t(3, 9, 5, 'method')
t(3, 2, 6, 'variable.defaultLibrary'), t(3, 9, 5, 'method.defaultLibrary')
], [Range.create(Position.create(2, 0), Position.create(4, 0))]);
await assertTokens(input, [
t(6, 2, 6, 'variable'),
t(6, 2, 6, 'variable.defaultLibrary'),
], [Range.create(Position.create(6, 2), Position.create(6, 8))]);
});