development/vscode
development/vscode
1
0
Fork 0
mirror of https://github.com/Microsoft/vscode synced 2024-08-28 13:29:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adopt vscode-textmate@7.0.0

Browse source
This commit is contained in:
Alex Dima 2022-04-07 16:38:21 +02:00
parent d3b7727fae
commit 9fe7dabbdb
No known key found for this signature in database
GPG key ID: 39563C1504FDD0C9
9 changed files with 64 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
package.json
View file

@ -83,7 +83,7 @@
"vscode-oniguruma": "1.6.1",
"vscode-proxy-agent": "^0.12.0",
"vscode-regexpp": "^3.1.0",
"vscode-textmate": "6.0.0",
"vscode-textmate": "7.0.0",
"xterm": "4.19.0-beta.20",
"xterm-addon-search": "0.9.0-beta.18",
"xterm-addon-serialize": "0.7.0-beta.12",

2
remote/package.json
View file

@ -23,7 +23,7 @@
"vscode-oniguruma": "1.6.1",
"vscode-proxy-agent": "^0.12.0",
"vscode-regexpp": "^3.1.0",
"vscode-textmate": "6.0.0",
"vscode-textmate": "7.0.0",
"xterm": "4.19.0-beta.20",
"xterm-addon-search": "0.9.0-beta.18",
"xterm-addon-serialize": "0.7.0-beta.12",

2
remote/web/package.json
View file

@ -9,7 +9,7 @@
"jschardet": "3.0.0",
"tas-client-umd": "0.1.4",
"vscode-oniguruma": "1.6.1",
"vscode-textmate": "6.0.0",
"vscode-textmate": "7.0.0",
"xterm": "4.19.0-beta.20",
"xterm-addon-search": "0.9.0-beta.18",
"xterm-addon-unicode11": "0.4.0-beta.3",

8
remote/web/yarn.lock
View file

@ -108,10 +108,10 @@ vscode-oniguruma@1.6.1:
resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.6.1.tgz#2bf4dfcfe3dd2e56eb549a3068c8ee39e6c30ce5"
integrity sha512-vc4WhSIaVpgJ0jJIejjYxPvURJavX6QG41vu0mGhqywMkQqulezEqEQ3cO3gc8GvcOpX6ycmKGqRoROEMBNXTQ==
vscode-textmate@6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-6.0.0.tgz#a3777197235036814ac9a92451492f2748589210"
integrity sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ==
vscode-textmate@7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-7.0.0.tgz#6c90a591c1291e80c6783bddf43e841ad6f8ccc3"
integrity sha512-27bFXGG5JtLrq/reWnkCxkfNq7gnFko7xORba2Gznokx2LmPpHfu880a44B1jPyoFiGNjprZ883qiYTvR/P+pA==
xterm-addon-search@0.9.0-beta.18:
version "0.9.0-beta.18"

8
remote/yarn.lock
View file

@ -883,10 +883,10 @@ vscode-regexpp@^3.1.0:
resolved "https://registry.yarnpkg.com/vscode-regexpp/-/vscode-regexpp-3.1.0.tgz#42d059b6fffe99bd42939c0d013f632f0cad823f"
integrity sha512-pqtN65VC1jRLawfluX4Y80MMG0DHJydWhe5ZwMHewZD6sys4LbU6lHwFAHxeuaVE6Y6+xZOtAw+9hvq7/0ejkg==
vscode-textmate@6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-6.0.0.tgz#a3777197235036814ac9a92451492f2748589210"
integrity sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ==
vscode-textmate@7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-7.0.0.tgz#6c90a591c1291e80c6783bddf43e841ad6f8ccc3"
integrity sha512-27bFXGG5JtLrq/reWnkCxkfNq7gnFko7xORba2Gznokx2LmPpHfu880a44B1jPyoFiGNjprZ883qiYTvR/P+pA==
vscode-windows-ca-certs@^0.3.0:
version "0.3.0"

29
src/vs/editor/common/languages.ts
View file

@ -75,10 +75,11 @@ export const enum StandardTokenType {
* 1098 7654 3210 9876 5432 1098 7654 3210
* - -------------------------------------------
* xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx
* bbbb bbbb bfff ffff ffFF FFTT LLLL LLLL
* bbbb bbbb ffff ffff fFFF FBTT LLLL LLLL
* - -------------------------------------------
* - L = LanguageId (8 bits)
* - T = StandardTokenType (2 bits)
* - B = Balanced bracket (1 bit)
* - F = FontStyle (4 bits)
* - f = foreground color (9 bits)
* - b = background color (9 bits)
@ -88,14 +89,15 @@ export const enum StandardTokenType {
export const enum MetadataConsts {
LANGUAGEID_MASK = 0b00000000000000000000000011111111,
TOKEN_TYPE_MASK = 0b00000000000000000000001100000000,
FONT_STYLE_MASK = 0b00000000000000000011110000000000,
FOREGROUND_MASK = 0b00000000011111111100000000000000,
BACKGROUND_MASK = 0b11111111100000000000000000000000,
BALANCED_BRACKETS_MASK = 0b00000000000000000000010000000000,
FONT_STYLE_MASK = 0b00000000000000000111100000000000,
FOREGROUND_MASK = 0b00000000111111111000000000000000,
BACKGROUND_MASK = 0b11111111000000000000000000000000,
ITALIC_MASK = 0b00000000000000000000010000000000,
BOLD_MASK = 0b00000000000000000000100000000000,
UNDERLINE_MASK = 0b00000000000000000001000000000000,
STRIKETHROUGH_MASK = 0b00000000000000000010000000000000,
ITALIC_MASK = 0b00000000000000000000100000000000,
BOLD_MASK = 0b00000000000000000001000000000000,
UNDERLINE_MASK = 0b00000000000000000010000000000000,
STRIKETHROUGH_MASK = 0b00000000000000000100000000000000,
// Semantic tokens cannot set the language id, so we can
// use the first 8 bits for control purposes
@ -108,9 +110,10 @@ export const enum MetadataConsts {
LANGUAGEID_OFFSET = 0,
TOKEN_TYPE_OFFSET = 8,
FONT_STYLE_OFFSET = 10,
FOREGROUND_OFFSET = 14,
BACKGROUND_OFFSET = 23
BALANCED_BRACKETS_OFFSET = 10,
FONT_STYLE_OFFSET = 11,
FOREGROUND_OFFSET = 15,
BACKGROUND_OFFSET = 24
}
/**
@ -126,6 +129,10 @@ export class TokenMetadata {
return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET;
}
public static containsBalancedBrackets(metadata: number): boolean {
return (metadata & MetadataConsts.BALANCED_BRACKETS_MASK) !== 0;
}
public static getFontStyle(metadata: number): FontStyle {
return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET;
}

12
src/vs/editor/test/common/core/lineTokens.test.ts
View file

@ -57,7 +57,7 @@ suite('LineTokens', () => {
test('withInserted 1', () => {
const lineTokens = createTestLineTokens();
assert.strictEqual(renderLineTokens(lineTokens), 'Hello (16384)world, (32768)this (49152)is (65536)a (81920)lovely (98304)day(114688)');
assert.strictEqual(renderLineTokens(lineTokens), 'Hello (32768)world, (65536)this (98304)is (131072)a (163840)lovely (196608)day(229376)');
const lineTokens2 = lineTokens.withInserted([
{ offset: 0, text: '1', tokenMetadata: 0, },
@ -65,12 +65,12 @@ suite('LineTokens', () => {
{ offset: 9, text: '3', tokenMetadata: 0, },
]);
assert.strictEqual(renderLineTokens(lineTokens2), '1(0)Hello (16384)2(0)wor(32768)3(0)ld, (32768)this (49152)is (65536)a (81920)lovely (98304)day(114688)');
assert.strictEqual(renderLineTokens(lineTokens2), '1(0)Hello (32768)2(0)wor(65536)3(0)ld, (65536)this (98304)is (131072)a (163840)lovely (196608)day(229376)');
});
test('withInserted (tokens at the same position)', () => {
const lineTokens = createTestLineTokens();
assert.strictEqual(renderLineTokens(lineTokens), 'Hello (16384)world, (32768)this (49152)is (65536)a (81920)lovely (98304)day(114688)');
assert.strictEqual(renderLineTokens(lineTokens), 'Hello (32768)world, (65536)this (98304)is (131072)a (163840)lovely (196608)day(229376)');
const lineTokens2 = lineTokens.withInserted([
{ offset: 0, text: '1', tokenMetadata: 0, },
@ -78,19 +78,19 @@ suite('LineTokens', () => {
{ offset: 0, text: '3', tokenMetadata: 0, },
]);
assert.strictEqual(renderLineTokens(lineTokens2), '1(0)2(0)3(0)Hello (16384)world, (32768)this (49152)is (65536)a (81920)lovely (98304)day(114688)');
assert.strictEqual(renderLineTokens(lineTokens2), '1(0)2(0)3(0)Hello (32768)world, (65536)this (98304)is (131072)a (163840)lovely (196608)day(229376)');
});
test('withInserted (tokens at the end)', () => {
const lineTokens = createTestLineTokens();
assert.strictEqual(renderLineTokens(lineTokens), 'Hello (16384)world, (32768)this (49152)is (65536)a (81920)lovely (98304)day(114688)');
assert.strictEqual(renderLineTokens(lineTokens), 'Hello (32768)world, (65536)this (98304)is (131072)a (163840)lovely (196608)day(229376)');
const lineTokens2 = lineTokens.withInserted([
{ offset: 'Hello world, this is a lovely day'.length - 1, text: '1', tokenMetadata: 0, },
{ offset: 'Hello world, this is a lovely day'.length, text: '2', tokenMetadata: 0, },
]);
assert.strictEqual(renderLineTokens(lineTokens2), 'Hello (16384)world, (32768)this (49152)is (65536)a (81920)lovely (98304)da(114688)1(0)y(114688)2(0)');
assert.strictEqual(renderLineTokens(lineTokens2), 'Hello (32768)world, (65536)this (98304)is (131072)a (163840)lovely (196608)da(229376)1(0)y(229376)2(0)');
});
test('basics', () => {

50
src/vs/editor/test/common/model/tokensStore.test.ts
View file

@ -176,15 +176,15 @@ suite('TokensStore', () => {
const model = createTextModel(' else if ($s = 08) then \'\\b\'');
model.tokenization.setSemanticTokens([
SparseMultilineTokens.create(1, new Uint32Array([
0, 20, 24, 0b0111100000000010000,
0, 25, 27, 0b0111100000000010000,
0, 28, 29, 0b0000100000000010000,
0, 29, 31, 0b1000000000000010000,
0, 32, 33, 0b0000100000000010000,
0, 34, 36, 0b0011000000000010000,
0, 36, 37, 0b0000100000000010000,
0, 38, 42, 0b0111100000000010000,
0, 43, 47, 0b0101100000000010000,
0, 20, 24, 0b01111000000000010000,
0, 25, 27, 0b01111000000000010000,
0, 28, 29, 0b00001000000000010000,
0, 29, 31, 0b10000000000000010000,
0, 32, 33, 0b00001000000000010000,
0, 34, 36, 0b00110000000000010000,
0, 36, 37, 0b00001000000000010000,
0, 38, 42, 0b01111000000000010000,
0, 43, 47, 0b01011000000000010000,
]))
], true);
const lineTokens = model.tokenization.getLineTokens(1);
@ -194,22 +194,22 @@ suite('TokensStore', () => {
}
assert.deepStrictEqual(decodedTokens, [
20, 0b1000000000100000000000001,
24, 0b1000000111100000000000001,
25, 0b1000000000100000000000001,
27, 0b1000000111100000000000001,
28, 0b1000000000100000000000001,
29, 0b1000000000100000000000001,
31, 0b1000001000000000000000001,
32, 0b1000000000100000000000001,
33, 0b1000000000100000000000001,
34, 0b1000000000100000000000001,
36, 0b1000000011000000000000001,
37, 0b1000000000100000000000001,
38, 0b1000000000100000000000001,
42, 0b1000000111100000000000001,
43, 0b1000000000100000000000001,
47, 0b1000000101100000000000001
20, 0b10000000001000000000000001,
24, 0b10000001111000000000000001,
25, 0b10000000001000000000000001,
27, 0b10000001111000000000000001,
28, 0b10000000001000000000000001,
29, 0b10000000001000000000000001,
31, 0b10000010000000000000000001,
32, 0b10000000001000000000000001,
33, 0b10000000001000000000000001,
34, 0b10000000001000000000000001,
36, 0b10000000110000000000000001,
37, 0b10000000001000000000000001,
38, 0b10000000001000000000000001,
42, 0b10000001111000000000000001,
43, 0b10000000001000000000000001,
47, 0b10000001011000000000000001
]);
model.dispose();

8
yarn.lock
View file

@ -11903,10 +11903,10 @@ vscode-regexpp@^3.1.0:
resolved "https://registry.yarnpkg.com/vscode-regexpp/-/vscode-regexpp-3.1.0.tgz#42d059b6fffe99bd42939c0d013f632f0cad823f"
integrity sha512-pqtN65VC1jRLawfluX4Y80MMG0DHJydWhe5ZwMHewZD6sys4LbU6lHwFAHxeuaVE6Y6+xZOtAw+9hvq7/0ejkg==
vscode-textmate@6.0.0:
version "6.0.0"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-6.0.0.tgz#a3777197235036814ac9a92451492f2748589210"
integrity sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ==
vscode-textmate@7.0.0:
version "7.0.0"
resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-7.0.0.tgz#6c90a591c1291e80c6783bddf43e841ad6f8ccc3"
integrity sha512-27bFXGG5JtLrq/reWnkCxkfNq7gnFko7xORba2Gznokx2LmPpHfu880a44B1jPyoFiGNjprZ883qiYTvR/P+pA==
vscode-uri@^3.0.3:
version "3.0.3"