diff --git a/package.json b/package.json index 745be33efdd..a1550d9e186 100644 --- a/package.json +++ b/package.json @@ -83,7 +83,7 @@ "vscode-proxy-agent": "^0.11.0", "vscode-regexpp": "^3.1.0", "vscode-ripgrep": "^1.12.1", - "vscode-textmate": "5.4.1", + "vscode-textmate": "5.5.0", "xterm": "4.16.0-beta.2", "xterm-addon-search": "0.9.0-beta.6", "xterm-addon-serialize": "0.7.0-beta.3", diff --git a/remote/package.json b/remote/package.json index 3d6c6242d1d..09901109015 100644 --- a/remote/package.json +++ b/remote/package.json @@ -23,7 +23,7 @@ "vscode-proxy-agent": "^0.11.0", "vscode-regexpp": "^3.1.0", "vscode-ripgrep": "^1.12.1", - "vscode-textmate": "5.4.1", + "vscode-textmate": "5.5.0", "xterm": "4.16.0-beta.2", "xterm-addon-search": "0.9.0-beta.6", "xterm-addon-serialize": "0.7.0-beta.3", diff --git a/remote/web/package.json b/remote/web/package.json index b282c699239..6093712246c 100644 --- a/remote/web/package.json +++ b/remote/web/package.json @@ -9,7 +9,7 @@ "jschardet": "3.0.0", "tas-client-umd": "0.1.4", "vscode-oniguruma": "1.6.1", - "vscode-textmate": "5.4.1", + "vscode-textmate": "5.5.0", "xterm": "4.16.0-beta.2", "xterm-addon-search": "0.9.0-beta.6", "xterm-addon-unicode11": "0.4.0-beta.1", diff --git a/remote/web/yarn.lock b/remote/web/yarn.lock index 70c4d55ec80..40d4b7c658b 100644 --- a/remote/web/yarn.lock +++ b/remote/web/yarn.lock @@ -108,10 +108,10 @@ vscode-oniguruma@1.6.1: resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.6.1.tgz#2bf4dfcfe3dd2e56eb549a3068c8ee39e6c30ce5" integrity sha512-vc4WhSIaVpgJ0jJIejjYxPvURJavX6QG41vu0mGhqywMkQqulezEqEQ3cO3gc8GvcOpX6ycmKGqRoROEMBNXTQ== -vscode-textmate@5.4.1: - version "5.4.1" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.4.1.tgz#09d566724fc76b60b3ad9791eebf1f0b50f29e5a" - integrity sha512-4CvPHmfuZQaXrcCpathdh6jo7myuR+MU8BvscgQADuponpbqfmu2rwTOtCXhGwwEgStvJF8V4s9FwMKRVLNmKQ== +vscode-textmate@5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.5.0.tgz#d83776562c07d1e3181c2c7f1b3d5f20afcab483" + integrity sha512-jToQkPGMNKn0eyKyitYeINJF0NoD240aYyKPIWJv5W2jfPt++jIRg0OSergubtGhbw6SoefkvBYEpX7TsfoSUQ== xterm-addon-search@0.9.0-beta.6: version "0.9.0-beta.6" diff --git a/remote/yarn.lock b/remote/yarn.lock index 54e55ea0069..a96bdf2daf4 100644 --- a/remote/yarn.lock +++ b/remote/yarn.lock @@ -512,10 +512,10 @@ vscode-ripgrep@^1.12.1: https-proxy-agent "^4.0.0" proxy-from-env "^1.1.0" -vscode-textmate@5.4.1: - version "5.4.1" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.4.1.tgz#09d566724fc76b60b3ad9791eebf1f0b50f29e5a" - integrity sha512-4CvPHmfuZQaXrcCpathdh6jo7myuR+MU8BvscgQADuponpbqfmu2rwTOtCXhGwwEgStvJF8V4s9FwMKRVLNmKQ== +vscode-textmate@5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.5.0.tgz#d83776562c07d1e3181c2c7f1b3d5f20afcab483" + integrity sha512-jToQkPGMNKn0eyKyitYeINJF0NoD240aYyKPIWJv5W2jfPt++jIRg0OSergubtGhbw6SoefkvBYEpX7TsfoSUQ== vscode-windows-ca-certs@^0.3.0: version "0.3.0" diff --git a/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts b/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts index f02cbe27ce6..387516cc874 100644 --- a/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts +++ b/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts @@ -484,7 +484,13 @@ class TMTokenization extends Disposable { } public tokenize2(line: string, state: StackElement): TokenizationResult2 { - let textMateResult = this._grammar.tokenizeLine2(line, state); + const textMateResult = this._grammar.tokenizeLine2(line, state, 500); + + if (textMateResult.stoppedEarly) { + console.warn(`Time limit reached when tokenizing line: ${line.substring(0, 100)}`); + // return the state at the beginning of the line + return new TokenizationResult2(textMateResult.tokens, state); + } if (this._containsEmbeddedLanguages) { let seenLanguages = this._seenLanguages; diff --git a/src/vs/workbench/services/textMate/common/textMateService.ts b/src/vs/workbench/services/textMate/common/textMateService.ts index 278efa9a571..4dcb0e635de 100644 --- a/src/vs/workbench/services/textMate/common/textMateService.ts +++ b/src/vs/workbench/services/textMate/common/textMateService.ts @@ -33,7 +33,7 @@ export interface IGrammar { /** * Tokenize `lineText` using previous line state `prevState`. */ - tokenizeLine(lineText: string, prevState: StackElement | null): ITokenizeLineResult; + tokenizeLine(lineText: string, prevState: StackElement | null, timeLimit?: number): ITokenizeLineResult; /** * Tokenize `lineText` using previous line state `prevState`. * The result contains the tokens in binary format, resolved with the following information: @@ -44,7 +44,7 @@ export interface IGrammar { * - background color * e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET` */ - tokenizeLine2(lineText: string, prevState: StackElement | null): ITokenizeLineResult2; + tokenizeLine2(lineText: string, prevState: StackElement | null, timeLimit?: number): ITokenizeLineResult2; } export interface ITokenizeLineResult { readonly tokens: IToken[]; @@ -52,6 +52,10 @@ export interface ITokenizeLineResult { * The `prevState` to be passed on to the next line tokenization. */ readonly ruleStack: StackElement; + /** + * Did tokenization stop early due to reaching the time limit. + */ + readonly stoppedEarly: boolean; } /** * Helpers to manage the "collapsed" metadata of an entire StackElement stack. @@ -97,6 +101,10 @@ export interface ITokenizeLineResult2 { * The `prevState` to be passed on to the next line tokenization. */ readonly ruleStack: StackElement; + /** + * Did tokenization stop early due to reaching the time limit. + */ + readonly stoppedEarly: boolean; } export interface IToken { startIndex: number; diff --git a/yarn.lock b/yarn.lock index c62f11cfbfa..09e3bab2c2d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10476,10 +10476,10 @@ vscode-telemetry-extractor@^1.9.5: ts-morph "^12.2.0" vscode-ripgrep "^1.12.1" -vscode-textmate@5.4.1: - version "5.4.1" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.4.1.tgz#09d566724fc76b60b3ad9791eebf1f0b50f29e5a" - integrity sha512-4CvPHmfuZQaXrcCpathdh6jo7myuR+MU8BvscgQADuponpbqfmu2rwTOtCXhGwwEgStvJF8V4s9FwMKRVLNmKQ== +vscode-textmate@5.5.0: + version "5.5.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-5.5.0.tgz#d83776562c07d1e3181c2c7f1b3d5f20afcab483" + integrity sha512-jToQkPGMNKn0eyKyitYeINJF0NoD240aYyKPIWJv5W2jfPt++jIRg0OSergubtGhbw6SoefkvBYEpX7TsfoSUQ== vscode-windows-ca-certs@^0.3.0: version "0.3.0"