diff --git a/.vscode/tasks.json b/.vscode/tasks.json index d690168ddb2..32254a98a08 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -262,15 +262,7 @@ "type": "shell", "command": "node_modules/.bin/http-server --cors --port 5001 -a 127.0.0.1 -s -c-1", "isBackground": true, - "problemMatcher": { - "pattern": { - "regexp": "" - }, - "background": { - "beginsPattern": ".*", - "endsPattern": ".*" - } - }, + "problemMatcher": [], "dependsOn": [ "Core - Build" ] diff --git a/package.json b/package.json index d0f11fef289..42ae1ad7505 100644 --- a/package.json +++ b/package.json @@ -86,7 +86,7 @@ "vscode-policy-watcher": "^1.1.1", "vscode-proxy-agent": "^0.12.0", "vscode-regexpp": "^3.1.0", - "vscode-textmate": "8.0.0", + "vscode-textmate": "9.0.0", "xterm": "5.2.0-beta.28", "xterm-addon-canvas": "0.4.0-beta.7", "xterm-addon-search": "0.11.0", diff --git a/remote/package.json b/remote/package.json index 1eec4929126..82a66fe847d 100644 --- a/remote/package.json +++ b/remote/package.json @@ -23,7 +23,7 @@ "vscode-oniguruma": "1.7.0", "vscode-proxy-agent": "^0.12.0", "vscode-regexpp": "^3.1.0", - "vscode-textmate": "8.0.0", + "vscode-textmate": "9.0.0", "xterm": "5.2.0-beta.28", "xterm-addon-canvas": "0.4.0-beta.7", "xterm-addon-search": "0.11.0", diff --git a/remote/web/package.json b/remote/web/package.json index ec717aee479..0604640c76f 100644 --- a/remote/web/package.json +++ b/remote/web/package.json @@ -10,7 +10,7 @@ "jschardet": "3.0.0", "tas-client-umd": "0.1.6", "vscode-oniguruma": "1.7.0", - "vscode-textmate": "8.0.0", + "vscode-textmate": "9.0.0", "xterm": "5.2.0-beta.28", "xterm-addon-canvas": "0.4.0-beta.7", "xterm-addon-search": "0.11.0", diff --git a/remote/web/yarn.lock b/remote/web/yarn.lock index c5118aaad4b..ae0a4d7eb1d 100644 --- a/remote/web/yarn.lock +++ b/remote/web/yarn.lock @@ -63,10 +63,10 @@ vscode-oniguruma@1.7.0: resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz#439bfad8fe71abd7798338d1cd3dc53a8beea94b" integrity sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA== -vscode-textmate@8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-8.0.0.tgz#2c7a3b1163ef0441097e0b5d6389cd5504b59e5d" - integrity sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg== +vscode-textmate@9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-9.0.0.tgz#313c6c8792b0507aef35aeb81b6b370b37c44d6c" + integrity sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg== xterm-addon-canvas@0.4.0-beta.7: version "0.4.0-beta.7" diff --git a/remote/yarn.lock b/remote/yarn.lock index 29cdd7e8028..080f46a10a4 100644 --- a/remote/yarn.lock +++ b/remote/yarn.lock @@ -810,10 +810,10 @@ vscode-regexpp@^3.1.0: resolved "https://registry.yarnpkg.com/vscode-regexpp/-/vscode-regexpp-3.1.0.tgz#42d059b6fffe99bd42939c0d013f632f0cad823f" integrity sha512-pqtN65VC1jRLawfluX4Y80MMG0DHJydWhe5ZwMHewZD6sys4LbU6lHwFAHxeuaVE6Y6+xZOtAw+9hvq7/0ejkg== -vscode-textmate@8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-8.0.0.tgz#2c7a3b1163ef0441097e0b5d6389cd5504b59e5d" - integrity sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg== +vscode-textmate@9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-9.0.0.tgz#313c6c8792b0507aef35aeb81b6b370b37c44d6c" + integrity sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg== vscode-windows-ca-certs@^0.3.0: version "0.3.0" diff --git a/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts b/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts index c5766deac9f..7a9e3970e89 100644 --- a/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts +++ b/src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts @@ -21,7 +21,7 @@ import { FontStyle, ColorId, StandardTokenType, TokenMetadata } from 'vs/editor/ import { ILanguageService } from 'vs/editor/common/languages/language'; import { INotificationService } from 'vs/platform/notification/common/notification'; import { findMatchingThemeRule } from 'vs/workbench/services/textMate/common/TMHelper'; -import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; +import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; import type { IGrammar, IToken, StateStack } from 'vscode-textmate'; import { IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService'; import { CancellationTokenSource } from 'vs/base/common/cancellation'; @@ -43,7 +43,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib } private _editor: ICodeEditor; - private _textMateService: ITextMateTokenizationFeature; + private _textMateService: ITextMateTokenizationService; private _themeService: IWorkbenchThemeService; private _languageService: ILanguageService; private _notificationService: INotificationService; @@ -53,7 +53,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib constructor( editor: ICodeEditor, - @ITextMateTokenizationFeature textMateService: ITextMateTokenizationFeature, + @ITextMateTokenizationService textMateService: ITextMateTokenizationService, @ILanguageService languageService: ILanguageService, @IWorkbenchThemeService themeService: IWorkbenchThemeService, @INotificationService notificationService: INotificationService, @@ -187,7 +187,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { private readonly _editor: IActiveCodeEditor; private readonly _languageService: ILanguageService; private readonly _themeService: IWorkbenchThemeService; - private readonly _textMateService: ITextMateTokenizationFeature; + private readonly _textMateService: ITextMateTokenizationService; private readonly _notificationService: INotificationService; private readonly _configurationService: IConfigurationService; private readonly _languageFeaturesService: ILanguageFeaturesService; @@ -197,7 +197,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget { constructor( editor: IActiveCodeEditor, - textMateService: ITextMateTokenizationFeature, + textMateService: ITextMateTokenizationService, languageService: ILanguageService, themeService: IWorkbenchThemeService, notificationService: INotificationService, diff --git a/src/vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint.ts b/src/vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint.ts index 18f69dcb900..2b49448c7b9 100644 --- a/src/vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint.ts +++ b/src/vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint.ts @@ -14,7 +14,7 @@ import { ILanguageService } from 'vs/editor/common/languages/language'; import { Extensions, IJSONContributionRegistry } from 'vs/platform/jsonschemas/common/jsonContributionRegistry'; import { Registry } from 'vs/platform/registry/common/platform'; import { IExtensionService } from 'vs/workbench/services/extensions/common/extensions'; -import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; +import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; import { getParseErrorMessage } from 'vs/base/common/jsonErrorMessages'; import { IExtensionResourceLoaderService } from 'vs/platform/extensionResourceLoader/common/extensionResourceLoader'; import { hash } from 'vs/base/common/hash'; @@ -95,7 +95,7 @@ export class LanguageConfigurationFileHandler extends Disposable { private readonly _done = new Map(); constructor( - @ITextMateTokenizationFeature textMateService: ITextMateTokenizationFeature, + @ITextMateTokenizationService textMateService: ITextMateTokenizationService, @ILanguageService private readonly _languageService: ILanguageService, @IExtensionResourceLoaderService private readonly _extensionResourceLoaderService: IExtensionResourceLoaderService, @IExtensionService private readonly _extensionService: IExtensionService, diff --git a/src/vs/workbench/contrib/codeEditor/electron-sandbox/startDebugTextMate.ts b/src/vs/workbench/contrib/codeEditor/electron-sandbox/startDebugTextMate.ts index e4d6a0b5a53..b73832636f7 100644 --- a/src/vs/workbench/contrib/codeEditor/electron-sandbox/startDebugTextMate.ts +++ b/src/vs/workbench/contrib/codeEditor/electron-sandbox/startDebugTextMate.ts @@ -7,7 +7,7 @@ import * as nls from 'vs/nls'; import { Range } from 'vs/editor/common/core/range'; import { Action2, registerAction2 } from 'vs/platform/actions/common/actions'; import { Categories } from 'vs/platform/action/common/actionCommonCategories'; -import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; +import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; import { IModelService } from 'vs/editor/common/services/model'; import { IEditorService } from 'vs/workbench/services/editor/common/editorService'; import { URI } from 'vs/base/common/uri'; @@ -52,7 +52,7 @@ class StartDebugTextMate extends Action2 { } async run(accessor: ServicesAccessor) { - const textMateService = accessor.get(ITextMateTokenizationFeature); + const textMateService = accessor.get(ITextMateTokenizationService); const modelService = accessor.get(IModelService); const editorService = accessor.get(IEditorService); const codeEditorService = accessor.get(ICodeEditorService); diff --git a/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts b/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts index 3009f8be61a..6916d87256c 100644 --- a/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts +++ b/src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts @@ -10,7 +10,7 @@ import { IInstantiationService, ServicesAccessor } from 'vs/platform/instantiati import { IWorkbenchThemeService, IWorkbenchColorTheme } from 'vs/workbench/services/themes/common/workbenchThemeService'; import { IEditorService } from 'vs/workbench/services/editor/common/editorService'; import { EditorResourceAccessor } from 'vs/workbench/common/editor'; -import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; +import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; import type { IGrammar, StateStack } from 'vscode-textmate'; import { TokenizationRegistry } from 'vs/editor/common/languages'; import { TokenMetadata } from 'vs/editor/common/encodedTokenAttributes'; @@ -92,7 +92,7 @@ class Snapper { constructor( @ILanguageService private readonly languageService: ILanguageService, @IWorkbenchThemeService private readonly themeService: IWorkbenchThemeService, - @ITextMateTokenizationFeature private readonly textMateService: ITextMateTokenizationFeature + @ITextMateTokenizationService private readonly textMateService: ITextMateTokenizationService ) { } diff --git a/src/vs/workbench/services/textMate/browser/arrayOperation.ts b/src/vs/workbench/services/textMate/browser/arrayOperation.ts index edb1a3d6b50..acdc653f5dd 100644 --- a/src/vs/workbench/services/textMate/browser/arrayOperation.ts +++ b/src/vs/workbench/services/textMate/browser/arrayOperation.ts @@ -25,6 +25,13 @@ export class ArrayEdit { array.insert(c.offset, c.newLength); } } + + applyToArray(array: any[]): void { + for (let i = this.edits.length - 1; i >= 0; i--) { + const c = this.edits[i]; + array.splice(c.offset, c.length, ...new Array(c.newLength)); + } + } } export class SingleArrayEdit { @@ -43,6 +50,9 @@ export interface IIndexTransformer { transform(index: number): number | undefined; } +/** + * Can only be called with increasing values of `index`. +*/ export class MonotonousIndexTransformer implements IIndexTransformer { public static fromMany(transformations: ArrayEdit[]): IIndexTransformer { // TODO improve performance by combining transformations first diff --git a/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.contribution.ts b/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.contribution.ts index ad6b1f99c00..f6090e4572f 100644 --- a/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.contribution.ts +++ b/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.contribution.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { registerSingleton, InstantiationType } from 'vs/platform/instantiation/common/extensions'; -import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; +import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; import { TextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl'; -registerSingleton(ITextMateTokenizationFeature, TextMateTokenizationFeature, InstantiationType.Eager); +registerSingleton(ITextMateTokenizationService, TextMateTokenizationFeature, InstantiationType.Eager); diff --git a/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.ts b/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.ts index db78edeb7c5..84c3a9023c3 100644 --- a/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.ts +++ b/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.ts @@ -7,9 +7,9 @@ import { Event } from 'vs/base/common/event'; import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; import type { IGrammar } from 'vscode-textmate'; -export const ITextMateTokenizationFeature = createDecorator('textMateTokenizationFeature'); +export const ITextMateTokenizationService = createDecorator('textMateTokenizationFeature'); -export interface ITextMateTokenizationFeature { +export interface ITextMateTokenizationService { readonly _serviceBrand: undefined; onDidEncounterLanguage: Event; diff --git a/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts b/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts index ac66eb7123d..57d0dd3a125 100644 --- a/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts +++ b/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts @@ -27,7 +27,7 @@ import { INotificationService } from 'vs/platform/notification/common/notificati import { IProgressService, ProgressLocation } from 'vs/platform/progress/common/progress'; import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService'; import { ExtensionMessageCollector, IExtensionPointUser } from 'vs/workbench/services/extensions/common/extensionsRegistry'; -import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; +import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature'; import { TextMateTokenizationSupport } from 'vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport'; import { TokenizationSupportWithLineLimit } from 'vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit'; import { TextMateWorkerHost } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost'; @@ -37,7 +37,7 @@ import { IValidEmbeddedLanguagesMap, IValidGrammarDefinition, IValidTokenTypeMap import { ITextMateThemingRule, IWorkbenchColorTheme, IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService'; import type { IGrammar, IOnigLib, IRawTheme } from 'vscode-textmate'; -export class TextMateTokenizationFeature extends Disposable implements ITextMateTokenizationFeature { +export class TextMateTokenizationFeature extends Disposable implements ITextMateTokenizationService { public _serviceBrand: undefined; private readonly _onDidEncounterLanguage: Emitter = this._register(new Emitter()); diff --git a/src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts b/src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts index 47c8d9c2832..b4b1a6ca147 100644 --- a/src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts +++ b/src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts @@ -49,7 +49,8 @@ export class TextMateWorkerModel extends MirrorTextModel { override onEvents(e: IModelChangedEvent): void { super.onEvents(e); if (this._tokenizationStateStore) { - for (let i = e.changes.length - 1; i >= 0; i--) { + // Changes are sorted in descending order + for (let i = 0; i < e.changes.length; i++) { const change = e.changes[i]; const [eolCount] = countEOL(change.text); this._tokenizationStateStore.applyEdits(change.range, eolCount); @@ -88,7 +89,7 @@ export class TextMateWorkerModel extends MirrorTextModel { } private _tokenize(): void { - if (!this._tokenizationStateStore) { + if (this._isDisposed || !this._tokenizationStateStore) { return; } @@ -123,6 +124,12 @@ export class TextMateWorkerModel extends MirrorTextModel { LineTokens.convertToEndOffset(tokenizeResult.tokens, text.length); builder.add(lineIndex + 1, tokenizeResult.tokens); + + const deltaMs = new Date().getTime() - startTime; + if (deltaMs > 20) { + // yield to check for changes + break; + } } if (tokenizedLines === 0) { @@ -136,7 +143,7 @@ export class TextMateWorkerModel extends MirrorTextModel { if (deltaMs > 20) { // yield to check for changes setTimeout(() => this._tokenize(), 3); - break; + return; } } } diff --git a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts index d8802645142..16547b5314f 100644 --- a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts +++ b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts @@ -6,6 +6,7 @@ import { BugIndicatingError } from 'vs/base/common/errors'; import { DisposableStore, IDisposable, toDisposable } from 'vs/base/common/lifecycle'; import { AppResourcePath, FileAccess, nodeModulesAsarPath, nodeModulesPath } from 'vs/base/common/network'; +import { isWeb } from 'vs/base/common/platform'; import { URI, UriComponents } from 'vs/base/common/uri'; import { createWebWorker, MonacoWebWorker } from 'vs/editor/browser/services/webWorker'; import { IBackgroundTokenizationStore, IBackgroundTokenizer } from 'vs/editor/common/languages'; @@ -14,6 +15,7 @@ import { ILanguageConfigurationService } from 'vs/editor/common/languages/langua import { ITextModel } from 'vs/editor/common/model'; import { IModelService } from 'vs/editor/common/services/model'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; +import { IEnvironmentService } from 'vs/platform/environment/common/environment'; import { IExtensionResourceLoaderService } from 'vs/platform/extensionResourceLoader/common/extensionResourceLoader'; import { ICreateData, TextMateTokenizationWorker } from 'vs/workbench/services/textMate/browser/worker/textMate.worker'; import { TextMateWorkerTokenizerController } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController'; @@ -36,6 +38,7 @@ export class TextMateWorkerHost implements IDisposable { @ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService, @IConfigurationService private readonly _configurationService: IConfigurationService, @ILanguageService private readonly _languageService: ILanguageService, + @IEnvironmentService private readonly _environmentService: IEnvironmentService, ) { } @@ -69,8 +72,9 @@ export class TextMateWorkerHost implements IDisposable { const onigurumaModuleLocation: AppResourcePath = `${nodeModulesPath}/vscode-oniguruma`; const onigurumaModuleLocationAsar: AppResourcePath = `${nodeModulesAsarPath}/vscode-oniguruma`; - const textmateLocation: AppResourcePath = true ? textmateModuleLocation : textmateModuleLocationAsar; - const onigurumaLocation: AppResourcePath = true ? onigurumaModuleLocation : onigurumaModuleLocationAsar; + const useAsar = this._environmentService.isBuilt && !isWeb; + const textmateLocation: AppResourcePath = useAsar ? textmateModuleLocationAsar : textmateModuleLocation; + const onigurumaLocation: AppResourcePath = useAsar ? onigurumaModuleLocationAsar : onigurumaModuleLocation; const textmateMain: AppResourcePath = `${textmateLocation}/release/main.js`; const onigurumaMain: AppResourcePath = `${onigurumaLocation}/release/main.js`; const onigurumaWASM: AppResourcePath = `${onigurumaLocation}/release/onig.wasm`; diff --git a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts index 0288e1dfd77..2cbd515f969 100644 --- a/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts +++ b/src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts @@ -70,9 +70,9 @@ export class TextMateWorkerTokenizerController extends Disposable { * This method is called from the worker through the worker host. */ public setTokensAndStates(versionId: number, rawTokens: ArrayBuffer, stateDeltas: StateDeltas[]): void { - // _states state, change{k}, ..., change{versionId}, state delta base, change{j}, ..., change{m}, current renderer state - // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^ - // | past changes | future states + // _states state, change{k}, ..., change{versionId}, state delta base & rawTokens, change{j}, ..., change{m}, current renderer state + // ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^ + // | past changes | future states // Apply past changes to _states while ( @@ -84,37 +84,53 @@ export class TextMateWorkerTokenizerController extends Disposable { op.applyTo(this._states); } - const curToFutureTransformer1 = MonotonousIndexTransformer.fromMany( - this._pendingChanges.map((c) => lineArrayEditFromModelContentChange(c.changes)) - ); - - const tokens = ContiguousMultilineTokensBuilder.deserialize( + let tokens = ContiguousMultilineTokensBuilder.deserialize( new Uint8Array(rawTokens) ); - // Apply future changes to tokens - for (const change of this._pendingChanges) { - for (const innerChanges of change.changes) { - for (let j = 0; j < tokens.length; j++) { - tokens[j].applyEdit(innerChanges.range, innerChanges.text); + if (this._pendingChanges.length > 0) { + const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany( + this._pendingChanges.map((c) => new ArrayEdit( + c.changes.map( + (c) => + new SingleArrayEdit( + c.range.startLineNumber - 1, + // Expand the edit range to include the entire line + (c.range.endLineNumber - c.range.startLineNumber) + 1, + countEOL(c.text)[0] + 1 + ) + ) + )) + ); + + // Filter tokens in lines that got changed in the future to prevent flickering + // These tokens are recomputed anyway. + const b = new ContiguousMultilineTokensBuilder(); + for (const t of tokens) { + for (let i = t.startLineNumber; i <= t.endLineNumber; i++) { + const result = curToFutureTransformerTokens.transform(i - 1); + // If result is undefined, the current line got touched by an edit. + // The webworker will send us new tokens for all the new/touched lines after it received the edits. + if (result !== undefined) { + b.add(i, t.getLineTokens(i) as Uint32Array); + } + } + } + tokens = b.finalize(); + + // Apply future changes to tokens + for (const change of this._pendingChanges) { + for (const innerChanges of change.changes) { + for (let j = 0; j < tokens.length; j++) { + tokens[j].applyEdit(innerChanges.range, innerChanges.text); + } } } } - // Filter tokens in lines that got changed in the future to prevent flickering - // These tokens are recomputed anyway. - const b = new ContiguousMultilineTokensBuilder(); - for (const t of tokens) { - for (let i = t.startLineNumber; i <= t.endLineNumber; i++) { - const result = curToFutureTransformer1.transform(i - 1); - if (result !== undefined) { - b.add(i, t.getLineTokens(i) as Uint32Array); - } - } - } - this._backgroundTokenizationStore.setTokens(b.finalize()); + this._backgroundTokenizationStore.setTokens(tokens); - const curToFutureTransformer = MonotonousIndexTransformer.fromMany( + const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany( this._pendingChanges.map((c) => lineArrayEditFromModelContentChange(c.changes)) ); @@ -126,7 +142,7 @@ export class TextMateWorkerTokenizerController extends Disposable { const state = applyStateStackDiff(prevState, delta)!; this._states.set(d.startLineNumber + i - 1, state); - const offset = curToFutureTransformer.transform(d.startLineNumber + i - 1); + const offset = curToFutureTransformerStates.transform(d.startLineNumber + i - 1); if (offset !== undefined) { this._backgroundTokenizationStore.setEndState(offset + 1, state); } diff --git a/src/vs/workbench/test/browser/arrayOperation.test.ts b/src/vs/workbench/test/browser/arrayOperation.test.ts new file mode 100644 index 00000000000..94f075e256f --- /dev/null +++ b/src/vs/workbench/test/browser/arrayOperation.test.ts @@ -0,0 +1,77 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert = require('assert'); +import { ArrayEdit, MonotonousIndexTransformer, SingleArrayEdit } from 'vs/workbench/services/textMate/browser/arrayOperation'; + +suite('array operation', () => { + function seq(start: number, end: number) { + const result: number[] = []; + for (let i = start; i < end; i++) { + result.push(i); + } + return result; + } + + test('simple', () => { + const edit = new ArrayEdit([ + new SingleArrayEdit(4, 3, 2), + new SingleArrayEdit(8, 0, 2), + new SingleArrayEdit(9, 2, 0), + ]); + + const arr = seq(0, 15).map(x => `item${x}`); + const newArr = arr.slice(); + + edit.applyToArray(newArr); + assert.deepStrictEqual(newArr, [ + 'item0', + 'item1', + 'item2', + 'item3', + undefined, + undefined, + 'item7', + undefined, + undefined, + 'item8', + 'item11', + 'item12', + 'item13', + 'item14', + ]); + + const transformer = new MonotonousIndexTransformer(edit); + assert.deepStrictEqual( + seq(0, 15).map((x) => { + const t = transformer.transform(x); + let r = `arr[${x}]: ${arr[x]} -> `; + if (t !== undefined) { + r += `newArr[${t}]: ${newArr[t]}`; + } else { + r += 'undefined'; + } + return r; + }), + [ + 'arr[0]: item0 -> newArr[0]: item0', + 'arr[1]: item1 -> newArr[1]: item1', + 'arr[2]: item2 -> newArr[2]: item2', + 'arr[3]: item3 -> newArr[3]: item3', + 'arr[4]: item4 -> undefined', + 'arr[5]: item5 -> undefined', + 'arr[6]: item6 -> undefined', + 'arr[7]: item7 -> newArr[6]: item7', + 'arr[8]: item8 -> newArr[9]: item8', + 'arr[9]: item9 -> undefined', + 'arr[10]: item10 -> undefined', + 'arr[11]: item11 -> newArr[10]: item11', + 'arr[12]: item12 -> newArr[11]: item12', + 'arr[13]: item13 -> newArr[12]: item13', + 'arr[14]: item14 -> newArr[13]: item14', + ] + ); + }); +}); diff --git a/yarn.lock b/yarn.lock index cdf37e12fb9..d7a63b6d240 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11452,10 +11452,10 @@ vscode-regexpp@^3.1.0: resolved "https://registry.yarnpkg.com/vscode-regexpp/-/vscode-regexpp-3.1.0.tgz#42d059b6fffe99bd42939c0d013f632f0cad823f" integrity sha512-pqtN65VC1jRLawfluX4Y80MMG0DHJydWhe5ZwMHewZD6sys4LbU6lHwFAHxeuaVE6Y6+xZOtAw+9hvq7/0ejkg== -vscode-textmate@8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-8.0.0.tgz#2c7a3b1163ef0441097e0b5d6389cd5504b59e5d" - integrity sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg== +vscode-textmate@9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-9.0.0.tgz#313c6c8792b0507aef35aeb81b6b370b37c44d6c" + integrity sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg== vscode-uri@^3.0.7: version "3.0.7"