Adresses PR feedback, adopts most recent textmate version.

.vscode/tasks.json

@@ -262,15 +262,7 @@
 			"type": "shell",
 			"command": "node_modules/.bin/http-server --cors --port 5001 -a 127.0.0.1 -s -c-1",
 			"isBackground": true,
-			"problemMatcher": {
+			"problemMatcher": [],
-				"pattern": {
-					"regexp": ""
-				},
-				"background": {
-					"beginsPattern": ".*",
-					"endsPattern": ".*"
-				}
-			},
 			"dependsOn": [
 				"Core - Build"
 			]

package.json

@@ -86,7 +86,7 @@
     "vscode-policy-watcher": "^1.1.1",
     "vscode-proxy-agent": "^0.12.0",
     "vscode-regexpp": "^3.1.0",
-    "vscode-textmate": "8.0.0",
+    "vscode-textmate": "9.0.0",
     "xterm": "5.2.0-beta.28",
     "xterm-addon-canvas": "0.4.0-beta.7",
     "xterm-addon-search": "0.11.0",

remote/package.json

@@ -23,7 +23,7 @@
     "vscode-oniguruma": "1.7.0",
     "vscode-proxy-agent": "^0.12.0",
     "vscode-regexpp": "^3.1.0",
-    "vscode-textmate": "8.0.0",
+    "vscode-textmate": "9.0.0",
     "xterm": "5.2.0-beta.28",
     "xterm-addon-canvas": "0.4.0-beta.7",
     "xterm-addon-search": "0.11.0",

remote/web/package.json

@@ -10,7 +10,7 @@
 		"jschardet": "3.0.0",
 		"tas-client-umd": "0.1.6",
 		"vscode-oniguruma": "1.7.0",
-		"vscode-textmate": "8.0.0",
+		"vscode-textmate": "9.0.0",
 		"xterm": "5.2.0-beta.28",
 		"xterm-addon-canvas": "0.4.0-beta.7",
 		"xterm-addon-search": "0.11.0",

remote/web/yarn.lock

@@ -63,10 +63,10 @@ vscode-oniguruma@1.7.0:
   resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.7.0.tgz#439bfad8fe71abd7798338d1cd3dc53a8beea94b"
   integrity sha512-L9WMGRfrjOhgHSdOYgCt/yRMsXzLDJSL7BPrOZt73gU0iWO4mpqzqQzOz5srxqTvMBaR0XZTSrVWo4j55Rc6cA==
 
-vscode-textmate@8.0.0:
+vscode-textmate@9.0.0:
-  version "8.0.0"
+  version "9.0.0"
-  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-8.0.0.tgz#2c7a3b1163ef0441097e0b5d6389cd5504b59e5d"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-9.0.0.tgz#313c6c8792b0507aef35aeb81b6b370b37c44d6c"
-  integrity sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==
+  integrity sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg==
 
 xterm-addon-canvas@0.4.0-beta.7:
   version "0.4.0-beta.7"

remote/yarn.lock

@@ -810,10 +810,10 @@ vscode-regexpp@^3.1.0:
   resolved "https://registry.yarnpkg.com/vscode-regexpp/-/vscode-regexpp-3.1.0.tgz#42d059b6fffe99bd42939c0d013f632f0cad823f"
   integrity sha512-pqtN65VC1jRLawfluX4Y80MMG0DHJydWhe5ZwMHewZD6sys4LbU6lHwFAHxeuaVE6Y6+xZOtAw+9hvq7/0ejkg==
 
-vscode-textmate@8.0.0:
+vscode-textmate@9.0.0:
-  version "8.0.0"
+  version "9.0.0"
-  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-8.0.0.tgz#2c7a3b1163ef0441097e0b5d6389cd5504b59e5d"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-9.0.0.tgz#313c6c8792b0507aef35aeb81b6b370b37c44d6c"
-  integrity sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==
+  integrity sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg==
 
 vscode-windows-ca-certs@^0.3.0:
   version "0.3.0"

src/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens.ts

@@ -21,7 +21,7 @@ import { FontStyle, ColorId, StandardTokenType, TokenMetadata } from 'vs/editor/
 import { ILanguageService } from 'vs/editor/common/languages/language';
 import { INotificationService } from 'vs/platform/notification/common/notification';
 import { findMatchingThemeRule } from 'vs/workbench/services/textMate/common/TMHelper';
-import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
+import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import type { IGrammar, IToken, StateStack } from 'vscode-textmate';
 import { IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import { CancellationTokenSource } from 'vs/base/common/cancellation';
@@ -43,7 +43,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib
 	}
 
 	private _editor: ICodeEditor;
-	private _textMateService: ITextMateTokenizationFeature;
+	private _textMateService: ITextMateTokenizationService;
 	private _themeService: IWorkbenchThemeService;
 	private _languageService: ILanguageService;
 	private _notificationService: INotificationService;
@@ -53,7 +53,7 @@ class InspectEditorTokensController extends Disposable implements IEditorContrib
 
 	constructor(
 		editor: ICodeEditor,
-		@ITextMateTokenizationFeature textMateService: ITextMateTokenizationFeature,
+		@ITextMateTokenizationService textMateService: ITextMateTokenizationService,
 		@ILanguageService languageService: ILanguageService,
 		@IWorkbenchThemeService themeService: IWorkbenchThemeService,
 		@INotificationService notificationService: INotificationService,
@@ -187,7 +187,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
 	private readonly _editor: IActiveCodeEditor;
 	private readonly _languageService: ILanguageService;
 	private readonly _themeService: IWorkbenchThemeService;
-	private readonly _textMateService: ITextMateTokenizationFeature;
+	private readonly _textMateService: ITextMateTokenizationService;
 	private readonly _notificationService: INotificationService;
 	private readonly _configurationService: IConfigurationService;
 	private readonly _languageFeaturesService: ILanguageFeaturesService;
@@ -197,7 +197,7 @@ class InspectEditorTokensWidget extends Disposable implements IContentWidget {
 
 	constructor(
 		editor: IActiveCodeEditor,
-		textMateService: ITextMateTokenizationFeature,
+		textMateService: ITextMateTokenizationService,
 		languageService: ILanguageService,
 		themeService: IWorkbenchThemeService,
 		notificationService: INotificationService,

src/vs/workbench/contrib/codeEditor/browser/languageConfigurationExtensionPoint.ts

@@ -14,7 +14,7 @@ import { ILanguageService } from 'vs/editor/common/languages/language';
 import { Extensions, IJSONContributionRegistry } from 'vs/platform/jsonschemas/common/jsonContributionRegistry';
 import { Registry } from 'vs/platform/registry/common/platform';
 import { IExtensionService } from 'vs/workbench/services/extensions/common/extensions';
-import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
+import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import { getParseErrorMessage } from 'vs/base/common/jsonErrorMessages';
 import { IExtensionResourceLoaderService } from 'vs/platform/extensionResourceLoader/common/extensionResourceLoader';
 import { hash } from 'vs/base/common/hash';
@@ -95,7 +95,7 @@ export class LanguageConfigurationFileHandler extends Disposable {
 	private readonly _done = new Map<string, number>();
 
 	constructor(
-		@ITextMateTokenizationFeature textMateService: ITextMateTokenizationFeature,
+		@ITextMateTokenizationService textMateService: ITextMateTokenizationService,
 		@ILanguageService private readonly _languageService: ILanguageService,
 		@IExtensionResourceLoaderService private readonly _extensionResourceLoaderService: IExtensionResourceLoaderService,
 		@IExtensionService private readonly _extensionService: IExtensionService,

src/vs/workbench/contrib/codeEditor/electron-sandbox/startDebugTextMate.ts

@@ -7,7 +7,7 @@ import * as nls from 'vs/nls';
 import { Range } from 'vs/editor/common/core/range';
 import { Action2, registerAction2 } from 'vs/platform/actions/common/actions';
 import { Categories } from 'vs/platform/action/common/actionCommonCategories';
-import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
+import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import { IModelService } from 'vs/editor/common/services/model';
 import { IEditorService } from 'vs/workbench/services/editor/common/editorService';
 import { URI } from 'vs/base/common/uri';
@@ -52,7 +52,7 @@ class StartDebugTextMate extends Action2 {
 	}
 
 	async run(accessor: ServicesAccessor) {
-		const textMateService = accessor.get(ITextMateTokenizationFeature);
+		const textMateService = accessor.get(ITextMateTokenizationService);
 		const modelService = accessor.get(IModelService);
 		const editorService = accessor.get(IEditorService);
 		const codeEditorService = accessor.get(ICodeEditorService);

src/vs/workbench/contrib/themes/browser/themes.test.contribution.ts

@@ -10,7 +10,7 @@ import { IInstantiationService, ServicesAccessor } from 'vs/platform/instantiati
 import { IWorkbenchThemeService, IWorkbenchColorTheme } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import { IEditorService } from 'vs/workbench/services/editor/common/editorService';
 import { EditorResourceAccessor } from 'vs/workbench/common/editor';
-import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
+import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import type { IGrammar, StateStack } from 'vscode-textmate';
 import { TokenizationRegistry } from 'vs/editor/common/languages';
 import { TokenMetadata } from 'vs/editor/common/encodedTokenAttributes';
@@ -92,7 +92,7 @@ class Snapper {
 	constructor(
 		@ILanguageService private readonly languageService: ILanguageService,
 		@IWorkbenchThemeService private readonly themeService: IWorkbenchThemeService,
-		@ITextMateTokenizationFeature private readonly textMateService: ITextMateTokenizationFeature
+		@ITextMateTokenizationService private readonly textMateService: ITextMateTokenizationService
 	) {
 	}
 

src/vs/workbench/services/textMate/browser/arrayOperation.ts

@@ -25,6 +25,13 @@ export class ArrayEdit {
 			array.insert(c.offset, c.newLength);
 		}
 	}
+
+	applyToArray(array: any[]): void {
+		for (let i = this.edits.length - 1; i >= 0; i--) {
+			const c = this.edits[i];
+			array.splice(c.offset, c.length, ...new Array(c.newLength));
+		}
+	}
 }
 
 export class SingleArrayEdit {
@@ -43,6 +50,9 @@ export interface IIndexTransformer {
 	transform(index: number): number | undefined;
 }
 
+/**
+ * Can only be called with increasing values of `index`.
+*/
 export class MonotonousIndexTransformer implements IIndexTransformer {
 	public static fromMany(transformations: ArrayEdit[]): IIndexTransformer {
 		// TODO improve performance by combining transformations first

src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.contribution.ts

@@ -4,7 +4,7 @@
  *--------------------------------------------------------------------------------------------*/
 
 import { registerSingleton, InstantiationType } from 'vs/platform/instantiation/common/extensions';
-import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
+import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import { TextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl';
 
-registerSingleton(ITextMateTokenizationFeature, TextMateTokenizationFeature, InstantiationType.Eager);
+registerSingleton(ITextMateTokenizationService, TextMateTokenizationFeature, InstantiationType.Eager);

src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.ts

@@ -7,9 +7,9 @@ import { Event } from 'vs/base/common/event';
 import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
 import type { IGrammar } from 'vscode-textmate';
 
-export const ITextMateTokenizationFeature = createDecorator<ITextMateTokenizationFeature>('textMateTokenizationFeature');
+export const ITextMateTokenizationService = createDecorator<ITextMateTokenizationService>('textMateTokenizationFeature');
 
-export interface ITextMateTokenizationFeature {
+export interface ITextMateTokenizationService {
 	readonly _serviceBrand: undefined;
 
 	onDidEncounterLanguage: Event<string>;

src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.ts

@@ -27,7 +27,7 @@ import { INotificationService } from 'vs/platform/notification/common/notificati
 import { IProgressService, ProgressLocation } from 'vs/platform/progress/common/progress';
 import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
 import { ExtensionMessageCollector, IExtensionPointUser } from 'vs/workbench/services/extensions/common/extensionsRegistry';
-import { ITextMateTokenizationFeature } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
+import { ITextMateTokenizationService } from 'vs/workbench/services/textMate/browser/textMateTokenizationFeature';
 import { TextMateTokenizationSupport } from 'vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport';
 import { TokenizationSupportWithLineLimit } from 'vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit';
 import { TextMateWorkerHost } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost';
@@ -37,7 +37,7 @@ import { IValidEmbeddedLanguagesMap, IValidGrammarDefinition, IValidTokenTypeMap
 import { ITextMateThemingRule, IWorkbenchColorTheme, IWorkbenchThemeService } from 'vs/workbench/services/themes/common/workbenchThemeService';
 import type { IGrammar, IOnigLib, IRawTheme } from 'vscode-textmate';
 
-export class TextMateTokenizationFeature extends Disposable implements ITextMateTokenizationFeature {
+export class TextMateTokenizationFeature extends Disposable implements ITextMateTokenizationService {
 	public _serviceBrand: undefined;
 
 	private readonly _onDidEncounterLanguage: Emitter<string> = this._register(new Emitter<string>());

src/vs/workbench/services/textMate/browser/worker/textMateWorkerModel.ts

@@ -49,7 +49,8 @@ export class TextMateWorkerModel extends MirrorTextModel {
 	override onEvents(e: IModelChangedEvent): void {
 		super.onEvents(e);
 		if (this._tokenizationStateStore) {
-			for (let i = e.changes.length - 1; i >= 0; i--) {
+			// Changes are sorted in descending order
+			for (let i = 0; i < e.changes.length; i++) {
 				const change = e.changes[i];
 				const [eolCount] = countEOL(change.text);
 				this._tokenizationStateStore.applyEdits(change.range, eolCount);
@@ -88,7 +89,7 @@ export class TextMateWorkerModel extends MirrorTextModel {
 	}
 
 	private _tokenize(): void {
-		if (!this._tokenizationStateStore) {
+		if (this._isDisposed || !this._tokenizationStateStore) {
 			return;
 		}
 
@@ -123,6 +124,12 @@ export class TextMateWorkerModel extends MirrorTextModel {
 
 				LineTokens.convertToEndOffset(tokenizeResult.tokens, text.length);
 				builder.add(lineIndex + 1, tokenizeResult.tokens);
+
+				const deltaMs = new Date().getTime() - startTime;
+				if (deltaMs > 20) {
+					// yield to check for changes
+					break;
+				}
 			}
 
 			if (tokenizedLines === 0) {
@@ -136,7 +143,7 @@ export class TextMateWorkerModel extends MirrorTextModel {
 			if (deltaMs > 20) {
 				// yield to check for changes
 				setTimeout(() => this._tokenize(), 3);
-				break;
+				return;
 			}
 		}
 	}

src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerHost.ts

@@ -6,6 +6,7 @@
 import { BugIndicatingError } from 'vs/base/common/errors';
 import { DisposableStore, IDisposable, toDisposable } from 'vs/base/common/lifecycle';
 import { AppResourcePath, FileAccess, nodeModulesAsarPath, nodeModulesPath } from 'vs/base/common/network';
+import { isWeb } from 'vs/base/common/platform';
 import { URI, UriComponents } from 'vs/base/common/uri';
 import { createWebWorker, MonacoWebWorker } from 'vs/editor/browser/services/webWorker';
 import { IBackgroundTokenizationStore, IBackgroundTokenizer } from 'vs/editor/common/languages';
@@ -14,6 +15,7 @@ import { ILanguageConfigurationService } from 'vs/editor/common/languages/langua
 import { ITextModel } from 'vs/editor/common/model';
 import { IModelService } from 'vs/editor/common/services/model';
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
+import { IEnvironmentService } from 'vs/platform/environment/common/environment';
 import { IExtensionResourceLoaderService } from 'vs/platform/extensionResourceLoader/common/extensionResourceLoader';
 import { ICreateData, TextMateTokenizationWorker } from 'vs/workbench/services/textMate/browser/worker/textMate.worker';
 import { TextMateWorkerTokenizerController } from 'vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController';
@@ -36,6 +38,7 @@ export class TextMateWorkerHost implements IDisposable {
 		@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,
 		@IConfigurationService private readonly _configurationService: IConfigurationService,
 		@ILanguageService private readonly _languageService: ILanguageService,
+		@IEnvironmentService private readonly _environmentService: IEnvironmentService,
 	) {
 	}
 
@@ -69,8 +72,9 @@ export class TextMateWorkerHost implements IDisposable {
 		const onigurumaModuleLocation: AppResourcePath = `${nodeModulesPath}/vscode-oniguruma`;
 		const onigurumaModuleLocationAsar: AppResourcePath = `${nodeModulesAsarPath}/vscode-oniguruma`;
 
-		const textmateLocation: AppResourcePath = true ? textmateModuleLocation : textmateModuleLocationAsar;
+		const useAsar = this._environmentService.isBuilt && !isWeb;
-		const onigurumaLocation: AppResourcePath = true ? onigurumaModuleLocation : onigurumaModuleLocationAsar;
+		const textmateLocation: AppResourcePath = useAsar ? textmateModuleLocationAsar : textmateModuleLocation;
+		const onigurumaLocation: AppResourcePath = useAsar ? onigurumaModuleLocationAsar : onigurumaModuleLocation;
 		const textmateMain: AppResourcePath = `${textmateLocation}/release/main.js`;
 		const onigurumaMain: AppResourcePath = `${onigurumaLocation}/release/main.js`;
 		const onigurumaWASM: AppResourcePath = `${onigurumaLocation}/release/onig.wasm`;

src/vs/workbench/services/textMate/browser/workerHost/textMateWorkerTokenizerController.ts

@@ -70,9 +70,9 @@ export class TextMateWorkerTokenizerController extends Disposable {
 	 * This method is called from the worker through the worker host.
 	 */
 	public setTokensAndStates(versionId: number, rawTokens: ArrayBuffer, stateDeltas: StateDeltas[]): void {
-		// _states state, change{k}, ..., change{versionId}, state delta base, change{j}, ..., change{m}, current renderer state
+		// _states state, change{k}, ..., change{versionId}, state delta base & rawTokens, change{j}, ..., change{m}, current renderer state
-		//                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^                    ^^^^^^^^^^^^^^^^^^^^^^^^^
+		//                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^                                ^^^^^^^^^^^^^^^^^^^^^^^^^
-		//                | past changes                                       | future states
+		//                | past changes                                                   | future states
 
 		// Apply past changes to _states
 		while (
@@ -84,37 +84,53 @@ export class TextMateWorkerTokenizerController extends Disposable {
 			op.applyTo(this._states);
 		}
 
-		const curToFutureTransformer1 = MonotonousIndexTransformer.fromMany(
+		let tokens = ContiguousMultilineTokensBuilder.deserialize(
-			this._pendingChanges.map((c) => lineArrayEditFromModelContentChange(c.changes))
-		);
-
-		const tokens = ContiguousMultilineTokensBuilder.deserialize(
 			new Uint8Array(rawTokens)
 		);
 
-		// Apply future changes to tokens
+		if (this._pendingChanges.length > 0) {
-		for (const change of this._pendingChanges) {
+			const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(
-			for (const innerChanges of change.changes) {
+				this._pendingChanges.map((c) => new ArrayEdit(
-				for (let j = 0; j < tokens.length; j++) {
+					c.changes.map(
-					tokens[j].applyEdit(innerChanges.range, innerChanges.text);
+						(c) =>
+							new SingleArrayEdit(
+								c.range.startLineNumber - 1,
+								// Expand the edit range to include the entire line
+								(c.range.endLineNumber - c.range.startLineNumber) + 1,
+								countEOL(c.text)[0] + 1
+							)
+					)
+				))
+			);
+
+			// Filter tokens in lines that got changed in the future to prevent flickering
+			// These tokens are recomputed anyway.
+			const b = new ContiguousMultilineTokensBuilder();
+			for (const t of tokens) {
+				for (let i = t.startLineNumber; i <= t.endLineNumber; i++) {
+					const result = curToFutureTransformerTokens.transform(i - 1);
+					// If result is undefined, the current line got touched by an edit.
+					// The webworker will send us new tokens for all the new/touched lines after it received the edits.
+					if (result !== undefined) {
+						b.add(i, t.getLineTokens(i) as Uint32Array);
+					}
+				}
+			}
+			tokens = b.finalize();
+
+			// Apply future changes to tokens
+			for (const change of this._pendingChanges) {
+				for (const innerChanges of change.changes) {
+					for (let j = 0; j < tokens.length; j++) {
+						tokens[j].applyEdit(innerChanges.range, innerChanges.text);
+					}
 				}
 			}
 		}
 
-		// Filter tokens in lines that got changed in the future to prevent flickering
+		this._backgroundTokenizationStore.setTokens(tokens);
-		// These tokens are recomputed anyway.
-		const b = new ContiguousMultilineTokensBuilder();
-		for (const t of tokens) {
-			for (let i = t.startLineNumber; i <= t.endLineNumber; i++) {
-				const result = curToFutureTransformer1.transform(i - 1);
-				if (result !== undefined) {
-					b.add(i, t.getLineTokens(i) as Uint32Array);
-				}
-			}
-		}
-		this._backgroundTokenizationStore.setTokens(b.finalize());
 
-		const curToFutureTransformer = MonotonousIndexTransformer.fromMany(
+		const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(
 			this._pendingChanges.map((c) => lineArrayEditFromModelContentChange(c.changes))
 		);
 
@@ -126,7 +142,7 @@ export class TextMateWorkerTokenizerController extends Disposable {
 				const state = applyStateStackDiff(prevState, delta)!;
 				this._states.set(d.startLineNumber + i - 1, state);
 
-				const offset = curToFutureTransformer.transform(d.startLineNumber + i - 1);
+				const offset = curToFutureTransformerStates.transform(d.startLineNumber + i - 1);
 				if (offset !== undefined) {
 					this._backgroundTokenizationStore.setEndState(offset + 1, state);
 				}

src/vs/workbench/test/browser/arrayOperation.test.ts

@@ -0,0 +1,77 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert = require('assert');
+import { ArrayEdit, MonotonousIndexTransformer, SingleArrayEdit } from 'vs/workbench/services/textMate/browser/arrayOperation';
+
+suite('array operation', () => {
+	function seq(start: number, end: number) {
+		const result: number[] = [];
+		for (let i = start; i < end; i++) {
+			result.push(i);
+		}
+		return result;
+	}
+
+	test('simple', () => {
+		const edit = new ArrayEdit([
+			new SingleArrayEdit(4, 3, 2),
+			new SingleArrayEdit(8, 0, 2),
+			new SingleArrayEdit(9, 2, 0),
+		]);
+
+		const arr = seq(0, 15).map(x => `item${x}`);
+		const newArr = arr.slice();
+
+		edit.applyToArray(newArr);
+		assert.deepStrictEqual(newArr, [
+			'item0',
+			'item1',
+			'item2',
+			'item3',
+			undefined,
+			undefined,
+			'item7',
+			undefined,
+			undefined,
+			'item8',
+			'item11',
+			'item12',
+			'item13',
+			'item14',
+		]);
+
+		const transformer = new MonotonousIndexTransformer(edit);
+		assert.deepStrictEqual(
+			seq(0, 15).map((x) => {
+				const t = transformer.transform(x);
+				let r = `arr[${x}]: ${arr[x]} -> `;
+				if (t !== undefined) {
+					r += `newArr[${t}]: ${newArr[t]}`;
+				} else {
+					r += 'undefined';
+				}
+				return r;
+			}),
+			[
+				'arr[0]: item0 -> newArr[0]: item0',
+				'arr[1]: item1 -> newArr[1]: item1',
+				'arr[2]: item2 -> newArr[2]: item2',
+				'arr[3]: item3 -> newArr[3]: item3',
+				'arr[4]: item4 -> undefined',
+				'arr[5]: item5 -> undefined',
+				'arr[6]: item6 -> undefined',
+				'arr[7]: item7 -> newArr[6]: item7',
+				'arr[8]: item8 -> newArr[9]: item8',
+				'arr[9]: item9 -> undefined',
+				'arr[10]: item10 -> undefined',
+				'arr[11]: item11 -> newArr[10]: item11',
+				'arr[12]: item12 -> newArr[11]: item12',
+				'arr[13]: item13 -> newArr[12]: item13',
+				'arr[14]: item14 -> newArr[13]: item14',
+			]
+		);
+	});
+});

yarn.lock

@@ -11452,10 +11452,10 @@ vscode-regexpp@^3.1.0:
   resolved "https://registry.yarnpkg.com/vscode-regexpp/-/vscode-regexpp-3.1.0.tgz#42d059b6fffe99bd42939c0d013f632f0cad823f"
   integrity sha512-pqtN65VC1jRLawfluX4Y80MMG0DHJydWhe5ZwMHewZD6sys4LbU6lHwFAHxeuaVE6Y6+xZOtAw+9hvq7/0ejkg==
 
-vscode-textmate@8.0.0:
+vscode-textmate@9.0.0:
-  version "8.0.0"
+  version "9.0.0"
-  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-8.0.0.tgz#2c7a3b1163ef0441097e0b5d6389cd5504b59e5d"
+  resolved "https://registry.yarnpkg.com/vscode-textmate/-/vscode-textmate-9.0.0.tgz#313c6c8792b0507aef35aeb81b6b370b37c44d6c"
-  integrity sha512-AFbieoL7a5LMqcnOF04ji+rpXadgOXnZsxQr//r83kLPr7biP7am3g9zbaZIaBGwBRWeSvoMD4mgPdX3e4NWBg==
+  integrity sha512-Cl65diFGxz7gpwbav10HqiY/eVYTO1sjQpmRmV991Bj7wAoOAjGQ97PpQcXorDE2Uc4hnGWLY17xme+5t6MlSg==
 
 vscode-uri@^3.0.7:
   version "3.0.7"