Add a mechanism to debug TM grammars

package.json

@@ -55,7 +55,7 @@
     "sudo-prompt": "9.1.1",
     "v8-inspect-profiler": "^0.0.20",
     "vscode-nsfw": "1.2.8",
-    "vscode-oniguruma": "1.1.1",
+    "vscode-oniguruma": "1.3.0",
     "vscode-proxy-agent": "^0.5.2",
     "vscode-ripgrep": "^1.5.8",
     "vscode-sqlite3": "4.0.10",

remote/package.json

@@ -16,7 +16,7 @@
     "semver-umd": "^5.5.6",
     "spdlog": "^0.11.1",
     "vscode-nsfw": "1.2.8",
-    "vscode-oniguruma": "1.1.1",
+    "vscode-oniguruma": "1.3.0",
     "vscode-proxy-agent": "^0.5.2",
     "vscode-ripgrep": "^1.5.8",
     "vscode-textmate": "5.0.2",

remote/web/package.json

@@ -3,7 +3,7 @@
 	"version": "0.0.0",
 	"dependencies": {
 		"semver-umd": "^5.5.6",
-		"vscode-oniguruma": "1.1.1",
+		"vscode-oniguruma": "1.3.0",
 		"vscode-textmate": "5.0.2",
 		"xterm": "4.6.0-beta.25",
 		"xterm-addon-search": "0.6.0",

remote/web/yarn.lock

@@ -7,10 +7,10 @@ semver-umd@^5.5.6:
   resolved "https://registry.yarnpkg.com/semver-umd/-/semver-umd-5.5.6.tgz#1d185bbd2caec825c564b54907cd09e14083f228"
   integrity sha512-6ARYXVi4Y4VO5HfyCjT/6xyykBtJwEXSGQ8ON4UPQSFOjZUDsbAE0J614QcBBsLTTyQMEqvsXN804vAqpydjzw==
 
-vscode-oniguruma@1.1.1:
+vscode-oniguruma@1.3.0:
-  version "1.1.1"
+  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.1.1.tgz#81460d148e70668b7c1abfd78ab7fcb2e2c6b316"
+  resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.3.0.tgz#6788a9db2f8b0781243b4eb8c7a1dd25f6c0e2c8"
-  integrity sha512-4bygPUQjD5o9nLxp8f0s2TbqLte+Kz3C/IZxheIwunos69jI6OOxAhXW06RgqFbnXAw9ggUydx5qIZ7LD6iNuw==
+  integrity sha512-m4Br19v6XD4MRbVrgsLNSZgQrBzk1BCMCleL8+GrcoGxKEJJd62zOFcTaoQR3hCrSlLqoxWmJ7Cc0VieVV3iTQ==
 
 vscode-textmate@5.0.2:
   version "5.0.2"

remote/yarn.lock

@@ -367,10 +367,10 @@ vscode-nsfw@1.2.8:
     lodash.isundefined "^3.0.1"
     nan "^2.10.0"
 
-vscode-oniguruma@1.1.1:
+vscode-oniguruma@1.3.0:
-  version "1.1.1"
+  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.1.1.tgz#81460d148e70668b7c1abfd78ab7fcb2e2c6b316"
+  resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.3.0.tgz#6788a9db2f8b0781243b4eb8c7a1dd25f6c0e2c8"
-  integrity sha512-4bygPUQjD5o9nLxp8f0s2TbqLte+Kz3C/IZxheIwunos69jI6OOxAhXW06RgqFbnXAw9ggUydx5qIZ7LD6iNuw==
+  integrity sha512-m4Br19v6XD4MRbVrgsLNSZgQrBzk1BCMCleL8+GrcoGxKEJJd62zOFcTaoQR3hCrSlLqoxWmJ7Cc0VieVV3iTQ==
 
 vscode-proxy-agent@^0.5.2:
   version "0.5.2"

src/vs/workbench/contrib/codeEditor/electron-browser/codeEditor.contribution.ts

@@ -6,3 +6,4 @@
 import './inputClipboardActions';
 import './sleepResumeRepaintMinimap';
 import './selectionClipboard';
+import './startDebugTextMate';

src/vs/workbench/contrib/codeEditor/electron-browser/startDebugTextMate.ts

@@ -0,0 +1,98 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as os from 'os';
+import * as path from 'path';
+
+import * as nls from 'vs/nls';
+import { Range } from 'vs/editor/common/core/range';
+import { Action } from 'vs/base/common/actions';
+import { SyncActionDescriptor } from 'vs/platform/actions/common/actions';
+import { Registry } from 'vs/platform/registry/common/platform';
+import { Extensions as ActionExtensions, IWorkbenchActionRegistry } from 'vs/workbench/common/actions';
+import { ITextMateService } from 'vs/workbench/services/textMate/common/textMateService';
+import { IModelService } from 'vs/editor/common/services/modelService';
+import { IEditorService } from 'vs/workbench/services/editor/common/editorService';
+import { URI } from 'vs/base/common/uri';
+import { createRotatingLogger } from 'vs/platform/log/node/spdlogService';
+import { generateUuid } from 'vs/base/common/uuid';
+import { ICodeEditorService } from 'vs/editor/browser/services/codeEditorService';
+import { ITextModel } from 'vs/editor/common/model';
+import { Constants } from 'vs/base/common/uint';
+
+class StartDebugTextMate extends Action {
+
+	private static resource = URI.parse(`inmemory:///tm-log.txt`);
+
+	public static readonly ID = 'editor.action.startDebugTextMate';
+	public static readonly LABEL = nls.localize('startDebugTextMate', "Start Text Mate Syntax Grammar Logging");
+
+	constructor(
+		id: string,
+		label: string,
+		@ITextMateService private readonly _textMateService: ITextMateService,
+		@IModelService private readonly _modelService: IModelService,
+		@IEditorService private readonly _editorService: IEditorService,
+		@ICodeEditorService private readonly _codeEditorService: ICodeEditorService
+	) {
+		super(id, label);
+	}
+
+	private _getOrCreateModel(): ITextModel {
+		const model = this._modelService.getModel(StartDebugTextMate.resource);
+		if (model) {
+			return model;
+		}
+		return this._modelService.createModel('', null, StartDebugTextMate.resource);
+	}
+
+	private _append(model: ITextModel, str: string) {
+		const lineCount = model.getLineCount();
+		model.applyEdits([{
+			range: new Range(lineCount, Constants.MAX_SAFE_SMALL_INTEGER, lineCount, Constants.MAX_SAFE_SMALL_INTEGER),
+			text: str
+		}]);
+	}
+
+	public async run(): Promise<any> {
+		const pathInTemp = path.join(os.tmpdir(), `vcode-tm-log-${generateUuid()}.txt`);
+		const logger = createRotatingLogger(`tm-log`, pathInTemp, 1024 * 1024 * 30, 1);
+		const model = this._getOrCreateModel();
+		this._append(model, [
+			`// Open the file you want to test to the side and watch here`,
+			`// Output mirrored at ${pathInTemp}`
+		].join('\n'));
+		const textEditorPane = await this._editorService.openEditor({
+			resource: model.uri
+		});
+		if (!textEditorPane) {
+			return;
+		}
+		const scrollEditor = () => {
+			const editors = this._codeEditorService.listCodeEditors();
+			for (const editor of editors) {
+				if (editor.hasModel()) {
+					if (editor.getModel().uri.toString() === StartDebugTextMate.resource.toString()) {
+						editor.revealLine(editor.getModel().getLineCount());
+					}
+				}
+			}
+		};
+		this._textMateService.startDebugMode(
+			(str) => {
+				this._append(model, str + '\n');
+				scrollEditor();
+				logger.info(str);
+				logger.flush();
+			},
+			() => {
+
+			}
+		);
+	}
+}
+
+const registry = Registry.as<IWorkbenchActionRegistry>(ActionExtensions.WorkbenchActions);
+registry.registerWorkbenchAction(SyncActionDescriptor.from(StartDebugTextMate), 'Start Text Mate Syntax Grammar Logging', nls.localize('developer', "Developer"));

src/vs/workbench/services/textMate/browser/abstractTextMateService.ts

@@ -30,6 +30,7 @@ import { IConfigurationService } from 'vs/platform/configuration/common/configur
 import { IValidGrammarDefinition, IValidEmbeddedLanguagesMap, IValidTokenTypeMap } from 'vs/workbench/services/textMate/common/TMScopeRegistry';
 import { TMGrammarFactory } from 'vs/workbench/services/textMate/common/TMGrammarFactory';
 import { IExtensionResourceLoaderService } from 'vs/workbench/services/extensionResourceLoader/common/extensionResourceLoader';
+import { IProgressService, ProgressLocation } from 'vs/platform/progress/common/progress';
 
 export abstract class AbstractTextMateService extends Disposable implements ITextMateService {
 	public _serviceBrand: undefined;
@@ -41,6 +42,9 @@ export abstract class AbstractTextMateService extends Disposable implements ITex
 	private readonly _createdModes: string[];
 	private readonly _encounteredLanguages: boolean[];
 
+	private _debugMode: boolean;
+	private _debugModePrintFunc: (str: string) => void;
+
 	private _grammarDefinitions: IValidGrammarDefinition[] | null;
 	private _grammarFactory: TMGrammarFactory | null;
 	private _tokenizersRegistrations: IDisposable[];
@@ -54,7 +58,8 @@ export abstract class AbstractTextMateService extends Disposable implements ITex
 		@INotificationService private readonly _notificationService: INotificationService,
 		@ILogService private readonly _logService: ILogService,
 		@IConfigurationService private readonly _configurationService: IConfigurationService,
-		@IStorageService private readonly _storageService: IStorageService
+		@IStorageService private readonly _storageService: IStorageService,
+		@IProgressService private readonly _progressService: IProgressService
 	) {
 		super();
 		this._styleElement = dom.createStyleSheet();
@@ -62,6 +67,9 @@ export abstract class AbstractTextMateService extends Disposable implements ITex
 		this._createdModes = [];
 		this._encounteredLanguages = [];
 
+		this._debugMode = false;
+		this._debugModePrintFunc = () => { };
+
 		this._grammarDefinitions = null;
 		this._grammarFactory = null;
 		this._tokenizersRegistrations = [];
@@ -174,6 +182,46 @@ export abstract class AbstractTextMateService extends Disposable implements ITex
 		});
 	}
 
+	public startDebugMode(printFn: (str: string) => void, onStop: () => void): void {
+		if (this._debugMode) {
+			this._notificationService.error(nls.localize('alreadyDebugging', "Already Logging."));
+			return;
+		}
+
+		this._debugModePrintFunc = printFn;
+		this._debugMode = true;
+
+		if (this._debugMode) {
+			this._progressService.withProgress(
+				{
+					location: ProgressLocation.Notification,
+					buttons: [nls.localize('stop', "Stop")]
+				},
+				(progress) => {
+					progress.report({
+						message: nls.localize('progress1', "Preparing to log TM Grammar parsing. Press Stop when finished.")
+					});
+
+					return this._getVSCodeOniguruma().then((vscodeOniguruma) => {
+						vscodeOniguruma.setDefaultDebugCall(true);
+						progress.report({
+							message: nls.localize('progress2', "Now logging TM Grammar parsing. Press Stop when finished.")
+						});
+						return new Promise<void>((resolve, reject) => { });
+					});
+				},
+				(choice) => {
+					this._getVSCodeOniguruma().then((vscodeOniguruma) => {
+						this._debugModePrintFunc = () => { };
+						this._debugMode = false;
+						vscodeOniguruma.setDefaultDebugCall(false);
+						onStop();
+					});
+				}
+			);
+		}
+	}
+
 	private _canCreateGrammarFactory(): boolean {
 		// Check if extension point is ready
 		return (this._grammarDefinitions ? true : false);
@@ -354,7 +402,13 @@ export abstract class AbstractTextMateService extends Disposable implements ITex
 
 	private async _doGetVSCodeOniguruma(): Promise<typeof import('vscode-oniguruma')> {
 		const [vscodeOniguruma, wasm] = await Promise.all([import('vscode-oniguruma'), this._loadVSCodeOnigurumWASM()]);
-		await vscodeOniguruma.loadWASM(wasm);
+		const options = {
+			data: wasm,
+			print: (str: string) => {
+				this._debugModePrintFunc(str);
+			}
+		};
+		await vscodeOniguruma.loadWASM(options);
 		return vscodeOniguruma;
 	}
 

src/vs/workbench/services/textMate/browser/textMateService.ts

@@ -13,6 +13,7 @@ import { IWorkbenchThemeService } from 'vs/workbench/services/themes/common/work
 import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
 import { IStorageService } from 'vs/platform/storage/common/storage';
 import { IExtensionResourceLoaderService } from 'vs/workbench/services/extensionResourceLoader/common/extensionResourceLoader';
+import { IProgressService } from 'vs/platform/progress/common/progress';
 
 export class TextMateService extends AbstractTextMateService {
 
@@ -23,9 +24,10 @@ export class TextMateService extends AbstractTextMateService {
 		@INotificationService notificationService: INotificationService,
 		@ILogService logService: ILogService,
 		@IConfigurationService configurationService: IConfigurationService,
-		@IStorageService storageService: IStorageService
+		@IStorageService storageService: IStorageService,
+		@IProgressService progressService: IProgressService
 	) {
-		super(modeService, themeService, extensionResourceLoaderService, notificationService, logService, configurationService, storageService);
+		super(modeService, themeService, extensionResourceLoaderService, notificationService, logService, configurationService, storageService, progressService);
 	}
 
 	protected async _loadVSCodeOnigurumWASM(): Promise<Response | ArrayBuffer> {

src/vs/workbench/services/textMate/common/textMateService.ts

@@ -15,6 +15,8 @@ export interface ITextMateService {
 	onDidEncounterLanguage: Event<LanguageId>;
 
 	createGrammar(modeId: string): Promise<IGrammar | null>;
+
+	startDebugMode(printFn: (str: string) => void, onStop: () => void): void;
 }
 
 // -------------- Types "liberated" from vscode-textmate due to usage in /common/

src/vs/workbench/services/textMate/electron-browser/textMateService.ts

@@ -25,6 +25,7 @@ import { IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvent
 import { IStorageService } from 'vs/platform/storage/common/storage';
 import { IExtensionResourceLoaderService } from 'vs/workbench/services/extensionResourceLoader/common/extensionResourceLoader';
 import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
+import { IProgressService } from 'vs/platform/progress/common/progress';
 
 const RUN_TEXTMATE_IN_WORKER = false;
 
@@ -147,10 +148,11 @@ export class TextMateService extends AbstractTextMateService {
 		@ILogService logService: ILogService,
 		@IConfigurationService configurationService: IConfigurationService,
 		@IStorageService storageService: IStorageService,
+		@IProgressService progressService: IProgressService,
 		@IModelService private readonly _modelService: IModelService,
-		@IWorkbenchEnvironmentService private readonly _environmentService: IWorkbenchEnvironmentService
+		@IWorkbenchEnvironmentService private readonly _environmentService: IWorkbenchEnvironmentService,
 	) {
-		super(modeService, themeService, extensionResourceLoaderService, notificationService, logService, configurationService, storageService);
+		super(modeService, themeService, extensionResourceLoaderService, notificationService, logService, configurationService, storageService, progressService);
 		this._worker = null;
 		this._workerProxy = null;
 		this._tokenizers = Object.create(null);

yarn.lock

@@ -9711,10 +9711,10 @@ vscode-nsfw@1.2.8:
     lodash.isundefined "^3.0.1"
     nan "^2.10.0"
 
-vscode-oniguruma@1.1.1:
+vscode-oniguruma@1.3.0:
-  version "1.1.1"
+  version "1.3.0"
-  resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.1.1.tgz#81460d148e70668b7c1abfd78ab7fcb2e2c6b316"
+  resolved "https://registry.yarnpkg.com/vscode-oniguruma/-/vscode-oniguruma-1.3.0.tgz#6788a9db2f8b0781243b4eb8c7a1dd25f6c0e2c8"
-  integrity sha512-4bygPUQjD5o9nLxp8f0s2TbqLte+Kz3C/IZxheIwunos69jI6OOxAhXW06RgqFbnXAw9ggUydx5qIZ7LD6iNuw==
+  integrity sha512-m4Br19v6XD4MRbVrgsLNSZgQrBzk1BCMCleL8+GrcoGxKEJJd62zOFcTaoQR3hCrSlLqoxWmJ7Cc0VieVV3iTQ==
 
 vscode-proxy-agent@^0.5.2:
   version "0.5.2"