Merge pull request #135774 from microsoft/alex/main-process-extension-host

Create extension host processes from a node worker in the main process

build/gulpfile.vscode.js

@@ -64,6 +64,7 @@ const vscodeResources = [
 	'out-build/vs/base/browser/ui/codicons/codicon/**',
 	'out-build/vs/base/parts/sandbox/electron-browser/preload.js',
 	'out-build/vs/platform/environment/node/userDataPath.js',
+	'out-build/vs/platform/extensions/node/extensionHostStarterWorkerMain.js',
 	'out-build/vs/workbench/browser/media/*-theme.css',
 	'out-build/vs/workbench/contrib/debug/**/*.json',
 	'out-build/vs/workbench/contrib/externalTerminal/**/*.scpt',

build/lib/bundle.js

@@ -14,6 +14,9 @@ const vm = require("vm");
 function bundle(entryPoints, config, callback) {
     const entryPointsMap = {};
     entryPoints.forEach((module) => {
+        if (entryPointsMap[module.name]) {
+            throw new Error(`Cannot have two entry points with the same name '${module.name}'`);
+        }
         entryPointsMap[module.name] = module;
     });
     const allMentionedModulesMap = {};

build/lib/bundle.ts

@@ -100,6 +100,9 @@ export interface ILoaderConfig {
 export function bundle(entryPoints: IEntryPoint[], config: ILoaderConfig, callback: (err: any, result: IBundleResult | null) => void): void {
 	const entryPointsMap: IEntryPointMap = {};
 	entryPoints.forEach((module: IEntryPoint) => {
+		if (entryPointsMap[module.name]) {
+			throw new Error(`Cannot have two entry points with the same name '${module.name}'`);
+		}
 		entryPointsMap[module.name] = module;
 	});
 

src/buildfile.js

@@ -5,13 +5,22 @@
 
 const { createModuleDescription, createEditorWorkerModuleDescription } = require('./vs/base/buildfile');
 
-exports.base = [{
-	name: 'vs/base/common/worker/simpleWorker',
-	include: ['vs/editor/common/services/editorSimpleWorker'],
-	prepend: ['vs/loader.js', 'vs/nls.js'],
-	append: ['vs/base/worker/workerMain'],
-	dest: 'vs/base/worker/workerMain.js'
-}];
+exports.base = [
+	{
+		name: 'vs/editor/common/services/editorSimpleWorker',
+		include: ['vs/base/common/worker/simpleWorker'],
+		prepend: ['vs/loader.js', 'vs/nls.js'],
+		append: ['vs/base/worker/workerMain'],
+		dest: 'vs/base/worker/workerMain.js'
+	},
+	{
+		name: 'vs/base/common/worker/simpleWorker',
+	},
+	{
+		name: 'vs/platform/extensions/node/extensionHostStarterWorker',
+		exclude: ['vs/base/common/worker/simpleWorker']
+	}
+];
 
 exports.workerExtensionHost = [createEditorWorkerModuleDescription('vs/workbench/services/extensions/worker/extensionHostWorker')];
 exports.workerNotebook = [createEditorWorkerModuleDescription('vs/workbench/contrib/notebook/common/services/notebookSimpleWorker')];

src/vs/code/electron-main/app.ts

@@ -44,6 +44,8 @@ import { isLaunchedFromCli } from 'vs/platform/environment/node/argvHelper';
 import { resolveShellEnv } from 'vs/platform/environment/node/shellEnv';
 import { IExtensionUrlTrustService } from 'vs/platform/extensionManagement/common/extensionUrlTrust';
 import { ExtensionUrlTrustService } from 'vs/platform/extensionManagement/node/extensionUrlTrustService';
+import { IExtensionHostStarter, ipcExtensionHostStarterChannelName } from 'vs/platform/extensions/common/extensionHostStarter';
+import { WorkerMainProcessExtensionHostStarter } from 'vs/platform/extensions/electron-main/workerMainProcessExtensionHostStarter';
 import { IExternalTerminalMainService } from 'vs/platform/externalTerminal/common/externalTerminal';
 import { LinuxExternalTerminalService, MacExternalTerminalService, WindowsExternalTerminalService } from 'vs/platform/externalTerminal/node/externalTerminalService';
 import { IFileService } from 'vs/platform/files/common/files';
@@ -514,6 +516,9 @@ export class CodeApplication extends Disposable {
 		// Extension URL Trust
 		services.set(IExtensionUrlTrustService, new SyncDescriptor(ExtensionUrlTrustService));
 
+		// Extension Host Starter
+		services.set(IExtensionHostStarter, new SyncDescriptor(WorkerMainProcessExtensionHostStarter));
+
 		// Storage
 		services.set(IStorageMainService, new SyncDescriptor(StorageMainService));
 
@@ -640,6 +645,10 @@ export class CodeApplication extends Disposable {
 		// Extension Host Debug Broadcasting
 		const electronExtensionHostDebugBroadcastChannel = new ElectronExtensionHostDebugBroadcastChannel(accessor.get(IWindowsMainService));
 		mainProcessElectronServer.registerChannel('extensionhostdebugservice', electronExtensionHostDebugBroadcastChannel);
+
+		// Extension Host Starter
+		const extensionHostStarterChannel = ProxyChannel.fromService(accessor.get(IExtensionHostStarter));
+		mainProcessElectronServer.registerChannel(ipcExtensionHostStarterChannelName, extensionHostStarterChannel);
 	}
 
 	private openFirstWindow(accessor: ServicesAccessor, mainProcessElectronServer: ElectronIPCServer): ICodeWindow[] {

src/vs/platform/extensions/electron-main/directMainProcessExtensionHostStarter.ts

@@ -0,0 +1,29 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { ExtensionHostStarter, IPartialLogService } from 'vs/platform/extensions/node/extensionHostStarter';
+import { ILifecycleMainService } from 'vs/platform/lifecycle/electron-main/lifecycleMainService';
+import { ILogService } from 'vs/platform/log/common/log';
+
+export class DirectMainProcessExtensionHostStarter extends ExtensionHostStarter {
+
+	constructor(
+		@ILogService logService: IPartialLogService,
+		@ILifecycleMainService lifecycleMainService: ILifecycleMainService
+	) {
+		super(logService);
+
+		// Abnormal shutdown: terminate extension hosts asap
+		lifecycleMainService.onWillKill(() => {
+			this.killAllNow();
+		});
+
+		// Normal shutdown: gracefully await extension host shutdowns
+		lifecycleMainService.onWillShutdown((e) => {
+			e.join(this.waitForAllExit(6000));
+		});
+	}
+
+}

src/vs/platform/extensions/electron-main/workerMainProcessExtensionHostStarter.ts

@@ -0,0 +1,160 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { canceled, SerializedError } from 'vs/base/common/errors';
+import { IDisposable } from 'vs/base/common/lifecycle';
+import { IExtensionHostProcessOptions, IExtensionHostStarter } from 'vs/platform/extensions/common/extensionHostStarter';
+import { Event } from 'vs/base/common/event';
+import { FileAccess } from 'vs/base/common/network';
+import { ILogService } from 'vs/platform/log/common/log';
+import { Worker } from 'worker_threads';
+import { IWorker, IWorkerCallback, IWorkerFactory, SimpleWorkerClient } from 'vs/base/common/worker/simpleWorker';
+import { IExtensionHostStarterWorkerHost } from 'vs/platform/extensions/node/extensionHostStarterWorker';
+import { ILifecycleMainService } from 'vs/platform/lifecycle/electron-main/lifecycleMainService';
+import { ExtensionHostStarter } from 'vs/platform/extensions/node/extensionHostStarter';
+
+class NodeWorker implements IWorker {
+
+	private readonly _worker: Worker;
+
+	constructor(callback: IWorkerCallback, onErrorCallback: (err: any) => void) {
+		this._worker = new Worker(
+			FileAccess.asFileUri('vs/platform/extensions/node/extensionHostStarterWorkerMain.js', require).fsPath,
+		);
+		this._worker.on('message', callback);
+		this._worker.on('error', onErrorCallback);
+		// this._worker.on('exit', (code) => {
+		// 	console.log(`worker exited with code `, code);
+		// });
+	}
+
+	getId(): number {
+		return 1;
+	}
+
+	postMessage(message: any, transfer: ArrayBuffer[]): void {
+		this._worker.postMessage(message, transfer);
+	}
+
+	dispose(): void {
+		this._worker.terminate();
+	}
+}
+
+class ExtensionHostStarterWorkerHost implements IExtensionHostStarterWorkerHost {
+	constructor(
+		@ILogService private readonly _logService: ILogService
+	) { }
+
+	public async logInfo(message: string): Promise<void> {
+		this._logService.info(message);
+	}
+}
+
+export class WorkerMainProcessExtensionHostStarter implements IDisposable, IExtensionHostStarter {
+	_serviceBrand: undefined;
+
+	private _proxy: ExtensionHostStarter | null;
+	private readonly _worker: SimpleWorkerClient<ExtensionHostStarter, IExtensionHostStarterWorkerHost>;
+	private _shutdown = false;
+
+	constructor(
+		@ILogService private readonly _logService: ILogService,
+		@ILifecycleMainService lifecycleMainService: ILifecycleMainService
+	) {
+		this._proxy = null;
+
+		const workerFactory: IWorkerFactory = {
+			create: (moduleId: string, callback: IWorkerCallback, onErrorCallback: (err: any) => void): IWorker => {
+				const worker = new NodeWorker(callback, onErrorCallback);
+				worker.postMessage(moduleId, []);
+				return worker;
+			}
+		};
+		this._worker = new SimpleWorkerClient<ExtensionHostStarter, IExtensionHostStarterWorkerHost>(
+			workerFactory,
+			'vs/platform/extensions/node/extensionHostStarterWorker',
+			new ExtensionHostStarterWorkerHost(this._logService)
+		);
+		this._initialize();
+
+		// Abnormal shutdown: terminate extension hosts asap
+		lifecycleMainService.onWillKill(async () => {
+			this._shutdown = true;
+			if (this._proxy) {
+				this._proxy.killAllNow();
+			}
+		});
+
+		// Normal shutdown: gracefully await extension host shutdowns
+		lifecycleMainService.onWillShutdown((e) => {
+			this._shutdown = true;
+			if (this._proxy) {
+				e.join(this._proxy.waitForAllExit(6000));
+			}
+		});
+	}
+
+	dispose(): void {
+		// Intentionally not killing the extension host processes
+	}
+
+	async _initialize(): Promise<void> {
+		this._proxy = await this._worker.getProxyObject();
+		this._logService.info(`ExtensionHostStarterWorker created`);
+	}
+
+	onDynamicStdout(id: string): Event<string> {
+		return this._proxy!.onDynamicStderr(id);
+	}
+
+	onDynamicStderr(id: string): Event<string> {
+		return this._proxy!.onDynamicStderr(id);
+	}
+
+	onDynamicMessage(id: string): Event<any> {
+		return this._proxy!.onDynamicMessage(id);
+	}
+
+	onDynamicError(id: string): Event<{ error: SerializedError; }> {
+		return this._proxy!.onDynamicError(id);
+	}
+
+	onDynamicExit(id: string): Event<{ code: number; signal: string; }> {
+		return this._proxy!.onDynamicExit(id);
+	}
+
+	async createExtensionHost(): Promise<{ id: string; }> {
+		const proxy = await this._worker.getProxyObject();
+		if (this._shutdown) {
+			throw canceled();
+		}
+		return proxy.createExtensionHost();
+	}
+
+	async start(id: string, opts: IExtensionHostProcessOptions): Promise<{ pid: number; }> {
+		const proxy = await this._worker.getProxyObject();
+		if (this._shutdown) {
+			throw canceled();
+		}
+		return proxy.start(id, opts);
+	}
+
+	async enableInspectPort(id: string): Promise<boolean> {
+		const proxy = await this._worker.getProxyObject();
+		if (this._shutdown) {
+			throw canceled();
+		}
+		return proxy.enableInspectPort(id);
+	}
+
+	async kill(id: string): Promise<void> {
+		const proxy = await this._worker.getProxyObject();
+		if (this._shutdown) {
+			throw canceled();
+		}
+		return proxy.kill(id);
+	}
+}

src/vs/platform/extensions/node/extensionHostStarter.ts

@@ -15,6 +15,13 @@ import { ILogService } from 'vs/platform/log/common/log';
 import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
 import { mixin } from 'vs/base/common/objects';
 import { cwd } from 'vs/base/common/process';
+import { StopWatch } from 'vs/base/common/stopwatch';
+import { Promises, timeout } from 'vs/base/common/async';
+
+export interface IPartialLogService {
+	readonly _serviceBrand: undefined;
+	info(message: string): void;
+}
 
 class ExtensionHostProcess extends Disposable {
 
@@ -34,27 +41,26 @@ class ExtensionHostProcess extends Disposable {
 	readonly onExit = this._onExit.event;
 
 	private _process: ChildProcess | null = null;
+	private _hasExited: boolean = false;
 
 	constructor(
 		public readonly id: string,
-		@ILogService private readonly _logService: ILogService
+		@ILogService private readonly _logService: IPartialLogService
 	) {
 		super();
 	}
 
-	register(disposable: IDisposable) {
-		this._register(disposable);
-	}
-
 	start(opts: IExtensionHostProcessOptions): { pid: number; } {
+		const sw = StopWatch.create(false);
 		this._process = fork(
 			FileAccess.asFileUri('bootstrap-fork', require).fsPath,
 			['--type=extensionHost', '--skipWorkspaceStorageLock'],
 			mixin({ cwd: cwd() }, opts),
 		);
+		const forkTime = sw.elapsed();
 		const pid = this._process.pid;
 
-		this._logService.info(`Starting extension host with pid ${pid}.`);
+		this._logService.info(`Starting extension host with pid ${pid} (fork() took ${forkTime} ms).`);
 
 		const stdoutDecoder = new StringDecoder('utf-8');
 		this._process.stdout?.on('data', (chunk) => {
@@ -77,6 +83,7 @@ class ExtensionHostProcess extends Disposable {
 		});
 
 		this._process.on('exit', (code: number, signal: string) => {
+			this._hasExited = true;
 			this._onExit.fire({ pid, code, signal });
 		});
 
@@ -115,6 +122,21 @@ class ExtensionHostProcess extends Disposable {
 		this._logService.info(`Killing extension host with pid ${this._process.pid}.`);
 		this._process.kill();
 	}
+
+	async waitForExit(maxWaitTimeMs: number): Promise<void> {
+		if (!this._process) {
+			return;
+		}
+		const pid = this._process.pid;
+		this._logService.info(`Waiting for extension host with pid ${pid} to exit.`);
+		await Promise.race([Event.toPromise(this.onExit), timeout(maxWaitTimeMs)]);
+
+		if (!this._hasExited) {
+			// looks like we timed out
+			this._logService.info(`Extension host with pid ${pid} did not exit within ${maxWaitTimeMs}ms.`);
+			this._process.kill();
+		}
+	}
 }
 
 export class ExtensionHostStarter implements IDisposable, IExtensionHostStarter {
@@ -122,10 +144,10 @@ export class ExtensionHostStarter implements IDisposable, IExtensionHostStarter
 
 	private static _lastId: number = 0;
 
-	private readonly _extHosts: Map<string, ExtensionHostProcess>;
+	protected readonly _extHosts: Map<string, ExtensionHostProcess>;
 
 	constructor(
-		@ILogService private readonly _logService: ILogService
+		@ILogService private readonly _logService: IPartialLogService
 	) {
 		this._extHosts = new Map<string, ExtensionHostProcess>();
 	}
@@ -196,6 +218,20 @@ export class ExtensionHostStarter implements IDisposable, IExtensionHostStarter
 		}
 		extHostProcess.kill();
 	}
+
+	async killAllNow(): Promise<void> {
+		for (const [, extHost] of this._extHosts) {
+			extHost.kill();
+		}
+	}
+
+	async waitForAllExit(maxWaitTimeMs: number): Promise<void> {
+		const exitPromises: Promise<void>[] = [];
+		for (const [, extHost] of this._extHosts) {
+			exitPromises.push(extHost.waitForExit(maxWaitTimeMs));
+		}
+		return Promises.settled(exitPromises).then(() => { });
+	}
 }
 
 registerSingleton(IExtensionHostStarter, ExtensionHostStarter, true);

src/vs/platform/extensions/node/extensionHostStarterWorker.ts

@@ -0,0 +1,24 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { ExtensionHostStarter, IPartialLogService } from 'vs/platform/extensions/node/extensionHostStarter';
+
+export interface IExtensionHostStarterWorkerHost {
+	logInfo(message: string): Promise<void>;
+}
+
+/**
+ * The `create` function needs to be there by convention because
+ * we are loaded via the `vs/base/common/worker/simpleWorker` utility.
+ */
+export function create(host: IExtensionHostStarterWorkerHost) {
+	const partialLogService: IPartialLogService = {
+		_serviceBrand: undefined,
+		info: (message: string): void => {
+			host.logInfo(message);
+		}
+	};
+	return new ExtensionHostStarter(partialLogService);
+}

src/vs/platform/extensions/node/extensionHostStarterWorkerMain.ts

@@ -0,0 +1,66 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+(function () {
+	'use strict';
+
+	const loader = require('../../../loader');
+	const bootstrap = require('../../../../bootstrap');
+	const path = require('path');
+	const parentPort = require('worker_threads').parentPort;
+
+	// Bootstrap: NLS
+	const nlsConfig = bootstrap.setupNLS();
+
+	// Bootstrap: Loader
+	loader.config({
+		baseUrl: bootstrap.fileUriFromPath(path.join(__dirname, '../../../../'), { isWindows: process.platform === 'win32' }),
+		catchError: true,
+		nodeRequire: require,
+		nodeMain: __filename,
+		'vs/nls': nlsConfig,
+		amdModulesPattern: /^vs\//,
+		recordStats: true
+	});
+
+	let isFirstMessage = true;
+	let beforeReadyMessages: any[] = [];
+
+	const initialMessageHandler = (data: any) => {
+		if (!isFirstMessage) {
+			beforeReadyMessages.push(data);
+			return;
+		}
+
+		isFirstMessage = false;
+		loadCode(data);
+	};
+
+	parentPort.on('message', initialMessageHandler);
+
+	const loadCode = function (moduleId: string) {
+		loader([moduleId], function (ws: any) {
+			setTimeout(() => {
+
+				const messageHandler = ws.create((msg: any, transfer?: ArrayBuffer[]) => {
+					parentPort.postMessage(msg, transfer);
+				}, null);
+				parentPort.off('message', initialMessageHandler);
+				parentPort.on('message', (data: any) => {
+					messageHandler.onmessage(data);
+				});
+				while (beforeReadyMessages.length > 0) {
+					const msg = beforeReadyMessages.shift()!;
+					messageHandler.onmessage(msg);
+				}
+
+			});
+		}, (err: any) => console.error(err));
+	};
+
+	parentPort.on('messageerror', (err: Error) => {
+		console.error(err);
+	});
+})();

src/vs/workbench/services/extensions/common/extensionHostMain.ts

@@ -137,19 +137,17 @@ export class ExtensionHostMain {
 
 		const extensionsDeactivated = this._extensionService.deactivateAll();
 
-		// Give extensions 1 second to wrap up any async dispose, then exit in at most 4 seconds
-		setTimeout(() => {
-			Promise.race([timeout(4000), extensionsDeactivated]).finally(() => {
-				if (this._hostUtils.pid) {
-					this._logService.info(`Extension host with pid ${this._hostUtils.pid} exiting with code 0`);
-				} else {
-					this._logService.info(`Extension host exiting with code 0`);
-				}
-				this._logService.flush();
-				this._logService.dispose();
-				this._hostUtils.exit(0);
-			});
-		}, 1000);
+		// Give extensions at most 5 seconds to wrap up any async deactivate, then exit
+		Promise.race([timeout(5000), extensionsDeactivated]).finally(() => {
+			if (this._hostUtils.pid) {
+				this._logService.info(`Extension host with pid ${this._hostUtils.pid} exiting with code 0`);
+			} else {
+				this._logService.info(`Extension host exiting with code 0`);
+			}
+			this._logService.flush();
+			this._logService.dispose();
+			this._hostUtils.exit(0);
+		});
 	}
 
 	private static _transform(initData: IInitData, rpcProtocol: RPCProtocol): IInitData {

src/vs/workbench/services/extensions/electron-sandbox/extensionHostStarter.ts

@@ -3,7 +3,13 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { registerSharedProcessRemoteService } from 'vs/platform/ipc/electron-sandbox/services';
+import { registerMainProcessRemoteService, registerSharedProcessRemoteService } from 'vs/platform/ipc/electron-sandbox/services';
 import { IExtensionHostStarter, ipcExtensionHostStarterChannelName } from 'vs/platform/extensions/common/extensionHostStarter';
 
-registerSharedProcessRemoteService(IExtensionHostStarter, ipcExtensionHostStarterChannelName, { supportsDelayedInstantiation: true });
+const location = 'main' as 'main' | 'shared';
+
+if (location === 'main') {
+	registerMainProcessRemoteService(IExtensionHostStarter, ipcExtensionHostStarterChannelName, { supportsDelayedInstantiation: true });
+} else {
+	registerSharedProcessRemoteService(IExtensionHostStarter, ipcExtensionHostStarterChannelName, { supportsDelayedInstantiation: true });
+}

src/vs/workbench/workbench.desktop.main.ts

@@ -58,7 +58,6 @@ import 'vs/workbench/electron-browser/desktop.main';
 
 
 import 'vs/workbench/services/extensions/electron-browser/extensionService';
-import 'vs/platform/extensions/node/extensionHostStarter';
 
 
 // !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

src/vs/workbench/workbench.desktop.sandbox.main.ts

@@ -33,7 +33,6 @@ import 'vs/workbench/electron-sandbox/desktop.main';
 
 //#region --- workbench services
 
-import 'vs/workbench/services/extensions/electron-sandbox/extensionHostStarter';
 
 //#endregion
 

src/vs/workbench/workbench.sandbox.main.ts

@@ -52,6 +52,7 @@ import 'vs/workbench/services/credentials/electron-sandbox/credentialsService';
 import 'vs/workbench/services/encryption/electron-sandbox/encryptionService';
 import 'vs/workbench/services/localizations/electron-sandbox/localizationsService';
 import 'vs/workbench/services/telemetry/electron-sandbox/telemetryService';
+import 'vs/workbench/services/extensions/electron-sandbox/extensionHostStarter';
 import 'vs/workbench/services/extensionManagement/electron-sandbox/extensionManagementServerService';
 import 'vs/workbench/services/extensionManagement/electron-sandbox/extensionTipsService';
 import 'vs/workbench/services/userDataSync/electron-sandbox/userDataSyncMachinesService';