perf - inline package.json and product.json (#219841)

build/darwin/create-universal-app.js

@@ -28,7 +28,6 @@ async function main(buildDir) {
         x64AsarPath,
         arm64AsarPath,
         filesToSkip: [
-            'product.json',
             'Credits.rtf',
             'CodeResources',
             'fsevents.node',

build/darwin/create-universal-app.ts

@@ -32,7 +32,6 @@ async function main(buildDir?: string) {
 		x64AsarPath,
 		arm64AsarPath,
 		filesToSkip: [
-			'product.json',
 			'Credits.rtf',
 			'CodeResources',
 			'fsevents.node',

build/gulpfile.reh.js

@@ -12,11 +12,13 @@ const util = require('./lib/util');
 const { getVersion } = require('./lib/getVersion');
 const task = require('./lib/task');
 const optimize = require('./lib/optimize');
+const { inlineMeta } = require('./lib/inlineMeta');
 const product = require('../product.json');
 const rename = require('gulp-rename');
 const replace = require('gulp-replace');
 const filter = require('gulp-filter');
 const { getProductionDependencies } = require('./lib/dependencies');
+const { date } = require('./lib/date');
 const vfs = require('vinyl-fs');
 const packageJson = require('../package.json');
 const flatmap = require('gulp-flatmap');
@@ -115,6 +117,12 @@ const serverWithWebEntryPoints = [
 	...vscodeWebEntryPoints
 ];
 
+const commonJSEntryPoints = [
+	'out-build/server-main.js',
+	'out-build/server-cli.js',
+	'out-build/bootstrap-fork.js',
+];
+
 function getNodeVersion() {
 	const yarnrc = fs.readFileSync(path.join(REPO_ROOT, 'remote', '.yarnrc'), 'utf8');
 	const nodeVersion = /^target "(.*)"$/m.exec(yarnrc)[1];
@@ -180,7 +188,6 @@ if (defaultNodeTask) {
 function nodejs(platform, arch) {
 	const { fetchUrls, fetchGithub } = require('./lib/fetch');
 	const untar = require('gulp-untar');
-	const crypto = require('crypto');
 
 	if (arch === 'armhf') {
 		arch = 'armv7l';
@@ -286,13 +293,22 @@ function packageTask(type, platform, arch, sourceFolderName, destinationFolderNa
 		}
 
 		const name = product.nameShort;
+
+		let packageJsonContents;
 		const packageJsonStream = gulp.src(['remote/package.json'], { base: 'remote' })
-			.pipe(json({ name, version, dependencies: undefined, optionalDependencies: undefined }));
-
-		const date = new Date().toISOString();
+			.pipe(json({ name, version, dependencies: undefined, optionalDependencies: undefined }))
+			.pipe(es.through(function (file) {
+				packageJsonContents = file.contents.toString();
+				this.emit('data', file);
+			}));
 
+		let productJsonContents;
 		const productJsonStream = gulp.src(['product.json'], { base: '.' })
-			.pipe(json({ commit, date, version }));
+			.pipe(json({ commit, date, version }))
+			.pipe(es.through(function (file) {
+				productJsonContents = file.contents.toString();
+				this.emit('data', file);
+			}));
 
 		const license = gulp.src(['remote/LICENSE'], { base: 'remote', allowEmpty: true });
 
@@ -385,6 +401,12 @@ function packageTask(type, platform, arch, sourceFolderName, destinationFolderNa
 			);
 		}
 
+		result = inlineMeta(result, {
+			targetPaths: commonJSEntryPoints,
+			packageJsonFn: () => packageJsonContents,
+			productJsonFn: () => productJsonContents
+		});
+
 		return result.pipe(vfs.dest(destination));
 	};
 }
@@ -416,17 +438,14 @@ function tweakProductForServerWeb(product) {
 				},
 				commonJS: {
 					src: 'out-build',
-					entryPoints: [
-						'out-build/server-main.js',
-						'out-build/server-cli.js',
-						'out-build/bootstrap-fork.js',
-					],
+					entryPoints: commonJSEntryPoints,
 					platform: 'node',
 					external: [
 						'minimist',
-						// TODO: we cannot inline `product.json` because
+						// We cannot inline `product.json` from here because
 						// it is being changed during build time at a later
 						// point in time (such as `checksums`)
+						// We have a manual step to inline these later.
 						'../product.json',
 						'../package.json'
 					]

build/gulpfile.vscode.js

@@ -6,10 +6,7 @@
 'use strict';
 
 const gulp = require('gulp');
-const merge = require('gulp-merge-json');
 const fs = require('fs');
-const os = require('os');
-const cp = require('child_process');
 const path = require('path');
 const es = require('event-stream');
 const vfs = require('vinyl-fs');
@@ -18,9 +15,11 @@ const replace = require('gulp-replace');
 const filter = require('gulp-filter');
 const util = require('./lib/util');
 const { getVersion } = require('./lib/getVersion');
+const { date } = require('./lib/date');
 const task = require('./lib/task');
 const buildfile = require('../src/buildfile');
 const optimize = require('./lib/optimize');
+const { inlineMeta } = require('./lib/inlineMeta');
 const root = path.dirname(__dirname);
 const commit = getVersion(root);
 const packageJson = require('../package.json');
@@ -85,6 +84,12 @@ const windowBootstrapFiles = [
 	'out-build/bootstrap-window.js'
 ];
 
+const commonJSEntryPoints = [
+	'out-build/main.js',
+	'out-build/cli.js',
+	'out-build/bootstrap-fork.js'
+];
+
 const optimizeVSCodeTask = task.define('optimize-vscode', task.series(
 	util.rimraf('out-vscode'),
 	// Optimize: bundles source files automatically based on
@@ -103,19 +108,16 @@ const optimizeVSCodeTask = task.define('optimize-vscode', task.series(
 			},
 			commonJS: {
 				src: 'out-build',
-				entryPoints: [
-					'out-build/main.js',
-					'out-build/cli.js',
-					'out-build/bootstrap-fork.js',
-				],
+				entryPoints: commonJSEntryPoints,
 				platform: 'node',
 				external: [
 					'electron',
 					'minimist',
 					'original-fs',
-					// TODO: we cannot inline `product.json` because
+					// We cannot inline `product.json` from here because
 					// it is being changed during build time at a later
 					// point in time (such as `checksums`)
+					// We have a manual step to inline these later.
 					'../product.json',
 					'../package.json',
 				]
@@ -247,14 +249,21 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
 			packageJsonUpdates.desktopName = `${product.applicationName}-url-handler.desktop`;
 		}
 
+		let packageJsonContents;
 		const packageJsonStream = gulp.src(['package.json'], { base: '.' })
-			.pipe(json(packageJsonUpdates));
-
-		const date = new Date().toISOString();
-		const productJsonUpdate = { commit, date, checksums, version };
+			.pipe(json(packageJsonUpdates))
+			.pipe(es.through(function (file) {
+				packageJsonContents = file.contents.toString();
+				this.emit('data', file);
+			}));
 
+		let productJsonContents;
 		const productJsonStream = gulp.src(['product.json'], { base: '.' })
-			.pipe(json(productJsonUpdate));
+			.pipe(json({ commit, date, checksums, version }))
+			.pipe(es.through(function (file) {
+				productJsonContents = file.contents.toString();
+				this.emit('data', file);
+			}));
 
 		const license = gulp.src([product.licenseFileName, 'ThirdPartyNotices.txt', 'licenses/**'], { base: '.', allowEmpty: true });
 
@@ -387,6 +396,12 @@ function packageTask(platform, arch, sourceFolderName, destinationFolderName, op
 				.pipe(rename('bin/' + product.applicationName)));
 		}
 
+		result = inlineMeta(result, {
+			targetPaths: commonJSEntryPoints,
+			packageJsonFn: () => packageJsonContents,
+			productJsonFn: () => productJsonContents
+		});
+
 		return result.pipe(vfs.dest(destination));
 	};
 }

build/gulpfile.vscode.web.js

@@ -12,12 +12,12 @@ const util = require('./lib/util');
 const { getVersion } = require('./lib/getVersion');
 const task = require('./lib/task');
 const optimize = require('./lib/optimize');
+const { date } = require('./lib/date');
 const product = require('../product.json');
 const rename = require('gulp-rename');
 const filter = require('gulp-filter');
 const { getProductionDependencies } = require('./lib/dependencies');
 const vfs = require('vinyl-fs');
-const replace = require('gulp-replace');
 const packageJson = require('../package.json');
 const { compileBuildTask } = require('./gulpfile.compile');
 const extensions = require('./lib/extensions');
@@ -76,8 +76,6 @@ const vscodeWebEntryPoints = [
 ].flat();
 exports.vscodeWebEntryPoints = vscodeWebEntryPoints;
 
-const buildDate = new Date().toISOString();
-
 /**
  * @param {object} product The parsed product.json file contents
  */
@@ -93,7 +91,7 @@ const createVSCodeWebProductConfigurationPatcher = (product) => {
 				...product,
 				version,
 				commit,
-				date: buildDate
+				date
 			});
 			return content.replace('/*BUILD->INSERT_PRODUCT_CONFIGURATION*/', () => productConfiguration.substr(1, productConfiguration.length - 2) /* without { and }*/);
 		}

build/lib/date.js

@@ -0,0 +1,24 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.date = void 0;
+function getRoundedBuildDate() {
+    const now = new Date();
+    const minutes = now.getMinutes();
+    if (minutes >= 30) {
+        now.setHours(now.getHours() + 1);
+    }
+    now.setMinutes(0, 0, 0);
+    return now;
+}
+/**
+ * An attempt to produce a stable date for the build that can be
+ * used across processes and build steps that run in parallel almost
+ * at the same time. The current time is rounded up or down to the
+ * closest hour.
+ */
+exports.date = getRoundedBuildDate();
+//# sourceMappingURL=date.js.map

build/lib/date.ts

@@ -0,0 +1,25 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+function getRoundedBuildDate() {
+	const now = new Date();
+
+	const minutes = now.getMinutes();
+	if (minutes >= 30) {
+		now.setHours(now.getHours() + 1);
+	}
+
+	now.setMinutes(0, 0, 0);
+
+	return now;
+}
+
+/**
+ * An attempt to produce a stable date for the build that can be
+ * used across processes and build steps that run in parallel almost
+ * at the same time. The current time is rounded up or down to the
+ * closest hour.
+ */
+export const date = getRoundedBuildDate();

build/lib/inlineMeta.js

@@ -0,0 +1,46 @@
+"use strict";
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.inlineMeta = inlineMeta;
+const es = require("event-stream");
+const path_1 = require("path");
+const packageJsonMarkerId = 'BUILD_INSERT_PACKAGE_CONFIGURATION';
+const productJsonMarkerId = 'BUILD_INSERT_PRODUCT_CONFIGURATION';
+function inlineMeta(result, ctx) {
+    return result.pipe(es.through(function (file) {
+        if (matchesFile(file, ctx)) {
+            let content = file.contents.toString();
+            let markerFound = false;
+            const packageMarker = `${packageJsonMarkerId}:"${packageJsonMarkerId}"`; // this needs to be the format after esbuild has processed the file (e.g. double quotes)
+            if (content.includes(packageMarker)) {
+                content = content.replace(packageMarker, JSON.stringify(JSON.parse(ctx.packageJsonFn())).slice(1, -1) /* trim braces */);
+                markerFound = true;
+            }
+            const productMarker = `${productJsonMarkerId}:"${productJsonMarkerId}"`; // this needs to be the format after esbuild has processed the file (e.g. double quotes)
+            if (content.includes(productMarker)) {
+                content = content.replace(productMarker, JSON.stringify(JSON.parse(ctx.productJsonFn())).slice(1, -1) /* trim braces */);
+                markerFound = true;
+            }
+            if (markerFound) {
+                file.contents = Buffer.from(content);
+            }
+            else if (content.includes(packageJsonMarkerId) || content.includes(productJsonMarkerId)) {
+                this.emit('error', new Error(`Unable to inline metadata because expected markers where not found in ${file.basename}.`));
+                return;
+            }
+        }
+        this.emit('data', file);
+    }));
+}
+function matchesFile(file, ctx) {
+    for (const targetPath of ctx.targetPaths) {
+        if (file.basename === (0, path_1.basename)(targetPath)) { // TODO would be nicer to figure out root relative path to not match on false positives
+            return true;
+        }
+    }
+    return false;
+}
+//# sourceMappingURL=inlineMeta.js.map

build/lib/inlineMeta.ts

@@ -0,0 +1,56 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as es from 'event-stream';
+import { basename } from 'path';
+import * as File from 'vinyl';
+
+export interface IInlineMetaContext {
+	readonly targetPaths: string[];
+	readonly packageJsonFn: () => string;
+	readonly productJsonFn: () => string;
+}
+
+const packageJsonMarkerId = 'BUILD_INSERT_PACKAGE_CONFIGURATION';
+const productJsonMarkerId = 'BUILD_INSERT_PRODUCT_CONFIGURATION';
+
+export function inlineMeta(result: NodeJS.ReadWriteStream, ctx: IInlineMetaContext): NodeJS.ReadWriteStream {
+	return result.pipe(es.through(function (file: File) {
+		if (matchesFile(file, ctx)) {
+			let content = file.contents.toString();
+			let markerFound = false;
+
+			const packageMarker = `${packageJsonMarkerId}:"${packageJsonMarkerId}"`; // this needs to be the format after esbuild has processed the file (e.g. double quotes)
+			if (content.includes(packageMarker)) {
+				content = content.replace(packageMarker, JSON.stringify(JSON.parse(ctx.packageJsonFn())).slice(1, -1) /* trim braces */);
+				markerFound = true;
+			}
+
+			const productMarker = `${productJsonMarkerId}:"${productJsonMarkerId}"`; // this needs to be the format after esbuild has processed the file (e.g. double quotes)
+			if (content.includes(productMarker)) {
+				content = content.replace(productMarker, JSON.stringify(JSON.parse(ctx.productJsonFn())).slice(1, -1) /* trim braces */);
+				markerFound = true;
+			}
+
+			if (markerFound) {
+				file.contents = Buffer.from(content);
+			} else if (content.includes(packageJsonMarkerId) || content.includes(productJsonMarkerId)) {
+				this.emit('error', new Error(`Unable to inline metadata because expected markers where not found in ${file.basename}.`));
+				return;
+			}
+		}
+
+		this.emit('data', file);
+	}));
+}
+
+function matchesFile(file: File, ctx: IInlineMetaContext): boolean {
+	for (const targetPath of ctx.targetPaths) {
+		if (file.basename === basename(targetPath)) { // TODO would be nicer to figure out root relative path to not match on false positives
+			return true;
+		}
+	}
+	return false;
+}

src/bootstrap-amd.js

@@ -8,6 +8,7 @@
 
 /**
  * @typedef {import('./vs/nls').INLSConfiguration} INLSConfiguration
+ * @import { IProductConfiguration } from './vs/base/common/product'
  */
 
 // Store the node.js require function in a variable
@@ -19,8 +20,8 @@ const nodeRequire = require;
 globalThis._VSCODE_NODE_MODULES = new Proxy(Object.create(null), { get: (_target, mod) => nodeRequire(String(mod)) });
 
 // VSCODE_GLOBALS: package/product.json
-/** @type Record<string, any> */
-globalThis._VSCODE_PRODUCT_JSON = require('../product.json');
+/** @type Partial<IProductConfiguration> */
+globalThis._VSCODE_PRODUCT_JSON = require('./bootstrap-meta').product;
 if (process.env['VSCODE_DEV']) {
 	// Patch product overrides when running out of sources
 	try {
@@ -29,7 +30,7 @@ if (process.env['VSCODE_DEV']) {
 		globalThis._VSCODE_PRODUCT_JSON = Object.assign(globalThis._VSCODE_PRODUCT_JSON, overrides);
 	} catch (error) { /* ignore */ }
 }
-globalThis._VSCODE_PACKAGE_JSON = require('../package.json');
+globalThis._VSCODE_PACKAGE_JSON = require('./bootstrap-meta').pkg;
 
 // @ts-ignore
 const loader = require('./vs/loader');

src/bootstrap-meta.js

@@ -0,0 +1,28 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+//@ts-check
+'use strict';
+
+/**
+ * @import { IProductConfiguration } from './vs/base/common/product'
+ */
+
+/** @type Partial<IProductConfiguration> & { BUILD_INSERT_PRODUCT_CONFIGURATION?: string } */
+let product = { BUILD_INSERT_PRODUCT_CONFIGURATION: 'BUILD_INSERT_PRODUCT_CONFIGURATION' }; // DO NOT MODIFY, PATCHED DURING BUILD
+if (product['BUILD_INSERT_PRODUCT_CONFIGURATION']) {
+	// @ts-ignore
+	product = require('../product.json'); // Running out of sources
+}
+
+/** @type object & { BUILD_INSERT_PACKAGE_CONFIGURATION?: string } */
+let pkg = { BUILD_INSERT_PACKAGE_CONFIGURATION: 'BUILD_INSERT_PACKAGE_CONFIGURATION' }; // DO NOT MODIFY, PATCHED DURING BUILD
+if (pkg['BUILD_INSERT_PACKAGE_CONFIGURATION']) {
+	// @ts-ignore
+	pkg = require('../package.json'); // Running out of sources
+}
+
+exports.product = product;
+exports.pkg = pkg;

src/cli.js

@@ -6,10 +6,6 @@
 //@ts-check
 'use strict';
 
-/**
- * @import { IProductConfiguration } from './vs/base/common/product'
- */
-
 // Delete `VSCODE_CWD` very early even before
 // importing bootstrap files. We have seen
 // reports where `code .` would use the wrong
@@ -20,9 +16,7 @@ delete process.env['VSCODE_CWD'];
 
 const bootstrap = require('./bootstrap');
 const bootstrapNode = require('./bootstrap-node');
-/** @type {Partial<IProductConfiguration>} */
-// @ts-ignore
-const product = require('../product.json');
+const product = require('./bootstrap-meta').product;
 const { resolveNLSConfiguration } = require('./vs/base/node/nls');
 
 async function start() {

src/main.js

@@ -7,7 +7,6 @@
 'use strict';
 
 /**
- * @import { IProductConfiguration } from './vs/base/common/product'
  * @import { INLSConfiguration } from './vs/nls'
  * @import { NativeParsedArgs } from './vs/platform/environment/common/argv'
  */
@@ -23,9 +22,7 @@ const bootstrapNode = require('./bootstrap-node');
 const { getUserDataPath } = require('./vs/platform/environment/node/userDataPath');
 const { parse } = require('./vs/base/common/jsonc');
 const { getUNCHost, addUNCHostToAllowlist } = require('./vs/base/node/unc');
-/** @type {Partial<IProductConfiguration>} */
-// @ts-ignore
-const product = require('../product.json');
+const product = require('./bootstrap-meta').product;
 const { app, protocol, crashReporter, Menu } = require('electron');
 
 // Enable portable support

src/server-cli.js

@@ -6,14 +6,8 @@
 // @ts-check
 'use strict';
 
-/**
- * @import { IProductConfiguration } from './vs/base/common/product'
- */
-
 const path = require('path');
-/** @type {Partial<IProductConfiguration>} */
-// @ts-ignore
-const product = require('../product.json');
+const product = require('./bootstrap-meta').product;
 const { resolveNLSConfiguration } = require('./vs/base/node/nls');
 
 async function start() {

src/server-main.js

@@ -7,7 +7,6 @@
 'use strict';
 
 /**
- * @import { IProductConfiguration } from './vs/base/common/product'
  * @import { INLSConfiguration } from './vs/nls'
  */
 
@@ -17,9 +16,7 @@
 
 const perf = require('./vs/base/common/performance');
 const performance = require('perf_hooks').performance;
-/** @type {Partial<IProductConfiguration>} */
-// @ts-ignore
-const product = require('../product.json');
+const product = require('./bootstrap-meta').product;
 const readline = require('readline');
 const http = require('http');
 const { resolveNLSConfiguration } = require('./vs/base/node/nls');