From 8feb40b9284c339e2d1b0a493641e603b7f84d3d Mon Sep 17 00:00:00 2001 From: Raymond Zhao <7199958+rzhao271@users.noreply.github.com> Date: Tue, 19 Jul 2022 10:43:05 -0700 Subject: [PATCH] Generate dependencies per package build for Debian (#147335) Fixes #13089 --- build/gulpfile.vscode.linux.js | 9 +- build/linux/debian/dep-lists.js | 156 ++++++++++++++++++ build/linux/debian/dep-lists.ts | 157 +++++++++++++++++++ build/linux/debian/dependencies-generator.js | 129 +++++++++++++++ build/linux/debian/dependencies-generator.ts | 145 +++++++++++++++++ build/linux/debian/install-sysroot.js | 81 ++++++++++ build/linux/debian/install-sysroot.ts | 90 +++++++++++ build/linux/debian/sysroots.js | 26 +++ build/linux/debian/sysroots.ts | 24 +++ build/linux/debian/types.js | 6 + build/linux/debian/types.ts | 6 + build/linux/rpm/dependencies-generator.js | 2 +- build/linux/rpm/dependencies-generator.ts | 2 +- resources/linux/debian/control.template | 3 +- 14 files changed, 832 insertions(+), 4 deletions(-) create mode 100644 build/linux/debian/dep-lists.js create mode 100644 build/linux/debian/dep-lists.ts create mode 100644 build/linux/debian/dependencies-generator.js create mode 100644 build/linux/debian/dependencies-generator.ts create mode 100644 build/linux/debian/install-sysroot.js create mode 100644 build/linux/debian/install-sysroot.ts create mode 100644 build/linux/debian/sysroots.js create mode 100644 build/linux/debian/sysroots.ts create mode 100644 build/linux/debian/types.js create mode 100644 build/linux/debian/types.ts diff --git a/build/gulpfile.vscode.linux.js b/build/gulpfile.vscode.linux.js index 210556eddb6..489d9ccfabd 100644 --- a/build/gulpfile.vscode.linux.js +++ b/build/gulpfile.vscode.linux.js @@ -16,6 +16,9 @@ const task = require('./lib/task'); const packageJson = require('../package.json'); const product = require('../product.json'); const rpmDependenciesGenerator = require('./linux/rpm/dependencies-generator'); +const debianDependenciesGenerator = require('./linux/debian/dependencies-generator'); +const sysrootInstaller = require('./linux/debian/install-sysroot'); +const debianRecommendedDependencies = require('./linux/debian/dep-lists').recommendedDeps; const path = require('path'); const root = path.dirname(__dirname); const commit = util.getVersion(root); @@ -74,12 +77,16 @@ function prepareDebPackage(arch) { let size = 0; const control = code.pipe(es.through( function (f) { size += f.isDirectory() ? 4096 : f.contents.length; }, - function () { + async function () { const that = this; + const sysroot = await sysrootInstaller.getSysroot(debArch); + const dependencies = debianDependenciesGenerator.getDependencies(binaryDir, product.applicationName, debArch, sysroot); gulp.src('resources/linux/debian/control.template', { base: '.' }) .pipe(replace('@@NAME@@', product.applicationName)) .pipe(replace('@@VERSION@@', packageJson.version + '-' + linuxPackageRevision)) .pipe(replace('@@ARCHITECTURE@@', debArch)) + .pipe(replace('@@DEPENDS@@', dependencies.join(', '))) + .pipe(replace('@@RECOMMENDS@@', debianRecommendedDependencies.join(', '))) .pipe(replace('@@INSTALLEDSIZE@@', Math.ceil(size / 1024))) .pipe(rename('DEBIAN/control')) .pipe(es.through(function (f) { that.emit('data', f); }, function () { that.emit('end'); })); diff --git a/build/linux/debian/dep-lists.js b/build/linux/debian/dep-lists.js new file mode 100644 index 00000000000..3db2ef1abff --- /dev/null +++ b/build/linux/debian/dep-lists.js @@ -0,0 +1,156 @@ +"use strict"; +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.referenceGeneratedDepsByArch = exports.bundledDeps = exports.recommendedDeps = exports.additionalDeps = void 0; +// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/additional_deps +// Additional dependencies not in the dpkg-shlibdeps output. +exports.additionalDeps = [ + 'ca-certificates', + 'libgtk-3-0 (>= 3.9.10) | libgtk-4-1', + 'libnss3 (>= 3.26)', + 'libcurl3-gnutls | libcurl3-nss | libcurl4 | libcurl3', + 'xdg-utils (>= 1.0.2)' // OS integration +]; +// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/manual_recommends +// Dependencies that we can only recommend +// for now since some of the older distros don't support them. +exports.recommendedDeps = [ + 'libvulkan1' // Move to additionalDeps once support for Trusty and Jessie are dropped. +]; +// Based on https://source.chromium.org/chromium/chromium/src/+/refs/tags/98.0.4758.109:chrome/installer/linux/BUILD.gn;l=64-80 +// and the Linux Archive build +// Shared library dependencies that we already bundle. +exports.bundledDeps = [ + 'libEGL.so', + 'libGLESv2.so', + 'libvulkan.so.1', + 'swiftshader_libEGL.so', + 'swiftshader_libGLESv2.so', + 'libvk_swiftshader.so', + 'libffmpeg.so' +]; +exports.referenceGeneratedDepsByArch = { + 'amd64': [ + 'ca-certificates', + 'libasound2 (>= 1.0.16)', + 'libatk-bridge2.0-0 (>= 2.5.3)', + 'libatk1.0-0 (>= 2.2.0)', + 'libatspi2.0-0 (>= 2.9.90)', + 'libc6 (>= 2.14)', + 'libc6 (>= 2.17)', + 'libc6 (>= 2.2.5)', + 'libcairo2 (>= 1.6.0)', + 'libcurl3-gnutls | libcurl3-nss | libcurl4 | libcurl3', + 'libdbus-1-3 (>= 1.5.12)', + 'libdrm2 (>= 2.4.38)', + 'libexpat1 (>= 2.0.1)', + 'libgbm1 (>= 8.1~0)', + 'libgcc1 (>= 1:3.0)', + 'libglib2.0-0 (>= 2.16.0)', + 'libglib2.0-0 (>= 2.39.4)', + 'libgtk-3-0 (>= 3.9.10)', + 'libgtk-3-0 (>= 3.9.10) | libgtk-4-1', + 'libnspr4 (>= 2:4.9-2~)', + 'libnss3 (>= 2:3.22)', + 'libnss3 (>= 3.26)', + 'libpango-1.0-0 (>= 1.14.0)', + 'libsecret-1-0 (>= 0.18)', + 'libx11-6', + 'libx11-6 (>= 2:1.4.99.1)', + 'libxcb1 (>= 1.9.2)', + 'libxcomposite1 (>= 1:0.4.4-1)', + 'libxdamage1 (>= 1:1.1)', + 'libxext6', + 'libxfixes3', + 'libxkbcommon0 (>= 0.4.1)', + 'libxkbfile1', + 'libxrandr2', + 'xdg-utils (>= 1.0.2)' + ], + 'armhf': [ + 'ca-certificates', + 'libasound2 (>= 1.0.16)', + 'libatk-bridge2.0-0 (>= 2.5.3)', + 'libatk1.0-0 (>= 2.2.0)', + 'libatspi2.0-0 (>= 2.9.90)', + 'libc6 (>= 2.17)', + 'libc6 (>= 2.4)', + 'libc6 (>= 2.9)', + 'libcairo2 (>= 1.6.0)', + 'libcurl3-gnutls | libcurl3-nss | libcurl4 | libcurl3', + 'libdbus-1-3 (>= 1.5.12)', + 'libdrm2 (>= 2.4.38)', + 'libexpat1 (>= 2.0.1)', + 'libgbm1 (>= 8.1~0)', + 'libgcc1 (>= 1:3.0)', + 'libgcc1 (>= 1:3.5)', + 'libglib2.0-0 (>= 2.16.0)', + 'libglib2.0-0 (>= 2.39.4)', + 'libgtk-3-0 (>= 3.9.10)', + 'libgtk-3-0 (>= 3.9.10) | libgtk-4-1', + 'libnspr4 (>= 2:4.9-2~)', + 'libnss3 (>= 2:3.22)', + 'libnss3 (>= 3.26)', + 'libpango-1.0-0 (>= 1.14.0)', + 'libsecret-1-0 (>= 0.18)', + 'libstdc++6 (>= 4.1.1)', + 'libstdc++6 (>= 5)', + 'libstdc++6 (>= 5.2)', + 'libstdc++6 (>= 6)', + 'libx11-6', + 'libx11-6 (>= 2:1.4.99.1)', + 'libxcb1 (>= 1.9.2)', + 'libxcomposite1 (>= 1:0.4.4-1)', + 'libxdamage1 (>= 1:1.1)', + 'libxext6', + 'libxfixes3', + 'libxkbcommon0 (>= 0.4.1)', + 'libxkbfile1', + 'libxrandr2', + 'xdg-utils (>= 1.0.2)' + ], + 'arm64': [ + 'ca-certificates', + 'libasound2 (>= 1.0.16)', + 'libatk-bridge2.0-0 (>= 2.5.3)', + 'libatk1.0-0 (>= 2.2.0)', + 'libatspi2.0-0 (>= 2.9.90)', + 'libc6 (>= 2.17)', + 'libcairo2 (>= 1.6.0)', + 'libcurl3-gnutls | libcurl3-nss | libcurl4 | libcurl3', + 'libdbus-1-3 (>= 1.0.2)', + 'libdrm2 (>= 2.4.38)', + 'libexpat1 (>= 2.0.1)', + 'libgbm1 (>= 8.1~0)', + 'libgcc1 (>= 1:3.0)', + 'libgcc1 (>= 1:4.2)', + 'libgcc1 (>= 1:4.5)', + 'libglib2.0-0 (>= 2.16.0)', + 'libglib2.0-0 (>= 2.39.4)', + 'libgtk-3-0 (>= 3.9.10)', + 'libgtk-3-0 (>= 3.9.10) | libgtk-4-1', + 'libnspr4 (>= 2:4.9-2~)', + 'libnss3 (>= 2:3.22)', + 'libnss3 (>= 3.26)', + 'libpango-1.0-0 (>= 1.14.0)', + 'libsecret-1-0 (>= 0.18)', + 'libstdc++6 (>= 4.1.1)', + 'libstdc++6 (>= 5)', + 'libstdc++6 (>= 5.2)', + 'libstdc++6 (>= 6)', + 'libx11-6', + 'libx11-6 (>= 2:1.4.99.1)', + 'libxcb1 (>= 1.9.2)', + 'libxcomposite1 (>= 1:0.4.4-1)', + 'libxdamage1 (>= 1:1.1)', + 'libxext6', + 'libxfixes3', + 'libxkbcommon0 (>= 0.4.1)', + 'libxkbfile1', + 'libxrandr2', + 'xdg-utils (>= 1.0.2)' + ] +}; diff --git a/build/linux/debian/dep-lists.ts b/build/linux/debian/dep-lists.ts new file mode 100644 index 00000000000..1e1b0e5b63f --- /dev/null +++ b/build/linux/debian/dep-lists.ts @@ -0,0 +1,157 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/additional_deps +// Additional dependencies not in the dpkg-shlibdeps output. +export const additionalDeps = [ + 'ca-certificates', // Make sure users have SSL certificates. + 'libgtk-3-0 (>= 3.9.10) | libgtk-4-1', + 'libnss3 (>= 3.26)', + 'libcurl3-gnutls | libcurl3-nss | libcurl4 | libcurl3', // For Breakpad crash reports. + 'xdg-utils (>= 1.0.2)' // OS integration +]; + +// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/manual_recommends +// Dependencies that we can only recommend +// for now since some of the older distros don't support them. +export const recommendedDeps = [ + 'libvulkan1' // Move to additionalDeps once support for Trusty and Jessie are dropped. +]; + +// Based on https://source.chromium.org/chromium/chromium/src/+/refs/tags/98.0.4758.109:chrome/installer/linux/BUILD.gn;l=64-80 +// and the Linux Archive build +// Shared library dependencies that we already bundle. +export const bundledDeps = [ + 'libEGL.so', + 'libGLESv2.so', + 'libvulkan.so.1', + 'swiftshader_libEGL.so', + 'swiftshader_libGLESv2.so', + 'libvk_swiftshader.so', + 'libffmpeg.so' +]; + +export const referenceGeneratedDepsByArch = { + 'amd64': [ + 'ca-certificates', + 'libasound2 (>= 1.0.16)', + 'libatk-bridge2.0-0 (>= 2.5.3)', + 'libatk1.0-0 (>= 2.2.0)', + 'libatspi2.0-0 (>= 2.9.90)', + 'libc6 (>= 2.14)', + 'libc6 (>= 2.17)', + 'libc6 (>= 2.2.5)', + 'libcairo2 (>= 1.6.0)', + 'libcurl3-gnutls | libcurl3-nss | libcurl4 | libcurl3', + 'libdbus-1-3 (>= 1.5.12)', + 'libdrm2 (>= 2.4.38)', + 'libexpat1 (>= 2.0.1)', + 'libgbm1 (>= 8.1~0)', + 'libgcc1 (>= 1:3.0)', + 'libglib2.0-0 (>= 2.16.0)', + 'libglib2.0-0 (>= 2.39.4)', + 'libgtk-3-0 (>= 3.9.10)', + 'libgtk-3-0 (>= 3.9.10) | libgtk-4-1', + 'libnspr4 (>= 2:4.9-2~)', + 'libnss3 (>= 2:3.22)', + 'libnss3 (>= 3.26)', + 'libpango-1.0-0 (>= 1.14.0)', + 'libsecret-1-0 (>= 0.18)', + 'libx11-6', + 'libx11-6 (>= 2:1.4.99.1)', + 'libxcb1 (>= 1.9.2)', + 'libxcomposite1 (>= 1:0.4.4-1)', + 'libxdamage1 (>= 1:1.1)', + 'libxext6', + 'libxfixes3', + 'libxkbcommon0 (>= 0.4.1)', + 'libxkbfile1', + 'libxrandr2', + 'xdg-utils (>= 1.0.2)' + ], + 'armhf': [ + 'ca-certificates', + 'libasound2 (>= 1.0.16)', + 'libatk-bridge2.0-0 (>= 2.5.3)', + 'libatk1.0-0 (>= 2.2.0)', + 'libatspi2.0-0 (>= 2.9.90)', + 'libc6 (>= 2.17)', + 'libc6 (>= 2.4)', + 'libc6 (>= 2.9)', + 'libcairo2 (>= 1.6.0)', + 'libcurl3-gnutls | libcurl3-nss | libcurl4 | libcurl3', + 'libdbus-1-3 (>= 1.5.12)', + 'libdrm2 (>= 2.4.38)', + 'libexpat1 (>= 2.0.1)', + 'libgbm1 (>= 8.1~0)', + 'libgcc1 (>= 1:3.0)', + 'libgcc1 (>= 1:3.5)', + 'libglib2.0-0 (>= 2.16.0)', + 'libglib2.0-0 (>= 2.39.4)', + 'libgtk-3-0 (>= 3.9.10)', + 'libgtk-3-0 (>= 3.9.10) | libgtk-4-1', + 'libnspr4 (>= 2:4.9-2~)', + 'libnss3 (>= 2:3.22)', + 'libnss3 (>= 3.26)', + 'libpango-1.0-0 (>= 1.14.0)', + 'libsecret-1-0 (>= 0.18)', + 'libstdc++6 (>= 4.1.1)', + 'libstdc++6 (>= 5)', + 'libstdc++6 (>= 5.2)', + 'libstdc++6 (>= 6)', + 'libx11-6', + 'libx11-6 (>= 2:1.4.99.1)', + 'libxcb1 (>= 1.9.2)', + 'libxcomposite1 (>= 1:0.4.4-1)', + 'libxdamage1 (>= 1:1.1)', + 'libxext6', + 'libxfixes3', + 'libxkbcommon0 (>= 0.4.1)', + 'libxkbfile1', + 'libxrandr2', + 'xdg-utils (>= 1.0.2)' + ], + 'arm64': [ + 'ca-certificates', + 'libasound2 (>= 1.0.16)', + 'libatk-bridge2.0-0 (>= 2.5.3)', + 'libatk1.0-0 (>= 2.2.0)', + 'libatspi2.0-0 (>= 2.9.90)', + 'libc6 (>= 2.17)', + 'libcairo2 (>= 1.6.0)', + 'libcurl3-gnutls | libcurl3-nss | libcurl4 | libcurl3', + 'libdbus-1-3 (>= 1.0.2)', + 'libdrm2 (>= 2.4.38)', + 'libexpat1 (>= 2.0.1)', + 'libgbm1 (>= 8.1~0)', + 'libgcc1 (>= 1:3.0)', + 'libgcc1 (>= 1:4.2)', + 'libgcc1 (>= 1:4.5)', + 'libglib2.0-0 (>= 2.16.0)', + 'libglib2.0-0 (>= 2.39.4)', + 'libgtk-3-0 (>= 3.9.10)', + 'libgtk-3-0 (>= 3.9.10) | libgtk-4-1', + 'libnspr4 (>= 2:4.9-2~)', + 'libnss3 (>= 2:3.22)', + 'libnss3 (>= 3.26)', + 'libpango-1.0-0 (>= 1.14.0)', + 'libsecret-1-0 (>= 0.18)', + 'libstdc++6 (>= 4.1.1)', + 'libstdc++6 (>= 5)', + 'libstdc++6 (>= 5.2)', + 'libstdc++6 (>= 6)', + 'libx11-6', + 'libx11-6 (>= 2:1.4.99.1)', + 'libxcb1 (>= 1.9.2)', + 'libxcomposite1 (>= 1:0.4.4-1)', + 'libxdamage1 (>= 1:1.1)', + 'libxext6', + 'libxfixes3', + 'libxkbcommon0 (>= 0.4.1)', + 'libxkbfile1', + 'libxrandr2', + 'xdg-utils (>= 1.0.2)' + ] +}; diff --git a/build/linux/debian/dependencies-generator.js b/build/linux/debian/dependencies-generator.js new file mode 100644 index 00000000000..64fd184aae0 --- /dev/null +++ b/build/linux/debian/dependencies-generator.js @@ -0,0 +1,129 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.getDependencies = void 0; +const child_process_1 = require("child_process"); +const fs_1 = require("fs"); +const os_1 = require("os"); +const path = require("path"); +const dep_lists_1 = require("./dep-lists"); +// A flag that can easily be toggled. +// Make sure to compile the build directory after toggling the value. +// If false, we warn about new dependencies if they show up +// while running the Debian prepare package task for a release. +// If true, we fail the build if there are new dependencies found during that task. +// The reference dependencies, which one has to update when the new dependencies +// are valid, are in dep-lists.ts +const FAIL_BUILD_FOR_NEW_DEPENDENCIES = true; +function getDependencies(buildDir, applicationName, arch, sysroot) { + // Get the files for which we want to find dependencies. + const nativeModulesPath = path.join(buildDir, 'resources', 'app', 'node_modules.asar.unpacked'); + const findResult = (0, child_process_1.spawnSync)('find', [nativeModulesPath, '-name', '*.node']); + if (findResult.status) { + console.error('Error finding files:'); + console.error(findResult.stderr.toString()); + return []; + } + const files = findResult.stdout.toString().trimEnd().split('\n'); + const appPath = path.join(buildDir, applicationName); + files.push(appPath); + // Add chrome sandbox and crashpad handler. + files.push(path.join(buildDir, 'chrome-sandbox')); + files.push(path.join(buildDir, 'chrome_crashpad_handler')); + // Generate the dependencies. + const dependencies = files.map((file) => calculatePackageDeps(file, arch, sysroot)); + // Add additional dependencies. + const additionalDepsSet = new Set(dep_lists_1.additionalDeps); + dependencies.push(additionalDepsSet); + // Merge all the dependencies. + const mergedDependencies = mergePackageDeps(dependencies); + let sortedDependencies = []; + for (const dependency of mergedDependencies) { + sortedDependencies.push(dependency); + } + sortedDependencies.sort(); + // Exclude bundled dependencies + sortedDependencies = sortedDependencies.filter(dependency => { + return !dep_lists_1.bundledDeps.some(bundledDep => dependency.startsWith(bundledDep)); + }); + const referenceGeneratedDeps = dep_lists_1.referenceGeneratedDepsByArch[arch]; + if (JSON.stringify(sortedDependencies) !== JSON.stringify(referenceGeneratedDeps)) { + const failMessage = 'The dependencies list has changed.' + + '\nOld:\n' + referenceGeneratedDeps.join('\n') + + '\nNew:\n' + sortedDependencies.join('\n'); + if (FAIL_BUILD_FOR_NEW_DEPENDENCIES) { + throw new Error(failMessage); + } + else { + console.warn(failMessage); + } + } + return sortedDependencies; +} +exports.getDependencies = getDependencies; +// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/calculate_package_deps.py. +function calculatePackageDeps(binaryPath, arch, sysroot) { + try { + if (!((0, fs_1.statSync)(binaryPath).mode & fs_1.constants.S_IXUSR)) { + throw new Error(`Binary ${binaryPath} needs to have an executable bit set.`); + } + } + catch (e) { + // The package might not exist. Don't re-throw the error here. + console.error('Tried to stat ' + binaryPath + ' but failed.'); + } + // Get the Chromium dpkg-shlibdeps file. + const dpkgShlibdepsUrl = 'https://raw.githubusercontent.com/chromium/chromium/main/third_party/dpkg-shlibdeps/dpkg-shlibdeps.pl'; + const dpkgShlibdepsScriptLocation = `${(0, os_1.tmpdir)()}/dpkg-shlibdeps.pl`; + const result = (0, child_process_1.spawnSync)('curl', [dpkgShlibdepsUrl, '-o', dpkgShlibdepsScriptLocation]); + if (result.status !== 0) { + throw new Error('Cannot retrieve dpkg-shlibdeps. Stderr:\n' + result.stderr); + } + const cmd = [dpkgShlibdepsScriptLocation, '--ignore-weak-undefined']; + switch (arch) { + case 'amd64': + cmd.push(`-l${sysroot}/usr/lib/x86_64-linux-gnu`, `-l${sysroot}/lib/x86_64-linux-gnu`); + break; + case 'armhf': + cmd.push(`-l${sysroot}/usr/lib/arm-linux-gnueabihf`, `-l${sysroot}/lib/arm-linux-gnueabihf`); + break; + case 'arm64': + cmd.push(`-l${sysroot}/usr/lib/aarch64-linux-gnu`, `-l${sysroot}/lib/aarch64-linux-gnu`); + break; + default: + throw new Error('Unsupported architecture ' + arch); + } + cmd.push(`-l${sysroot}/usr/lib`); + cmd.push('-O', '-e', path.resolve(binaryPath)); + const dpkgShlibdepsResult = (0, child_process_1.spawnSync)('perl', cmd, { cwd: sysroot }); + if (dpkgShlibdepsResult.status !== 0) { + throw new Error(`dpkg-shlibdeps failed with exit code ${dpkgShlibdepsResult.status}. stderr:\n${dpkgShlibdepsResult.stderr} `); + } + const shlibsDependsPrefix = 'shlibs:Depends='; + const requiresList = dpkgShlibdepsResult.stdout.toString('utf-8').trimEnd().split('\n'); + let depsStr = ''; + for (const line of requiresList) { + if (line.startsWith(shlibsDependsPrefix)) { + depsStr = line.substring(shlibsDependsPrefix.length); + } + } + const requires = new Set(depsStr.split(', ').sort()); + return requires; +} +// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/rpm/merge_package_deps.py. +function mergePackageDeps(inputDeps) { + // For now, see if directly appending the dependencies helps. + const requires = new Set(); + for (const depSet of inputDeps) { + for (const dep of depSet) { + const trimmedDependency = dep.trim(); + if (trimmedDependency.length && !trimmedDependency.startsWith('#')) { + requires.add(trimmedDependency); + } + } + } + return requires; +} diff --git a/build/linux/debian/dependencies-generator.ts b/build/linux/debian/dependencies-generator.ts new file mode 100644 index 00000000000..8e1c3fc042f --- /dev/null +++ b/build/linux/debian/dependencies-generator.ts @@ -0,0 +1,145 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +'use strict'; + +import { spawnSync } from 'child_process'; +import { constants, statSync } from 'fs'; +import { tmpdir } from 'os'; +import path = require('path'); +import { additionalDeps, bundledDeps, referenceGeneratedDepsByArch } from './dep-lists'; +import { ArchString } from './types'; + +// A flag that can easily be toggled. +// Make sure to compile the build directory after toggling the value. +// If false, we warn about new dependencies if they show up +// while running the Debian prepare package task for a release. +// If true, we fail the build if there are new dependencies found during that task. +// The reference dependencies, which one has to update when the new dependencies +// are valid, are in dep-lists.ts +const FAIL_BUILD_FOR_NEW_DEPENDENCIES: boolean = true; + +export function getDependencies(buildDir: string, applicationName: string, arch: ArchString, sysroot: string): string[] { + // Get the files for which we want to find dependencies. + const nativeModulesPath = path.join(buildDir, 'resources', 'app', 'node_modules.asar.unpacked'); + const findResult = spawnSync('find', [nativeModulesPath, '-name', '*.node']); + if (findResult.status) { + console.error('Error finding files:'); + console.error(findResult.stderr.toString()); + return []; + } + + const files = findResult.stdout.toString().trimEnd().split('\n'); + + const appPath = path.join(buildDir, applicationName); + files.push(appPath); + + // Add chrome sandbox and crashpad handler. + files.push(path.join(buildDir, 'chrome-sandbox')); + files.push(path.join(buildDir, 'chrome_crashpad_handler')); + + // Generate the dependencies. + const dependencies: Set[] = files.map((file) => calculatePackageDeps(file, arch, sysroot)); + // Add additional dependencies. + const additionalDepsSet = new Set(additionalDeps); + dependencies.push(additionalDepsSet); + + // Merge all the dependencies. + const mergedDependencies = mergePackageDeps(dependencies); + let sortedDependencies: string[] = []; + for (const dependency of mergedDependencies) { + sortedDependencies.push(dependency); + } + sortedDependencies.sort(); + + // Exclude bundled dependencies + sortedDependencies = sortedDependencies.filter(dependency => { + return !bundledDeps.some(bundledDep => dependency.startsWith(bundledDep)); + }); + + const referenceGeneratedDeps = referenceGeneratedDepsByArch[arch]; + if (JSON.stringify(sortedDependencies) !== JSON.stringify(referenceGeneratedDeps)) { + const failMessage = 'The dependencies list has changed.' + + '\nOld:\n' + referenceGeneratedDeps.join('\n') + + '\nNew:\n' + sortedDependencies.join('\n'); + if (FAIL_BUILD_FOR_NEW_DEPENDENCIES) { + throw new Error(failMessage); + } else { + console.warn(failMessage); + } + } + + return sortedDependencies; +} + +// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/debian/calculate_package_deps.py. +function calculatePackageDeps(binaryPath: string, arch: ArchString, sysroot: string): Set { + try { + if (!(statSync(binaryPath).mode & constants.S_IXUSR)) { + throw new Error(`Binary ${binaryPath} needs to have an executable bit set.`); + } + } catch (e) { + // The package might not exist. Don't re-throw the error here. + console.error('Tried to stat ' + binaryPath + ' but failed.'); + } + + // Get the Chromium dpkg-shlibdeps file. + const dpkgShlibdepsUrl = 'https://raw.githubusercontent.com/chromium/chromium/main/third_party/dpkg-shlibdeps/dpkg-shlibdeps.pl'; + const dpkgShlibdepsScriptLocation = `${tmpdir()}/dpkg-shlibdeps.pl`; + const result = spawnSync('curl', [dpkgShlibdepsUrl, '-o', dpkgShlibdepsScriptLocation]); + if (result.status !== 0) { + throw new Error('Cannot retrieve dpkg-shlibdeps. Stderr:\n' + result.stderr); + } + const cmd = [dpkgShlibdepsScriptLocation, '--ignore-weak-undefined']; + switch (arch) { + case 'amd64': + cmd.push(`-l${sysroot}/usr/lib/x86_64-linux-gnu`, + `-l${sysroot}/lib/x86_64-linux-gnu`); + break; + case 'armhf': + cmd.push(`-l${sysroot}/usr/lib/arm-linux-gnueabihf`, + `-l${sysroot}/lib/arm-linux-gnueabihf`); + break; + case 'arm64': + cmd.push(`-l${sysroot}/usr/lib/aarch64-linux-gnu`, + `-l${sysroot}/lib/aarch64-linux-gnu`); + break; + default: + throw new Error('Unsupported architecture ' + arch); + } + cmd.push(`-l${sysroot}/usr/lib`); + cmd.push('-O', '-e', path.resolve(binaryPath)); + + const dpkgShlibdepsResult = spawnSync('perl', cmd, { cwd: sysroot }); + if (dpkgShlibdepsResult.status !== 0) { + throw new Error(`dpkg-shlibdeps failed with exit code ${dpkgShlibdepsResult.status}. stderr:\n${dpkgShlibdepsResult.stderr} `); + } + + const shlibsDependsPrefix = 'shlibs:Depends='; + const requiresList = dpkgShlibdepsResult.stdout.toString('utf-8').trimEnd().split('\n'); + let depsStr = ''; + for (const line of requiresList) { + if (line.startsWith(shlibsDependsPrefix)) { + depsStr = line.substring(shlibsDependsPrefix.length); + } + } + const requires = new Set(depsStr.split(', ').sort()); + return requires; +} + +// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/rpm/merge_package_deps.py. +function mergePackageDeps(inputDeps: Set[]): Set { + // For now, see if directly appending the dependencies helps. + const requires = new Set(); + for (const depSet of inputDeps) { + for (const dep of depSet) { + const trimmedDependency = dep.trim(); + if (trimmedDependency.length && !trimmedDependency.startsWith('#')) { + requires.add(trimmedDependency); + } + } + } + return requires; +} diff --git a/build/linux/debian/install-sysroot.js b/build/linux/debian/install-sysroot.js new file mode 100644 index 00000000000..c6b57a7b8a3 --- /dev/null +++ b/build/linux/debian/install-sysroot.js @@ -0,0 +1,81 @@ +"use strict"; +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.getSysroot = void 0; +const child_process_1 = require("child_process"); +const crypto_1 = require("crypto"); +const os_1 = require("os"); +const fs = require("fs"); +const https = require("https"); +const path = require("path"); +const sysroots_1 = require("./sysroots"); +// Based on https://source.chromium.org/chromium/chromium/src/+/main:build/linux/sysroot_scripts/install-sysroot.py. +const URL_PREFIX = 'https://msftelectron.blob.core.windows.net'; +const URL_PATH = 'sysroots/toolchain'; +function getSha(filename) { + const hash = (0, crypto_1.createHash)('sha1'); + // Read file 1 MB at a time + const fd = fs.openSync(filename, 'r'); + const buffer = Buffer.alloc(1024 * 1024); + let position = 0; + let bytesRead = 0; + while ((bytesRead = fs.readSync(fd, buffer, 0, buffer.length, position)) === buffer.length) { + hash.update(buffer); + position += bytesRead; + } + hash.update(buffer.slice(0, bytesRead)); + return hash.digest('hex'); +} +async function getSysroot(arch) { + const sysrootDict = sysroots_1.sysrootInfo[arch]; + const tarballFilename = sysrootDict['Tarball']; + const tarballSha = sysrootDict['Sha1Sum']; + const sysroot = path.join((0, os_1.tmpdir)(), sysrootDict['SysrootDir']); + const url = [URL_PREFIX, URL_PATH, tarballSha, tarballFilename].join('/'); + const stamp = path.join(sysroot, '.stamp'); + if (fs.existsSync(stamp) && fs.readFileSync(stamp).toString() === url) { + return sysroot; + } + console.log(`Installing Debian ${arch} root image: ${sysroot}`); + fs.rmSync(sysroot, { recursive: true, force: true }); + fs.mkdirSync(sysroot); + const tarball = path.join(sysroot, tarballFilename); + console.log(`Downloading ${url}`); + let downloadSuccess = false; + for (let i = 0; i < 3 && !downloadSuccess; i++) { + await new Promise((c) => { + https.get(url, (res) => { + const chunks = []; + res.on('data', (chunk) => { + chunks.push(chunk); + }); + res.on('end', () => { + fs.writeFileSync(tarball, Buffer.concat(chunks)); + downloadSuccess = true; + c(); + }); + }).on('error', (err) => { + console.error('Encountered an error during the download attempt: ' + err.message); + c(); + }); + }); + } + if (!downloadSuccess) { + throw new Error('Failed to download ' + url); + } + const sha = getSha(tarball); + if (sha !== tarballSha) { + throw new Error(`Tarball sha1sum is wrong. Expected ${tarballSha}, actual ${sha}`); + } + const proc = (0, child_process_1.spawnSync)('tar', ['xf', tarball, '-C', sysroot]); + if (proc.status) { + throw new Error('Tarball extraction failed with code ' + proc.status); + } + fs.rmSync(tarball); + fs.writeFileSync(stamp, url); + return sysroot; +} +exports.getSysroot = getSysroot; diff --git a/build/linux/debian/install-sysroot.ts b/build/linux/debian/install-sysroot.ts new file mode 100644 index 00000000000..7c04c44f730 --- /dev/null +++ b/build/linux/debian/install-sysroot.ts @@ -0,0 +1,90 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { spawnSync } from 'child_process'; +import { createHash } from 'crypto'; +import { tmpdir } from 'os'; +import * as fs from 'fs'; +import * as https from 'https'; +import * as path from 'path'; +import { sysrootInfo } from './sysroots'; +import { ArchString } from './types'; + +// Based on https://source.chromium.org/chromium/chromium/src/+/main:build/linux/sysroot_scripts/install-sysroot.py. +const URL_PREFIX = 'https://msftelectron.blob.core.windows.net'; +const URL_PATH = 'sysroots/toolchain'; + +function getSha(filename: fs.PathLike): string { + const hash = createHash('sha1'); + // Read file 1 MB at a time + const fd = fs.openSync(filename, 'r'); + const buffer = Buffer.alloc(1024 * 1024); + let position = 0; + let bytesRead = 0; + while ((bytesRead = fs.readSync(fd, buffer, 0, buffer.length, position)) === buffer.length) { + hash.update(buffer); + position += bytesRead; + } + hash.update(buffer.slice(0, bytesRead)); + return hash.digest('hex'); +} + +type SysrootDictEntry = { + Sha1Sum: string; + SysrootDir: string; + Tarball: string; +}; + +export async function getSysroot(arch: ArchString): Promise { + const sysrootDict: SysrootDictEntry = sysrootInfo[arch]; + const tarballFilename = sysrootDict['Tarball']; + const tarballSha = sysrootDict['Sha1Sum']; + const sysroot = path.join(tmpdir(), sysrootDict['SysrootDir']); + const url = [URL_PREFIX, URL_PATH, tarballSha, tarballFilename].join('/'); + const stamp = path.join(sysroot, '.stamp'); + if (fs.existsSync(stamp) && fs.readFileSync(stamp).toString() === url) { + return sysroot; + } + + console.log(`Installing Debian ${arch} root image: ${sysroot}`); + fs.rmSync(sysroot, { recursive: true, force: true }); + fs.mkdirSync(sysroot); + const tarball = path.join(sysroot, tarballFilename); + console.log(`Downloading ${url}`); + let downloadSuccess = false; + for (let i = 0; i < 3 && !downloadSuccess; i++) { + await new Promise((c) => { + https.get(url, (res) => { + const chunks: Uint8Array[] = []; + res.on('data', (chunk) => { + chunks.push(chunk); + }); + res.on('end', () => { + fs.writeFileSync(tarball, Buffer.concat(chunks)); + downloadSuccess = true; + c(); + }); + }).on('error', (err) => { + console.error('Encountered an error during the download attempt: ' + err.message); + c(); + }); + }); + } + if (!downloadSuccess) { + throw new Error('Failed to download ' + url); + } + const sha = getSha(tarball); + if (sha !== tarballSha) { + throw new Error(`Tarball sha1sum is wrong. Expected ${tarballSha}, actual ${sha}`); + } + + const proc = spawnSync('tar', ['xf', tarball, '-C', sysroot]); + if (proc.status) { + throw new Error('Tarball extraction failed with code ' + proc.status); + } + fs.rmSync(tarball); + fs.writeFileSync(stamp, url); + return sysroot; +} diff --git a/build/linux/debian/sysroots.js b/build/linux/debian/sysroots.js new file mode 100644 index 00000000000..3825de44028 --- /dev/null +++ b/build/linux/debian/sysroots.js @@ -0,0 +1,26 @@ +"use strict"; +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.sysrootInfo = void 0; +// Based on https://github.com/electron/electron/blob/main/script/sysroots.json, +// which itself is based on https://source.chromium.org/chromium/chromium/src/+/main:build/linux/sysroot_scripts/sysroots.json. +exports.sysrootInfo = { + 'amd64': { + 'Sha1Sum': '7e008cea9eae822d80d55c67fbb5ef4204678e74', + 'SysrootDir': 'debian_sid_amd64-sysroot', + 'Tarball': 'debian_sid_amd64_sysroot.tar.xz' + }, + 'armhf': { + 'Sha1Sum': 'b6f4bb07817bea91b06514a9c1e3832df5a90dbf', + 'SysrootDir': 'debian_sid_arm-sysroot', + 'Tarball': 'debian_sid_arm_sysroot.tar.xz' + }, + 'arm64': { + 'Sha1Sum': '5a56c1ef714154ea5003bcafb16f21b0f8dde023', + 'SysrootDir': 'debian_sid_arm64-sysroot', + 'Tarball': 'debian_sid_arm64_sysroot.tar.xz' + } +}; diff --git a/build/linux/debian/sysroots.ts b/build/linux/debian/sysroots.ts new file mode 100644 index 00000000000..5fda6e6c7bb --- /dev/null +++ b/build/linux/debian/sysroots.ts @@ -0,0 +1,24 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Based on https://github.com/electron/electron/blob/main/script/sysroots.json, +// which itself is based on https://source.chromium.org/chromium/chromium/src/+/main:build/linux/sysroot_scripts/sysroots.json. +export const sysrootInfo = { + 'amd64': { + 'Sha1Sum': '7e008cea9eae822d80d55c67fbb5ef4204678e74', + 'SysrootDir': 'debian_sid_amd64-sysroot', + 'Tarball': 'debian_sid_amd64_sysroot.tar.xz' + }, + 'armhf': { + 'Sha1Sum': 'b6f4bb07817bea91b06514a9c1e3832df5a90dbf', + 'SysrootDir': 'debian_sid_arm-sysroot', + 'Tarball': 'debian_sid_arm_sysroot.tar.xz' + }, + 'arm64': { + 'Sha1Sum': '5a56c1ef714154ea5003bcafb16f21b0f8dde023', + 'SysrootDir': 'debian_sid_arm64-sysroot', + 'Tarball': 'debian_sid_arm64_sysroot.tar.xz' + } +}; diff --git a/build/linux/debian/types.js b/build/linux/debian/types.js new file mode 100644 index 00000000000..56d4e6c56ce --- /dev/null +++ b/build/linux/debian/types.js @@ -0,0 +1,6 @@ +"use strict"; +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +Object.defineProperty(exports, "__esModule", { value: true }); diff --git a/build/linux/debian/types.ts b/build/linux/debian/types.ts new file mode 100644 index 00000000000..ae2347cb12b --- /dev/null +++ b/build/linux/debian/types.ts @@ -0,0 +1,6 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export type ArchString = 'amd64' | 'armhf' | 'arm64'; diff --git a/build/linux/rpm/dependencies-generator.js b/build/linux/rpm/dependencies-generator.js index 3b1e404129c..90cfca96791 100644 --- a/build/linux/rpm/dependencies-generator.js +++ b/build/linux/rpm/dependencies-generator.js @@ -81,7 +81,7 @@ function calculatePackageDeps(binaryPath) { const requires = new Set(findRequiresResult.stdout.toString('utf-8').trimEnd().split('\n')); return requires; } -// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/rpm/merge_package_deps.py +// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/rpm/merge_package_deps.py. function mergePackageDeps(inputDeps) { const requires = new Set(); for (const depSet of inputDeps) { diff --git a/build/linux/rpm/dependencies-generator.ts b/build/linux/rpm/dependencies-generator.ts index e33210284dc..4b84640e280 100644 --- a/build/linux/rpm/dependencies-generator.ts +++ b/build/linux/rpm/dependencies-generator.ts @@ -92,7 +92,7 @@ function calculatePackageDeps(binaryPath: string): Set { return requires; } -// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/rpm/merge_package_deps.py +// Based on https://source.chromium.org/chromium/chromium/src/+/main:chrome/installer/linux/rpm/merge_package_deps.py. function mergePackageDeps(inputDeps: Set[]): Set { const requires = new Set(); for (const depSet of inputDeps) { diff --git a/resources/linux/debian/control.template b/resources/linux/debian/control.template index bfe0aa52da2..1a5981bb219 100644 --- a/resources/linux/debian/control.template +++ b/resources/linux/debian/control.template @@ -1,7 +1,8 @@ Package: @@NAME@@ Version: @@VERSION@@ Section: devel -Depends: libnss3 (>= 2:3.26), gnupg, apt, libxkbfile1, libsecret-1-0, libgtk-3-0 (>= 3.10.0), libxss1, libgbm1 +Depends: @@DEPENDS@@ +Recommends: @@RECOMMENDS@@ Priority: optional Architecture: @@ARCHITECTURE@@ Maintainer: Microsoft Corporation