From 2c5ff0bd1a0eaa4ea5facbc85715ab8f15104e10 Mon Sep 17 00:00:00 2001 From: Ryan Macnak Date: Mon, 8 Aug 2022 23:05:31 +0000 Subject: [PATCH] [build] Update Linux sysroots from jessie to bullseye. Change-Id: I90022916b13e61717b7e38c6ccab914ae1a5c18e Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/254080 Commit-Queue: Ryan Macnak Reviewed-by: Alexander Aprelev --- DEPS | 24 +- build/.gitignore | 10 +- build/config/sysroot.gni | 8 +- .../linux/sysroot_scripts/install-sysroot.py | 250 +++++++++--------- build/linux/sysroot_scripts/sysroots.json | 63 ++--- 5 files changed, 174 insertions(+), 181 deletions(-) diff --git a/DEPS b/DEPS index 6d9761358d5..5b3f14b1c58 100644 --- a/DEPS +++ b/DEPS @@ -669,32 +669,32 @@ hooks = [ 'action': ['python3', 'sdk/tools/generate_sdk_version_file.py'], }, { - # Pull Debian sysroot for i386 Linux - 'name': 'sysroot_i386', + 'name': 'sysroot_arm', 'pattern': '.', + 'condition': 'checkout_linux', 'action': ['python3', 'sdk/build/linux/sysroot_scripts/install-sysroot.py', - '--arch', 'i386'], + '--arch=arm'], }, { - # Pull Debian sysroot for amd64 Linux - 'name': 'sysroot_amd64', + 'name': 'sysroot_arm64', 'pattern': '.', + 'condition': 'checkout_linux', 'action': ['python3', 'sdk/build/linux/sysroot_scripts/install-sysroot.py', - '--arch', 'amd64'], + '--arch=arm64'], }, { - # Pull Debian sysroot for arm Linux - 'name': 'sysroot_amd64', + 'name': 'sysroot_x86', 'pattern': '.', + 'condition': 'checkout_linux', 'action': ['python3', 'sdk/build/linux/sysroot_scripts/install-sysroot.py', - '--arch', 'arm'], + '--arch=x86'], }, { - # Pull Debian jessie sysroot for arm64 Linux - 'name': 'sysroot_amd64', + 'name': 'sysroot_x64', 'pattern': '.', + 'condition': 'checkout_linux', 'action': ['python3', 'sdk/build/linux/sysroot_scripts/install-sysroot.py', - '--arch', 'arm64'], + '--arch=x64'], }, { 'name': 'buildtools', diff --git a/build/.gitignore b/build/.gitignore index f7bbb04e896..73494beca3a 100644 --- a/build/.gitignore +++ b/build/.gitignore @@ -1,8 +1,8 @@ # Generated file containing information about the VS toolchain on Windows win_toolchain.json -# Pulled Debian jessie sysroots -linux/debian_jessie_arm-sysroot -linux/debian_jessie_arm64-sysroot -linux/debian_jessie_amd64-sysroot -linux/debian_jessie_i386-sysroot +# Pulled Debian bullseye sysroots +linux/debian_bullseye_arm-sysroot +linux/debian_bullseye_arm64-sysroot +linux/debian_bullseye_amd64-sysroot +linux/debian_bullseye_i386-sysroot diff --git a/build/config/sysroot.gni b/build/config/sysroot.gni index c4ee9ad8689..7fb6890cffd 100644 --- a/build/config/sysroot.gni +++ b/build/config/sysroot.gni @@ -17,16 +17,16 @@ declare_args() { if (is_linux && dart_use_debian_sysroot) { if (current_cpu == "x86") { target_sysroot = - rebase_path("//build/linux/debian_jessie_i386-sysroot", root_build_dir) + rebase_path("//build/linux/debian_bullseye_i386-sysroot", root_build_dir) } else if (current_cpu == "x64") { target_sysroot = - rebase_path("//build/linux/debian_jessie_amd64-sysroot", root_build_dir) + rebase_path("//build/linux/debian_bullseye_amd64-sysroot", root_build_dir) } else if (current_cpu == "arm") { target_sysroot = - rebase_path("//build/linux/debian_jessie_arm-sysroot", root_build_dir) + rebase_path("//build/linux/debian_bullseye_arm-sysroot", root_build_dir) } else if (current_cpu == "arm64") { target_sysroot = - rebase_path("//build/linux/debian_jessie_arm64-sysroot", root_build_dir) + rebase_path("//build/linux/debian_bullseye_arm64-sysroot", root_build_dir) } else { print("There is no Debian sysroot present for $current_cpu") assert(false) diff --git a/build/linux/sysroot_scripts/install-sysroot.py b/build/linux/sysroot_scripts/install-sysroot.py index 5e2fffec584..eaa1c2edfd6 100755 --- a/build/linux/sysroot_scripts/install-sysroot.py +++ b/build/linux/sysroot_scripts/install-sysroot.py @@ -2,169 +2,169 @@ # Copyright (c) 2013 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. + """Install Debian sysroots for building chromium. """ -# The sysroot is needed to ensure that binaries will run on Debian Jessie, -# the oldest supported linux distribution. This script can be run manually -# but is more often run as part of gclient hooks. When run from hooks this -# script is a no-op on non-linux platforms. +# The sysroot is needed to ensure that binaries that get built will run on +# the oldest stable version of Debian that we currently support. +# This script can be run manually but is more often run as part of gclient +# hooks. When run from hooks this script is a no-op on non-linux platforms. -# The sysroot image could be constructed from scratch based on the current -# state or Debian Jessie but for consistency we currently use a pre-built root -# image. The image will normally need to be rebuilt every time chrome's build -# dependencies are changed. +# The sysroot image could be constructed from scratch based on the current state +# of the Debian archive but for consistency we use a pre-built root image (we +# don't want upstream changes to Debian to effect the chromium build until we +# choose to pull them in). The images will normally need to be rebuilt every +# time chrome's build dependencies are changed but should also be updated +# periodically to include upstream security fixes from Debian. + +# This script looks at sysroots.json next to it to find the name of a .tar.xz +# to download and the location to extract it to. The extracted sysroot could for +# example be in build/linux/debian_bullseye_amd64-sysroot/. + +from __future__ import print_function import hashlib import json +import platform import optparse import os +import re import shutil import subprocess import sys -from urllib.request import urlopen +try: + # For Python 3.0 and later + from urllib.request import urlopen +except ImportError: + # Fall back to Python 2's urllib2 + from urllib2 import urlopen SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.dirname(os.path.dirname(SCRIPT_DIR))) -import detect_host_arch URL_PREFIX = 'https://commondatastorage.googleapis.com' URL_PATH = 'chrome-linux-sysroot/toolchain' -VALID_ARCHS = ('arm', 'arm64', 'i386', 'amd64', 'mips') +VALID_ARCHS = ('arm', 'arm64', 'i386', 'amd64', 'mips', 'mips64el') + +ARCH_TRANSLATIONS = { + 'x64': 'amd64', + 'x86': 'i386', + 'mipsel': 'mips', + 'mips64': 'mips64el', +} + +DEFAULT_TARGET_PLATFORM = 'bullseye' class Error(Exception): - pass + pass def GetSha1(filename): - sha1 = hashlib.sha1() - with open(filename, 'rb') as f: - while True: - # Read in 1mb chunks, so it doesn't all have to be loaded into memory. - chunk = f.read(1024 * 1024) - if not chunk: - break - sha1.update(chunk) - return sha1.hexdigest() - - -def DetectHostArch(): - # Figure out host arch using build/detect_host_arch.py and - # set target_arch to host arch - detected_host_arch = detect_host_arch.HostArch() - if detected_host_arch == 'x64': - return 'amd64' - elif detected_host_arch == 'ia32': - return 'i386' - elif detected_host_arch == 'arm': - return 'arm' - elif detected_host_arch == 'arm64': - return 'arm64' - elif detected_host_arch == 'mips': - return 'mips' - elif detected_host_arch == 'ppc': - return 'ppc' - elif detected_host_arch == 's390': - return 's390' - - raise Error('Unrecognized host arch: %s' % detected_host_arch) + sha1 = hashlib.sha1() + with open(filename, 'rb') as f: + while True: + # Read in 1mb chunks, so it doesn't all have to be loaded into memory. + chunk = f.read(1024*1024) + if not chunk: + break + sha1.update(chunk) + return sha1.hexdigest() def main(args): - parser = optparse.OptionParser( - 'usage: %prog [OPTIONS]', description=__doc__) - parser.add_option( - '--arch', - type='choice', - choices=VALID_ARCHS, - help='Sysroot architecture: %s' % ', '.join(VALID_ARCHS)) - options, _ = parser.parse_args(args) - if not sys.platform.startswith('linux'): - return 0 - - if not options.arch: - print('You much specify either --arch or --running-as-hook') - return 1 - InstallDefaultSysrootForArch(options.arch) + parser = optparse.OptionParser('usage: %prog [OPTIONS]', description=__doc__) + parser.add_option('--arch', + help='Sysroot architecture: %s' % ', '.join(VALID_ARCHS)) + parser.add_option('--all', action='store_true', + help='Install all sysroot images (useful when updating the' + ' images)') + parser.add_option('--print-hash', + help='Print the hash of the sysroot for the given arch.') + options, _ = parser.parse_args(args) + if options.print_hash: + arch = options.print_hash + print(GetSysrootDict(DEFAULT_TARGET_PLATFORM, + ARCH_TRANSLATIONS.get(arch, arch))['Sha1Sum']) return 0 + if options.arch: + InstallSysroot(DEFAULT_TARGET_PLATFORM, + ARCH_TRANSLATIONS.get(options.arch, options.arch)) + elif options.all: + for arch in VALID_ARCHS: + InstallSysroot(DEFAULT_TARGET_PLATFORM, arch) + else: + print('You much specify one of the options.') + return 1 + + return 0 -def InstallDefaultSysrootForArch(target_arch): - if target_arch == 'amd64': - InstallSysroot('Jessie', 'amd64') - elif target_arch == 'arm': - InstallSysroot('Jessie', 'arm') - elif target_arch == 'arm64': - InstallSysroot('Jessie', 'arm64') - elif target_arch == 'i386': - InstallSysroot('Jessie', 'i386') - elif target_arch == 'mips': - InstallSysroot('Jessie', 'mips') - else: - raise Error('Unknown architecture: %s' % target_arch) +def GetSysrootDict(target_platform, target_arch): + if target_arch not in VALID_ARCHS: + raise Error('Unknown architecture: %s' % target_arch) + + sysroots_file = os.path.join(SCRIPT_DIR, 'sysroots.json') + sysroots = json.load(open(sysroots_file)) + sysroot_key = '%s_%s' % (target_platform, target_arch) + if sysroot_key not in sysroots: + raise Error('No sysroot for: %s %s' % (target_platform, target_arch)) + return sysroots[sysroot_key] def InstallSysroot(target_platform, target_arch): - # The sysroot directory should match the one specified in build/common.gypi. - # TODO(thestig) Consider putting this elsewhere to avoid having to recreate - # it on every build. - linux_dir = os.path.dirname(SCRIPT_DIR) + sysroot_dict = GetSysrootDict(target_platform, target_arch) + tarball_filename = sysroot_dict['Tarball'] + tarball_sha1sum = sysroot_dict['Sha1Sum'] + # TODO(thestig) Consider putting this elsewhere to avoid having to recreate + # it on every build. + linux_dir = os.path.dirname(SCRIPT_DIR) + sysroot = os.path.join(linux_dir, sysroot_dict['SysrootDir']) - sysroots_file = os.path.join(SCRIPT_DIR, 'sysroots.json') - sysroots = json.load(open(sysroots_file)) - sysroot_key = '%s_%s' % (target_platform.lower(), target_arch) - if sysroot_key not in sysroots: - raise Error('No sysroot for: %s %s' % (target_platform, target_arch)) - sysroot_dict = sysroots[sysroot_key] - revision = sysroot_dict['Revision'] - tarball_filename = sysroot_dict['Tarball'] - tarball_sha1sum = sysroot_dict['Sha1Sum'] - sysroot = os.path.join(linux_dir, sysroot_dict['SysrootDir']) + url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, tarball_sha1sum, + tarball_filename) - url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, revision, tarball_filename) + stamp = os.path.join(sysroot, '.stamp') + if os.path.exists(stamp): + with open(stamp) as s: + if s.read() == url: + return - stamp = os.path.join(sysroot, '.stamp') - if os.path.exists(stamp): - with open(stamp) as s: - if s.read() == url: - return + print('Installing Debian %s %s root image: %s' % \ + (target_platform, target_arch, sysroot)) + if os.path.isdir(sysroot): + shutil.rmtree(sysroot) + os.mkdir(sysroot) + tarball = os.path.join(sysroot, tarball_filename) + print('Downloading %s' % url) + sys.stdout.flush() + sys.stderr.flush() + for _ in range(3): + try: + response = urlopen(url) + with open(tarball, "wb") as f: + f.write(response.read()) + break + except Exception: # Ignore exceptions. + pass + else: + raise Error('Failed to download %s' % url) + sha1sum = GetSha1(tarball) + if sha1sum != tarball_sha1sum: + raise Error('Tarball sha1sum is wrong.' + 'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum)) + subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot]) + os.remove(tarball) - print('Installing Debian %s %s root image: %s' % \ - (target_platform, target_arch, sysroot)) - if os.path.isdir(sysroot): - shutil.rmtree(sysroot) - os.mkdir(sysroot) - tarball = os.path.join(sysroot, tarball_filename) - print('Downloading %s' % url) - sys.stdout.flush() - sys.stderr.flush() - for _ in range(3): - try: - response = urlopen(url) - with open(tarball, "wb") as f: - f.write(response.read()) - break - except: - pass - else: - raise Error('Failed to download %s' % url) - sha1sum = GetSha1(tarball) - if sha1sum != tarball_sha1sum: - raise Error('Tarball sha1sum is wrong.' - 'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum)) - subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot]) - os.remove(tarball) - - with open(stamp, 'w') as s: - s.write(url) + with open(stamp, 'w') as s: + s.write(url) if __name__ == '__main__': - try: - sys.exit(main(sys.argv[1:])) - except Error as e: - sys.stderr.write(str(e) + '\n') - sys.exit(1) + try: + sys.exit(main(sys.argv[1:])) + except Error as e: + sys.stderr.write(str(e) + '\n') + sys.exit(1) diff --git a/build/linux/sysroot_scripts/sysroots.json b/build/linux/sysroot_scripts/sysroots.json index 5688cd052bc..e851f4229cc 100644 --- a/build/linux/sysroot_scripts/sysroots.json +++ b/build/linux/sysroot_scripts/sysroots.json @@ -1,44 +1,37 @@ { - "jessie_amd64": { - "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f", - "Sha1Sum": "046a82b027f069bdde98424850490c315e2f31f9", - "SysrootDir": "debian_jessie_amd64-sysroot", - "Tarball": "debian_jessie_amd64_sysroot.tgz" + "bullseye_amd64": { + "Sha1Sum": "cb4fa34f1faddafb72cace35faf62a611f2ca7c9", + "SysrootDir": "debian_bullseye_amd64-sysroot", + "Tarball": "debian_bullseye_amd64_sysroot.tar.xz" }, - "jessie_arm": { - "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f", - "Sha1Sum": "7addeb28195f7f367cbaba46cdba99925a0646b2", - "SysrootDir": "debian_jessie_arm-sysroot", - "Tarball": "debian_jessie_arm_sysroot.tgz" + "bullseye_arm": { + "Sha1Sum": "a1084eba943f797a1c9aee8d5f8213ed680315a8", + "SysrootDir": "debian_bullseye_arm-sysroot", + "Tarball": "debian_bullseye_arm_sysroot.tar.xz" }, - "jessie_arm64": { - "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f", - "Sha1Sum": "1b853d8082321256fe646589558da3980875d767", - "SysrootDir": "debian_jessie_arm64-sysroot", - "Tarball": "debian_jessie_arm64_sysroot.tgz" + "bullseye_arm64": { + "Sha1Sum": "f00ece500aef0ff2a431a07de48bd3e1aa6d1caf", + "SysrootDir": "debian_bullseye_arm64-sysroot", + "Tarball": "debian_bullseye_arm64_sysroot.tar.xz" }, - "jessie_i386": { - "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f", - "Sha1Sum": "7edfe877ea71473def2d9e6958474d76c9ff98b6", - "SysrootDir": "debian_jessie_i386-sysroot", - "Tarball": "debian_jessie_i386_sysroot.tgz" + "bullseye_armel": { + "Sha1Sum": "54ccf4898fe923b22141c9e21e80b455dec47ecf", + "SysrootDir": "debian_bullseye_armel-sysroot", + "Tarball": "debian_bullseye_armel_sysroot.tar.xz" }, - "jessie_mips": { - "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f", - "Sha1Sum": "3d0c5640a39e264d7180df2bac1192308f8a256e", - "SysrootDir": "debian_jessie_mips-sysroot", - "Tarball": "debian_jessie_mips_sysroot.tgz" + "bullseye_i386": { + "Sha1Sum": "07bb72368889fea19a1f123aa11528f0d792121b", + "SysrootDir": "debian_bullseye_i386-sysroot", + "Tarball": "debian_bullseye_i386_sysroot.tar.xz" }, - "precise_amd64": { - "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f", - "Sha1Sum": "69e1d3c9efda92ba265ce7bb653be546e8dde7e4", - "SysrootDir": "ubuntu_precise_amd64-sysroot", - "Tarball": "ubuntu_precise_amd64_sysroot.tgz" + "bullseye_mips": { + "Sha1Sum": "f19f07f7e62edbf837d6e7f173aea3f47110c2ba", + "SysrootDir": "debian_bullseye_mips-sysroot", + "Tarball": "debian_bullseye_mips_sysroot.tar.xz" }, - "trusty_arm": { - "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f", - "Sha1Sum": "d1591e8fc6a273af1fbf3d1a864f098081d06dd1", - "SysrootDir": "ubuntu_trusty_arm-sysroot", - "Tarball": "ubuntu_trusty_arm_sysroot.tgz" + "bullseye_mips64el": { + "Sha1Sum": "3cfc9e7fe2d0cad3873bbce8fc3ffde510ed5638", + "SysrootDir": "debian_bullseye_mips64el-sysroot", + "Tarball": "debian_bullseye_mips64el_sysroot.tar.xz" } }