[build] Update Linux sysroots from jessie to bullseye.

Change-Id: I90022916b13e61717b7e38c6ccab914ae1a5c18e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/254080
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Aprelev <aam@google.com>

DEPS

@@ -669,32 +669,32 @@ hooks = [
     'action': ['python3', 'sdk/tools/generate_sdk_version_file.py'],
   },
   {
-    # Pull Debian sysroot for i386 Linux
+    'name': 'sysroot_arm',
-    'name': 'sysroot_i386',
     'pattern': '.',
+    'condition': 'checkout_linux',
     'action': ['python3', 'sdk/build/linux/sysroot_scripts/install-sysroot.py',
-               '--arch', 'i386'],
+               '--arch=arm'],
   },
   {
-    # Pull Debian sysroot for amd64 Linux
+    'name': 'sysroot_arm64',
-    'name': 'sysroot_amd64',
     'pattern': '.',
+    'condition': 'checkout_linux',
     'action': ['python3', 'sdk/build/linux/sysroot_scripts/install-sysroot.py',
-               '--arch', 'amd64'],
+               '--arch=arm64'],
   },
   {
-    # Pull Debian sysroot for arm Linux
+    'name': 'sysroot_x86',
-    'name': 'sysroot_amd64',
     'pattern': '.',
+    'condition': 'checkout_linux',
     'action': ['python3', 'sdk/build/linux/sysroot_scripts/install-sysroot.py',
-               '--arch', 'arm'],
+               '--arch=x86'],
   },
   {
-    # Pull Debian jessie sysroot for arm64 Linux
+    'name': 'sysroot_x64',
-    'name': 'sysroot_amd64',
     'pattern': '.',
+    'condition': 'checkout_linux',
     'action': ['python3', 'sdk/build/linux/sysroot_scripts/install-sysroot.py',
-               '--arch', 'arm64'],
+               '--arch=x64'],
   },
   {
     'name': 'buildtools',

build/.gitignore

@@ -1,8 +1,8 @@
 # Generated file containing information about the VS toolchain on Windows
 win_toolchain.json
 
-# Pulled Debian jessie sysroots
+# Pulled Debian bullseye sysroots
-linux/debian_jessie_arm-sysroot
+linux/debian_bullseye_arm-sysroot
-linux/debian_jessie_arm64-sysroot
+linux/debian_bullseye_arm64-sysroot
-linux/debian_jessie_amd64-sysroot
+linux/debian_bullseye_amd64-sysroot
-linux/debian_jessie_i386-sysroot
+linux/debian_bullseye_i386-sysroot

build/config/sysroot.gni

@@ -17,16 +17,16 @@ declare_args() {
 if (is_linux && dart_use_debian_sysroot) {
   if (current_cpu == "x86") {
     target_sysroot =
-        rebase_path("//build/linux/debian_jessie_i386-sysroot", root_build_dir)
+        rebase_path("//build/linux/debian_bullseye_i386-sysroot", root_build_dir)
   } else if (current_cpu == "x64") {
     target_sysroot =
-        rebase_path("//build/linux/debian_jessie_amd64-sysroot", root_build_dir)
+        rebase_path("//build/linux/debian_bullseye_amd64-sysroot", root_build_dir)
   } else if (current_cpu == "arm") {
     target_sysroot =
-        rebase_path("//build/linux/debian_jessie_arm-sysroot", root_build_dir)
+        rebase_path("//build/linux/debian_bullseye_arm-sysroot", root_build_dir)
   } else if (current_cpu == "arm64") {
     target_sysroot =
-        rebase_path("//build/linux/debian_jessie_arm64-sysroot", root_build_dir)
+        rebase_path("//build/linux/debian_bullseye_arm64-sysroot", root_build_dir)
   } else {
     print("There is no Debian sysroot present for $current_cpu")
     assert(false)

build/linux/sysroot_scripts/install-sysroot.py

@@ -2,169 +2,169 @@
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
 """Install Debian sysroots for building chromium.
 """
 
-# The sysroot is needed to ensure that binaries will run on Debian Jessie,
+# The sysroot is needed to ensure that binaries that get built will run on
-# the oldest supported linux distribution. This script can be run manually
+# the oldest stable version of Debian that we currently support.
-# but is more often run as part of gclient hooks. When run from hooks this
+# This script can be run manually but is more often run as part of gclient
-# script is a no-op on non-linux platforms.
+# hooks. When run from hooks this script is a no-op on non-linux platforms.
 
-# The sysroot image could be constructed from scratch based on the current
+# The sysroot image could be constructed from scratch based on the current state
-# state or Debian Jessie but for consistency we currently use a pre-built root
+# of the Debian archive but for consistency we use a pre-built root image (we
-# image. The image will normally need to be rebuilt every time chrome's build
+# don't want upstream changes to Debian to effect the chromium build until we
-# dependencies are changed.
+# choose to pull them in). The images will normally need to be rebuilt every
+# time chrome's build dependencies are changed but should also be updated
+# periodically to include upstream security fixes from Debian.
+
+# This script looks at sysroots.json next to it to find the name of a .tar.xz
+# to download and the location to extract it to. The extracted sysroot could for
+# example be in build/linux/debian_bullseye_amd64-sysroot/.
+
+from __future__ import print_function
 
 import hashlib
 import json
+import platform
 import optparse
 import os
+import re
 import shutil
 import subprocess
 import sys
-from urllib.request import urlopen
+try:
+  # For Python 3.0 and later
+  from urllib.request import urlopen
+except ImportError:
+  # Fall back to Python 2's urllib2
+  from urllib2 import urlopen
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.dirname(os.path.dirname(SCRIPT_DIR)))
-import detect_host_arch
 
 URL_PREFIX = 'https://commondatastorage.googleapis.com'
 URL_PATH = 'chrome-linux-sysroot/toolchain'
 
-VALID_ARCHS = ('arm', 'arm64', 'i386', 'amd64', 'mips')
+VALID_ARCHS = ('arm', 'arm64', 'i386', 'amd64', 'mips', 'mips64el')
+
+ARCH_TRANSLATIONS = {
+    'x64': 'amd64',
+    'x86': 'i386',
+    'mipsel': 'mips',
+    'mips64': 'mips64el',
+}
+
+DEFAULT_TARGET_PLATFORM = 'bullseye'
 
 
 class Error(Exception):
-    pass
+  pass
 
 
 def GetSha1(filename):
-    sha1 = hashlib.sha1()
+  sha1 = hashlib.sha1()
-    with open(filename, 'rb') as f:
+  with open(filename, 'rb') as f:
-        while True:
+    while True:
-            # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
+      # Read in 1mb chunks, so it doesn't all have to be loaded into memory.
-            chunk = f.read(1024 * 1024)
+      chunk = f.read(1024*1024)
-            if not chunk:
+      if not chunk:
-                break
+        break
-            sha1.update(chunk)
+      sha1.update(chunk)
-    return sha1.hexdigest()
+  return sha1.hexdigest()
-
-
-def DetectHostArch():
-    # Figure out host arch using build/detect_host_arch.py and
-    # set target_arch to host arch
-    detected_host_arch = detect_host_arch.HostArch()
-    if detected_host_arch == 'x64':
-        return 'amd64'
-    elif detected_host_arch == 'ia32':
-        return 'i386'
-    elif detected_host_arch == 'arm':
-        return 'arm'
-    elif detected_host_arch == 'arm64':
-        return 'arm64'
-    elif detected_host_arch == 'mips':
-        return 'mips'
-    elif detected_host_arch == 'ppc':
-        return 'ppc'
-    elif detected_host_arch == 's390':
-        return 's390'
-
-    raise Error('Unrecognized host arch: %s' % detected_host_arch)
 
 
 def main(args):
-    parser = optparse.OptionParser(
+  parser = optparse.OptionParser('usage: %prog [OPTIONS]', description=__doc__)
-        'usage: %prog [OPTIONS]', description=__doc__)
+  parser.add_option('--arch',
-    parser.add_option(
+                    help='Sysroot architecture: %s' % ', '.join(VALID_ARCHS))
-        '--arch',
+  parser.add_option('--all', action='store_true',
-        type='choice',
+                    help='Install all sysroot images (useful when updating the'
-        choices=VALID_ARCHS,
+                         ' images)')
-        help='Sysroot architecture: %s' % ', '.join(VALID_ARCHS))
+  parser.add_option('--print-hash',
-    options, _ = parser.parse_args(args)
+                    help='Print the hash of the sysroot for the given arch.')
-    if not sys.platform.startswith('linux'):
+  options, _ = parser.parse_args(args)
-        return 0
-
-    if not options.arch:
-        print('You much specify either --arch or --running-as-hook')
-        return 1
-    InstallDefaultSysrootForArch(options.arch)
 
+  if options.print_hash:
+    arch = options.print_hash
+    print(GetSysrootDict(DEFAULT_TARGET_PLATFORM,
+                         ARCH_TRANSLATIONS.get(arch, arch))['Sha1Sum'])
     return 0
+  if options.arch:
+    InstallSysroot(DEFAULT_TARGET_PLATFORM,
+                   ARCH_TRANSLATIONS.get(options.arch, options.arch))
+  elif options.all:
+    for arch in VALID_ARCHS:
+      InstallSysroot(DEFAULT_TARGET_PLATFORM, arch)
+  else:
+    print('You much specify one of the options.')
+    return 1
+
+  return 0
 
 
-def InstallDefaultSysrootForArch(target_arch):
+def GetSysrootDict(target_platform, target_arch):
-    if target_arch == 'amd64':
+  if target_arch not in VALID_ARCHS:
-        InstallSysroot('Jessie', 'amd64')
+    raise Error('Unknown architecture: %s' % target_arch)
-    elif target_arch == 'arm':
+
-        InstallSysroot('Jessie', 'arm')
+  sysroots_file = os.path.join(SCRIPT_DIR, 'sysroots.json')
-    elif target_arch == 'arm64':
+  sysroots = json.load(open(sysroots_file))
-        InstallSysroot('Jessie', 'arm64')
+  sysroot_key = '%s_%s' % (target_platform, target_arch)
-    elif target_arch == 'i386':
+  if sysroot_key not in sysroots:
-        InstallSysroot('Jessie', 'i386')
+    raise Error('No sysroot for: %s %s' % (target_platform, target_arch))
-    elif target_arch == 'mips':
+  return sysroots[sysroot_key]
-        InstallSysroot('Jessie', 'mips')
-    else:
-        raise Error('Unknown architecture: %s' % target_arch)
 
 
 def InstallSysroot(target_platform, target_arch):
-    # The sysroot directory should match the one specified in build/common.gypi.
+  sysroot_dict = GetSysrootDict(target_platform, target_arch)
-    # TODO(thestig) Consider putting this elsewhere to avoid having to recreate
+  tarball_filename = sysroot_dict['Tarball']
-    # it on every build.
+  tarball_sha1sum = sysroot_dict['Sha1Sum']
-    linux_dir = os.path.dirname(SCRIPT_DIR)
+  # TODO(thestig) Consider putting this elsewhere to avoid having to recreate
+  # it on every build.
+  linux_dir = os.path.dirname(SCRIPT_DIR)
+  sysroot = os.path.join(linux_dir, sysroot_dict['SysrootDir'])
 
-    sysroots_file = os.path.join(SCRIPT_DIR, 'sysroots.json')
+  url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, tarball_sha1sum,
-    sysroots = json.load(open(sysroots_file))
+                         tarball_filename)
-    sysroot_key = '%s_%s' % (target_platform.lower(), target_arch)
-    if sysroot_key not in sysroots:
-        raise Error('No sysroot for: %s %s' % (target_platform, target_arch))
-    sysroot_dict = sysroots[sysroot_key]
-    revision = sysroot_dict['Revision']
-    tarball_filename = sysroot_dict['Tarball']
-    tarball_sha1sum = sysroot_dict['Sha1Sum']
-    sysroot = os.path.join(linux_dir, sysroot_dict['SysrootDir'])
 
-    url = '%s/%s/%s/%s' % (URL_PREFIX, URL_PATH, revision, tarball_filename)
+  stamp = os.path.join(sysroot, '.stamp')
+  if os.path.exists(stamp):
+    with open(stamp) as s:
+      if s.read() == url:
+        return
 
-    stamp = os.path.join(sysroot, '.stamp')
+  print('Installing Debian %s %s root image: %s' % \
-    if os.path.exists(stamp):
+      (target_platform, target_arch, sysroot))
-        with open(stamp) as s:
+  if os.path.isdir(sysroot):
-            if s.read() == url:
+    shutil.rmtree(sysroot)
-                return
+  os.mkdir(sysroot)
+  tarball = os.path.join(sysroot, tarball_filename)
+  print('Downloading %s' % url)
+  sys.stdout.flush()
+  sys.stderr.flush()
+  for _ in range(3):
+    try:
+      response = urlopen(url)
+      with open(tarball, "wb") as f:
+        f.write(response.read())
+      break
+    except Exception:  # Ignore exceptions.
+      pass
+  else:
+    raise Error('Failed to download %s' % url)
+  sha1sum = GetSha1(tarball)
+  if sha1sum != tarball_sha1sum:
+    raise Error('Tarball sha1sum is wrong.'
+                'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum))
+  subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot])
+  os.remove(tarball)
 
-    print('Installing Debian %s %s root image: %s' % \
+  with open(stamp, 'w') as s:
-        (target_platform, target_arch, sysroot))
+    s.write(url)
-    if os.path.isdir(sysroot):
-        shutil.rmtree(sysroot)
-    os.mkdir(sysroot)
-    tarball = os.path.join(sysroot, tarball_filename)
-    print('Downloading %s' % url)
-    sys.stdout.flush()
-    sys.stderr.flush()
-    for _ in range(3):
-        try:
-            response = urlopen(url)
-            with open(tarball, "wb") as f:
-                f.write(response.read())
-            break
-        except:
-            pass
-    else:
-        raise Error('Failed to download %s' % url)
-    sha1sum = GetSha1(tarball)
-    if sha1sum != tarball_sha1sum:
-        raise Error('Tarball sha1sum is wrong.'
-                    'Expected %s, actual: %s' % (tarball_sha1sum, sha1sum))
-    subprocess.check_call(['tar', 'xf', tarball, '-C', sysroot])
-    os.remove(tarball)
-
-    with open(stamp, 'w') as s:
-        s.write(url)
 
 
 if __name__ == '__main__':
-    try:
+  try:
-        sys.exit(main(sys.argv[1:]))
+    sys.exit(main(sys.argv[1:]))
-    except Error as e:
+  except Error as e:
-        sys.stderr.write(str(e) + '\n')
+    sys.stderr.write(str(e) + '\n')
-        sys.exit(1)
+    sys.exit(1)

build/linux/sysroot_scripts/sysroots.json

@@ -1,44 +1,37 @@
 {
-    "jessie_amd64": {
+    "bullseye_amd64": {
-        "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f",
+        "Sha1Sum": "cb4fa34f1faddafb72cace35faf62a611f2ca7c9",
-        "Sha1Sum": "046a82b027f069bdde98424850490c315e2f31f9",
+        "SysrootDir": "debian_bullseye_amd64-sysroot",
-        "SysrootDir": "debian_jessie_amd64-sysroot",
+        "Tarball": "debian_bullseye_amd64_sysroot.tar.xz"
-        "Tarball": "debian_jessie_amd64_sysroot.tgz"
     },
-    "jessie_arm": {
+    "bullseye_arm": {
-        "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f",
+        "Sha1Sum": "a1084eba943f797a1c9aee8d5f8213ed680315a8",
-        "Sha1Sum": "7addeb28195f7f367cbaba46cdba99925a0646b2",
+        "SysrootDir": "debian_bullseye_arm-sysroot",
-        "SysrootDir": "debian_jessie_arm-sysroot",
+        "Tarball": "debian_bullseye_arm_sysroot.tar.xz"
-        "Tarball": "debian_jessie_arm_sysroot.tgz"
     },
-    "jessie_arm64": {
+    "bullseye_arm64": {
-        "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f",
+        "Sha1Sum": "f00ece500aef0ff2a431a07de48bd3e1aa6d1caf",
-        "Sha1Sum": "1b853d8082321256fe646589558da3980875d767",
+        "SysrootDir": "debian_bullseye_arm64-sysroot",
-        "SysrootDir": "debian_jessie_arm64-sysroot",
+        "Tarball": "debian_bullseye_arm64_sysroot.tar.xz"
-        "Tarball": "debian_jessie_arm64_sysroot.tgz"
     },
-    "jessie_i386": {
+    "bullseye_armel": {
-        "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f",
+        "Sha1Sum": "54ccf4898fe923b22141c9e21e80b455dec47ecf",
-        "Sha1Sum": "7edfe877ea71473def2d9e6958474d76c9ff98b6",
+        "SysrootDir": "debian_bullseye_armel-sysroot",
-        "SysrootDir": "debian_jessie_i386-sysroot",
+        "Tarball": "debian_bullseye_armel_sysroot.tar.xz"
-        "Tarball": "debian_jessie_i386_sysroot.tgz"
     },
-    "jessie_mips": {
+    "bullseye_i386": {
-        "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f",
+        "Sha1Sum": "07bb72368889fea19a1f123aa11528f0d792121b",
-        "Sha1Sum": "3d0c5640a39e264d7180df2bac1192308f8a256e",
+        "SysrootDir": "debian_bullseye_i386-sysroot",
-        "SysrootDir": "debian_jessie_mips-sysroot",
+        "Tarball": "debian_bullseye_i386_sysroot.tar.xz"
-        "Tarball": "debian_jessie_mips_sysroot.tgz"
     },
-    "precise_amd64": {
+    "bullseye_mips": {
-        "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f",
+        "Sha1Sum": "f19f07f7e62edbf837d6e7f173aea3f47110c2ba",
-        "Sha1Sum": "69e1d3c9efda92ba265ce7bb653be546e8dde7e4",
+        "SysrootDir": "debian_bullseye_mips-sysroot",
-        "SysrootDir": "ubuntu_precise_amd64-sysroot",
+        "Tarball": "debian_bullseye_mips_sysroot.tar.xz"
-        "Tarball": "ubuntu_precise_amd64_sysroot.tgz"
     },
-    "trusty_arm": {
+    "bullseye_mips64el": {
-        "Revision": "7031a828c5dcedc937bbf375c42daab08ca6162f",
+        "Sha1Sum": "3cfc9e7fe2d0cad3873bbce8fc3ffde510ed5638",
-        "Sha1Sum": "d1591e8fc6a273af1fbf3d1a864f098081d06dd1",
+        "SysrootDir": "debian_bullseye_mips64el-sysroot",
-        "SysrootDir": "ubuntu_trusty_arm-sysroot",
+        "Tarball": "debian_bullseye_mips64el_sysroot.tar.xz"
-        "Tarball": "ubuntu_trusty_arm_sysroot.tgz"
     }
 }