[infra] Clean up debian package builder

* Add a docker file to avoid rebuilding the same image on every run.
* Move all files involved into the linux_dist_support folder.
* Exclude more directories in the tarball.
* Delete version checker from test_matrix.
* Disable goma when running build.py.

Change-Id: Ic7b0a2359027d532c009cdf3187d873323170f30
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/195901
Reviewed-by: William Hesse <whesse@google.com>

tools/bots/test_matrix.json

@@ -2918,27 +2918,12 @@
       "steps": [
         {
           "name": "build debian package",
-          "script": "tools/run_debian_build.sh",
+          "script": "tools/linux_dist_support/run_debian_build.sh",
           "arguments": []
         },
         {
           "name": "upload debian packages",
-          "script": "tools/bots/upload_debian_packages.py",
-          "arguments": []
-        }
-      ]
-    },
-    {
-      "builders": [
-        "versionchecker-linux"
-      ],
-      "meta": {
-        "description": "This configuration is used by the versionchecker-builder."
-      },
-      "steps": [
-        {
-          "name": "check version",
-          "script": "tools/bots/version_checker.py",
+          "script": "tools/linux_dist_support/upload_debian_packages.py",
           "arguments": []
         }
       ]

tools/create_debian_chroot.sh

@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
-# for details. All rights reserved. Use of this source code is governed by a
-# BSD-style license that can be found in the LICENSE file.
-#
-# Script to create a Debian jessie chroot environment for building Dart
-# Debian packages.
-#
-
-function usage {
-  USAGE="Usage: $0 i386|amd64 [target dir] [be|beta|dev|<stable version>]]\n
-\n
-The first mandatory argument speciifies the CPU architecture using\n
-the Debian convention (e.g. i386 and amd64).\n
-\n
-The second optional argument specifies the destination\n
-directory. This defaults to 'debian_<architecture>'.\n
-\n
-The third optional argument specifies whether the chroot is\n
-populated with a Dart checkout. Use 'be' for bleeding edge, 'dev'\n
-for trunk/developer or the specific version number for a stable\n
-version (e.g. 1.2)."
-
-  echo -e $USAGE
-  exit 1
-}
-
-# Expect one to three arguments, architecture, optional directory and
-# optional channel.
-if [ $# -lt 1 ] || [ $# -gt 3 ]
-then
-  usage
-fi
-
-ARCH=$1
-
-if [ -n "$2" ]
-then
-  CHROOT=$2
-else
-  CHROOT=debian_$ARCH
-fi
-
-if [ -n "$3" ]
-then
-  CHANNEL=$3
-fi
-
-if [ "$ARCH" != "i386" ] && [ "$ARCH" != "amd64" ]
-then
-  usage
-fi
-
-SVN_REPRO="http://dart.googlecode.com/svn/"
-if [ -n "$CHANNEL" ]
-then
-  if [ "$CHANNEL" == "be" ]
-  then
-    SVN_PATH="branches/bleeding_edge/deps/all.deps"
-  elif [ "$CHANNEL" == "dev" ]
-  then
-    SVN_PATH="trunk/deps/all.deps"
-  else
-    SVN_PATH="branches/$CHANNEL/deps/all.deps"
-  fi
-  SRC_URI=$SVN_REPRO$SVN_PATH
-fi
-
-# Create Debian jessie chroot.
-debootstrap --arch=$ARCH --components=main,restricted,universe,multiverse \
-    jessie $CHROOT http://http.us.debian.org/debian/
-chroot $CHROOT apt-get update
-chroot $CHROOT apt-get -y install \
-    debhelper python3 git gcc sudo make
-
-# Add chrome-bot user.
-chroot $CHROOT groupadd --gid 1001 chrome-bot
-chroot $CHROOT useradd --gid 1001 --uid 1001 --create-home chrome-bot
-mkdir $CHROOT/b
-chown 1001:1001 $CHROOT/b
-
-# Create trampoline script for running the initialization as chrome-bot.
-cat << EOF > $CHROOT/b/init_chroot_trampoline.sh
-#!/bin/sh
-su -c /b/init_chroot.sh chrome-bot
-EOF
-
-# Create initialization script which does nothing.
-cat << 'EOF' > $CHROOT/b/init_chroot.sh
-#!/bin/sh
-cd /b
-EOF
-
-# If the channel is set extend the initialization script to check out
-# the Dart sources. This uses two cat commands as the first part needs
-# to bypass variable interpretation.
-if [ -n "$SRC_URI" ]
-then
-cat << 'EOF' >> $CHROOT/b/init_chroot.sh
-git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
-export PATH=$PATH:/b/depot_tools
-EOF
-
-cat << EOF >> $CHROOT/b/init_chroot.sh
-gclient config $SRC_URI
-gclient sync
-gclient runhooks
-EOF
-fi
-
-chmod 755 $CHROOT/b/init_chroot_trampoline.sh
-
-chown 1001:1001 $CHROOT/b/init_chroot.sh
-chmod 755 $CHROOT/b/init_chroot.sh
-chroot $CHROOT /bin/sh /b/init_chroot_trampoline.sh

tools/linux_dist_support/Debian.dockerfile

@@ -0,0 +1,11 @@
+# Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+FROM launcher.gcr.io/google/debian8:latest
+ARG depot_tools
+RUN sed -i /jessie-updates/d /etc/apt/sources.list
+RUN apt-get update \
+  && apt-get install -y build-essential debhelper git python3 \
+  && rm -rf /var/lib/apt/lists/*
+ENV PATH="$depot_tools:${PATH}"
+ENTRYPOINT python3 tools/linux_dist_support/linux_distribution_support.py

tools/linux_dist_support/create_debian_packages.py

@@ -14,32 +14,33 @@ import os
 import sys
 import tarfile
 import subprocess
+from os.path import join, exists, abspath, dirname
+sys.path.append(join(dirname(__file__), '..'))
 import utils
-from os.path import join, exists, abspath
 from shutil import copyfile
 
 HOST_OS = utils.GuessOS()
 HOST_CPUS = utils.GuessCpus()
-DART_DIR = abspath(join(__file__, '..', '..'))
+DART_DIR = abspath(join(dirname(__file__), '..', '..'))
 
 
 def BuildOptions():
     result = optparse.OptionParser()
-    result.add_option(
-        "--tar_filename", default=None, help="The tar file to build from.")
-    result.add_option(
-        "--out_dir", default=None, help="Where to put the packages.")
-    result.add_option(
-        "-a",
-        "--arch",
-        help='Target architectures (comma-separated).',
-        metavar='[all,ia32,x64,armel,armhf]',
-        default='x64')
-    result.add_option(
-        "-t",
-        "--toolchain",
-        help='Cross-compilation toolchain prefix',
-        default=None)
+    result.add_option("--tar_filename",
+                      default=None,
+                      help="The tar file to build from.")
+    result.add_option("--out_dir",
+                      default=None,
+                      help="Where to put the packages.")
+    result.add_option("-a",
+                      "--arch",
+                      help='Target architectures (comma-separated).',
+                      metavar='[all,ia32,x64,armel,armhf]',
+                      default='x64')
+    result.add_option("-t",
+                      "--toolchain",
+                      help='Cross-compilation toolchain prefix',
+                      default=None)
 
     return result
 

tools/linux_dist_support/create_tarball.py

@@ -26,12 +26,12 @@ import optparse
 import sys
 import tarfile
 from os import listdir
-from os.path import join, split, abspath
-
+from os.path import join, split, abspath, dirname
+sys.path.append(join(dirname(__file__), '..'))
 import utils
 
 HOST_OS = utils.GuessOS()
-DART_DIR = abspath(join(__file__, '..', '..'))
+DART_DIR = abspath(join(dirname(__file__), '..', '..'))
 # Flags.
 verbose = False
 
@@ -40,21 +40,40 @@ versiondir = ''
 
 # Ignore Git/SVN files, checked-in binaries, backup files, etc..
 ignoredPaths = [
-    'third_party/7zip', 'third_party/android_tools', 'third_party/clang',
-    'third_party/d8', 'third_party/firefox_jsshell'
+    'buildtools/linux-x64/go',
+    'buildtools/linux-x64/rust',
+    'samples',
+    'samples_2',
+    'third_party/7zip',
+    'third_party/android_tools',
+    'third_party/clang',
+    'third_party/d8',
+    'third_party/firefox_jsshell',
+    'third_party/gsutil',
+    'third_party/llvm-build',
+    'third_party/mdn',
+]
+ignoredDirs = [
+    '.cipd',
+    '.git',
+    'benchmarks',
+    'docs',
+    'fuchsia',
+    'parser_testcases',
+    'test',
+    'testcases',
+    'tests',
 ]
-ignoredDirs = ['.svn', '.git']
 ignoredEndings = ['.mk', '.pyc', 'Makefile', '~']
 
 
 def BuildOptions():
     result = optparse.OptionParser()
-    result.add_option(
-        "-v",
-        "--verbose",
-        help='Verbose output.',
-        default=False,
-        action="store_true")
+    result.add_option("-v",
+                      "--verbose",
+                      help='Verbose output.',
+                      default=False,
+                      action="store_true")
     result.add_option("--tar_filename", default=None, help="The output file.")
 
     return result
@@ -134,9 +153,8 @@ def CreateTarball(tarfilename):
         for f in listdir(DART_DIR):
             tar.add(join(DART_DIR, f), filter=Filter)
         for f in listdir(join(DART_DIR, debian_dir)):
-            tar.add(
-                join(DART_DIR, debian_dir, f),
-                arcname='%s/debian/%s' % (versiondir, f))
+            tar.add(join(DART_DIR, debian_dir, f),
+                    arcname='%s/debian/%s' % (versiondir, f))
 
         with utils.TempDir() as temp_dir:
             # Generate and add debian/copyright
@@ -158,9 +176,8 @@ def CreateTarball(tarfilename):
             if utils.GetChannel() == 'be':
                 git_revision = join(temp_dir, 'GIT_REVISION')
                 GenerateGitRevision(git_revision, utils.GetGitRevision())
-                tar.add(
-                    git_revision,
-                    arcname='%s/dart/tools/GIT_REVISION' % versiondir)
+                tar.add(git_revision,
+                        arcname='%s/dart/tools/GIT_REVISION' % versiondir)
 
 
 def Main():

tools/linux_dist_support/debian/rules

@@ -57,7 +57,8 @@ override_dh_auto_configure:
 
 override_dh_auto_build:
 	cd dart; \
-	python3 tools/build.py -v -m release -a $(ARCH) $(TOOLCHAIN) create_sdk; \
+	python3 tools/build.py --no-goma --mode release \
+	  --arch $(ARCH) $(TOOLCHAIN) create_sdk; \
 	cd ..
 
 # Building the Dart SDK will already strip all binaries.

tools/linux_dist_support/linux_distribution_support.py

@@ -11,16 +11,17 @@ Archive tarball and debian package to google cloud storage.
 """
 
 import os
-import re
 import subprocess
 import sys
 
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'bots'))
 import bot_utils
 
 utils = bot_utils.GetUtils()
 
 HOST_OS = utils.GuessOS()
 
+
 def InstallFromDep(builddir):
     for entry in os.listdir(builddir):
         if entry.endswith("_amd64.deb"):
@@ -47,9 +48,7 @@ def Run(command):
     sys.stdout.flush()
     no_color_env = dict(os.environ)
     no_color_env['TERM'] = 'nocolor'
-    exit_code = subprocess.call(command, env=no_color_env)
-    if exit_code != 0:
-        raise OSError(exit_code)
+    subprocess.check_call(command, env=no_color_env)
 
 
 def TestInstallation(assume_installed=True):
@@ -90,13 +89,14 @@ def SrcSteps():
 
     print('Building src tarball')
     Run([
-        sys.executable, './tools/create_tarball.py', '--tar_filename', tarfile
+        sys.executable, 'tools/linux_dist_support/create_tarball.py',
+        '--tar_filename', tarfile
     ])
 
     print('Building Debian packages')
     Run([
-        sys.executable, './tools/create_debian_packages.py', '--tar_filename',
-        tarfile, '--out_dir', builddir
+        sys.executable, 'tools/linux_dist_support/create_debian_packages.py',
+        '--tar_filename', tarfile, '--out_dir', builddir
     ])
 
     if os.path.exists('/usr/bin/dart') or os.path.exists(
@@ -110,7 +110,10 @@ def SrcSteps():
 
     # We build the runtime target to get everything we need to test the
     # standalone target.
-    Run([sys.executable, './tools/build.py', '-mrelease', '-ax64', 'runtime'])
+    Run([
+        sys.executable, 'tools/build.py', '--no-goma', '--mode=release',
+        '--arch=x64', 'runtime'
+    ])
     # Copy in the installed binary to avoid poluting /usr/bin (and having to
     # run as root)
     Run(['cp', '/usr/bin/dart', 'out/ReleaseX64/dart'])

tools/linux_dist_support/run_debian_build.sh

@@ -6,11 +6,9 @@ set -x
 
 ninja=$(which ninja)
 depot_tools=$(dirname $ninja)
-cmd="sed -i /jessie-updates/d /etc/apt/sources.list \
-    && apt-get update \
-    && apt-get -y install build-essential debhelper git python3 \
-    && PATH=\"$depot_tools:\$PATH\" \
-    python3 tools/bots/linux_distribution_support.py"
-image="launcher.gcr.io/google/debian8:latest"
+image="debian-package:0.1"
+dockerfile=tools/linux_dist_support/Debian.dockerfile
+docker build --build-arg depot_tools=$depot_tools -t $image - < $dockerfile
+checkout=$(pwd)
 docker run -e BUILDBOT_BUILDERNAME -v $depot_tools:$depot_tools\
-    -v `pwd`:`pwd` -w `pwd` -i --rm $image bash -c "$cmd"
+    -v $checkout:$checkout -w $checkout -i --rm $image

tools/linux_dist_support/upload_debian_packages.py

@@ -5,7 +5,8 @@
 # BSD-style license that can be found in the LICENSE file.
 
 import os
-
+import sys
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'bots'))
 import bot_utils
 
 utils = bot_utils.GetUtils()