[infra] Update Clang toolchain and GN

We no longer rely on Fuchsia's buildtools, instead we directly fetch
Clang and GN packages from CIPD using gclient support.

Change-Id: I8995d89babf9c3c6b30214a1133e7b0a1cb9a796
Reviewed-on: https://dart-review.googlesource.com/c/85416
Reviewed-by: Ryan Macnak <rmacnak@google.com>
Commit-Queue: Zach Anderson <zra@google.com>

DEPS

@@ -38,11 +38,13 @@ vars = {
 
   "co19_2_rev": "92eb86a60b26089eaffc4fa9703895f71a251a76",
 
-  # As Flutter does, we pull buildtools, including the clang toolchain, from
-  # Fuchsia. This revision should be kept up to date with the revision pulled
-  # by the Flutter engine. If there are problems with the toolchain, contact
-  # fuchsia-toolchain@.
-  "buildtools_revision": "446d5b1019dcbe7835236dc85261e91cf29a9239",
+  # As Flutter does, we use Fuchsia's GN and Clang toolchain. These revision
+  # should be kept up to date with the revisions pulled by the Flutter engine.
+  # The list of revisions for these tools comes from Fuchsia, here:
+  # https://fuchsia.googlesource.com/buildtools/+/master/fuchsia.ensure
+  # If there are problems with the toolchain, contact fuchsia-toolchain@.
+  "clang_revision": "de39621f0f03f20633bdfa50bde97a3908bf6e98",
+  "gn_revision": "bdb0fd02324b120cacde634a9235405061c8ea06",
 
   # Scripts that make 'git cl format' work.
   "clang_format_scripts_rev": "c09c8deeac31f05bd801995c475e7c8070f9ecda",
@@ -152,8 +154,6 @@ vars = {
 
 deps = {
   # Stuff needed for GN build.
-  Var("dart_root") + "/buildtools":
-     Var("fuchsia_git") + "/buildtools" + "@" + Var("buildtools_revision"),
   Var("dart_root") + "/buildtools/clang_format/script":
     Var("chromium_git") + "/chromium/llvm-project/cfe/tools/clang-format.git" +
     "@" + Var("clang_format_scripts_rev"),
@@ -379,6 +379,27 @@ deps = {
       "@" + Var("web_socket_channel_tag"),
   Var("dart_root") + "/third_party/pkg/yaml":
       Var("dart_git") + "yaml.git" + "@" + Var("yaml_tag"),
+
+  Var("dart_root") + "/buildtools/" + Var("host_os") + "-" + Var("host_cpu") + "/clang": {
+      "packages": [
+          {
+              "package": "fuchsia/clang/${{platform}}",
+              "version": "git_revision:" + Var("clang_revision"),
+          },
+      ],
+      "condition": "(host_os == 'linux' or host_os == 'mac') and (host_cpu == 'x64' or host_cpu == 'arm64')",
+      "dep_type": "cipd",
+  },
+
+  Var("dart_root") + "/buildtools": {
+      "packages": [
+          {
+              "package": "gn/gn/${{platform}}",
+              "version": "git_revision:" + Var("gn_revision"),
+          },
+      ],
+      "dep_type": "cipd",
+  },
 }
 
 deps_os = {

build/config/android/BUILD.gn

@@ -8,7 +8,9 @@ import("//build/config/sysroot.gni")
 config("sdk") {
   if (sysroot != "") {
     cflags = [ "--sysroot=" + sysroot ]
-    ldflags = [ "--sysroot=" + sysroot ]
+    ldflags = [
+      "--sysroot=" + rebase_path("$android_ndk_root/$android_sysroot_subdir"),
+    ]
 
     # Need to get some linker flags out of the sysroot.
     sysroot_ld_path = rebase_path("//build/config/linux/sysroot_ld_path.py")

build/config/android/config.gni

@@ -63,18 +63,16 @@ if (is_android) {
   # Subdirectories inside android_ndk_root that contain the sysroot for the
   # associated platform.
   if (current_cpu == "x64" || current_cpu == "arm64") {
-    _android_api_level = 22
+    android_api_level = 22
   } else {
-    _android_api_level = 16
+    android_api_level = 16
   }
-  x86_android_sysroot_subdir =
-      "platforms/android-${_android_api_level}/arch-x86"
-  arm_android_sysroot_subdir =
-      "platforms/android-${_android_api_level}/arch-arm"
+  x86_android_sysroot_subdir = "platforms/android-${android_api_level}/arch-x86"
+  arm_android_sysroot_subdir = "platforms/android-${android_api_level}/arch-arm"
   x86_64_android_sysroot_subdir =
-      "platforms/android-${_android_api_level}/arch-x86_64"
+      "platforms/android-${android_api_level}/arch-x86_64"
   arm64_android_sysroot_subdir =
-      "platforms/android-${_android_api_level}/arch-arm64"
+      "platforms/android-${android_api_level}/arch-arm64"
 
   # Toolchain root directory for each build. The actual binaries are inside
   # a "bin" directory inside of these.
@@ -90,25 +88,29 @@ if (is_android) {
   # like the toolchain roots.
   if (current_cpu == "x86") {
     android_prebuilt_arch = "android-x86"
-    _binary_prefix = "i686-linux-android"
+    android_target_triple = "i686-linux-android"
     android_toolchain_root = "$x86_android_toolchain_root"
+    android_sysroot_subdir = "$x86_android_sysroot_subdir"
   } else if (current_cpu == "arm") {
     android_prebuilt_arch = "android-arm"
-    _binary_prefix = "arm-linux-androideabi"
+    android_target_triple = "arm-linux-androideabi"
     android_toolchain_root = "$arm_android_toolchain_root"
+    android_sysroot_subdir = "$arm_android_sysroot_subdir"
   } else if (current_cpu == "x64") {
     android_prebuilt_arch = "android-x86_64"
-    _binary_prefix = "x86_64-linux-android"
+    android_target_triple = "x86_64-linux-android"
     android_toolchain_root = "$x86_64_android_toolchain_root"
+    android_sysroot_subdir = "$x86_64_android_sysroot_subdir"
   } else if (current_cpu == "arm64") {
     android_prebuilt_arch = "android-arm64"
-    _binary_prefix = "aarch64-linux-android"
+    android_target_triple = "aarch64-linux-android"
     android_toolchain_root = "$arm64_android_toolchain_root"
+    android_sysroot_subdir = "$arm64_android_sysroot_subdir"
   } else {
     assert(false, "Need android libgcc support for your target arch.")
   }
 
-  android_tool_prefix = "$android_toolchain_root/bin/$_binary_prefix-"
+  android_tool_prefix = "$android_toolchain_root/bin/$android_target_triple-"
   android_readelf = "${android_tool_prefix}readelf"
   android_objcopy = "${android_tool_prefix}objcopy"
   android_gdbserver =

build/config/compiler/BUILD.gn

@@ -246,11 +246,14 @@ config("compiler") {
     ldflags += [ "-pthread" ]
     if (is_clang) {
       if (current_cpu == "arm") {
-        cflags += [ "--target=arm-linux-gnueabihf" ]
-        ldflags += [ "--target=arm-linux-gnueabihf" ]
+        cflags += [ "--target=armv7-linux-gnueabihf" ]
+        ldflags += [ "--target=armv7-linux-gnueabihf" ]
       } else if (current_cpu == "arm64") {
         cflags += [ "--target=aarch64-linux-gnu" ]
         ldflags += [ "--target=aarch64-linux-gnu" ]
+      } else if (current_cpu == "x86") {
+        cflags += [ "--target=i386-linux-gnu" ]
+        ldflags += [ "--target=i386-linux-gnu" ]
       }
     }
   }
@@ -278,6 +281,7 @@ config("compiler") {
       "-ffunction-sections",
       "-funwind-tables",
       "-fno-short-enums",
+      "-nostdinc++",
     ]
     if (!is_clang) {
       # Clang doesn't support these flags.
@@ -444,14 +448,17 @@ config("runtime_library") {
     # strange errors. The include ordering here is important; change with
     # caution.
     cflags += [
-      "-isystem" +
-          rebase_path("$android_libcpp_root/libcxx/include", root_build_dir),
+      "-isystem" + rebase_path("$android_libcpp_root/include", root_build_dir),
       "-isystem" + rebase_path(
-              "$android_ndk_root/sources/cxx-stl/llvm-libc++abi/libcxxabi/include",
+              "$android_ndk_root/sources/cxx-stl/llvm-libc++abi/include",
               root_build_dir),
       "-isystem" +
           rebase_path("$android_ndk_root/sources/android/support/include",
                       root_build_dir),
+      "-isystem" + rebase_path(
+              "$android_ndk_root/sysroot/usr/include/$android_target_triple",
+              root_build_dir),
+      "-D__ANDROID_API__=$android_api_level",
     ]
 
     lib_dirs += [ "$android_libcpp_root/libs/$android_app_abi" ]
@@ -459,9 +466,12 @@ config("runtime_library") {
     libs += [
       "$android_libcpp_library",
       "c++abi",
-      "android_support",
     ]
 
+    if (android_api_level < 21) {
+      libs += [ "android_support" ]
+    }
+
     if (current_cpu == "arm") {
       libs += [ "unwind" ]
     }
@@ -692,7 +702,7 @@ if (is_win) {
       "-Wl,--gc-sections",
     ]
 
-    if (is_clang) {
+    if (is_clang && !using_sanitizer) {
       # Identical code folding to reduce size.
       # Warning: This changes C/C++ semantics of function pointer comparison.
       common_optimize_on_ldflags += [ "-Wl,--icf=all" ]

build/config/sysroot.gni

@@ -33,17 +33,7 @@ if (current_toolchain == default_toolchain && target_sysroot != "") {
   sysroot = target_sysroot
 } else if (is_android) {
   import("//build/config/android/config.gni")
-  if (current_cpu == "x86") {
-    sysroot = rebase_path("$android_ndk_root/$x86_android_sysroot_subdir", root_build_dir)
-  } else if (current_cpu == "arm") {
-    sysroot = rebase_path("$android_ndk_root/$arm_android_sysroot_subdir", root_build_dir)
-  } else if (current_cpu == "x64") {
-    sysroot = rebase_path("$android_ndk_root/$x86_64_android_sysroot_subdir", root_build_dir)
-  } else if (current_cpu == "arm64") {
-    sysroot = rebase_path("$android_ndk_root/$arm64_android_sysroot_subdir", root_build_dir)
-  } else {
-    sysroot = ""
-  }
+  sysroot = rebase_path("$android_ndk_root/sysroot")
 } else if (is_mac) {
   import("//build/config/mac/mac_sdk.gni")
   sysroot = mac_sdk_path

tools/android/VERSION_LINUX_NDK

@@ -1 +1 @@
-50bb526ee77fee88b9382d2bf48e7399751b98ae
+e626c47cb82a7439b0eda03ac6e0e9e1e41c6093

tools/android/VERSION_MACOSX_NDK

@@ -1 +1 @@
-ed883dfaad6f27350eb3de426ccbf9abdc2943e8
+e8b6ecb5d15c4c4018a62b52aabc13e41b17df8f

tools/buildtools/README.md

@@ -1,7 +1,7 @@
 To build Dart for Mac and Linux, we pull Fuchsia's buildtools, which is also
-used by Flutter. Fuchsia's buildtools includes gn, ninja, and the clang
-toolchain. Fuchsia buildtools vends clang-format as part of the clang toolchain.
-Since Fuchsia's buildtools doesn't vend a clang toolchain for Windows, we can't
+used by Flutter. Fuchsia's buildtools includes gn, and the clang toolchain.
+Fuchsia buildtools vends clang-format as part of the clang toolchain. Since
+Fuchsia's buildtools doesn't vend a clang toolchain for Windows, we can't
 get a Windows clang-format binary from it. Therefore, from Chromium's buildtools
 here:
 

tools/buildtools/update.py

@@ -5,10 +5,10 @@
 
 """Pulls down tools required to build Dart."""
 
+import errno
 import os
 import platform
 import subprocess
-import shutil
 import sys
 
 THIS_DIR = os.path.abspath(os.path.dirname(__file__))
@@ -22,34 +22,6 @@ import find_depot_tools
 DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
 
 
-def Update():
-  path = os.path.join(BUILDTOOLS, 'update.sh')
-  command = ['/bin/bash', path, '--clang', '--gn']
-  return subprocess.call(command, cwd=DART_ROOT)
-
-
-def UpdateGNOnWindows():
-  sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'gn.exe.sha1')
-  output_dir = os.path.join(BUILDTOOLS, 'win', 'gn.exe')
-  downloader_script = os.path.join(
-      DEPOT_PATH, 'download_from_google_storage.py')
-  download_cmd = [
-    'python',
-    downloader_script,
-    '--no_auth',
-    '--no_resume',
-    '--quiet',
-    '--platform=win*',
-    '--bucket',
-    'chromium-gn',
-    '-s',
-    sha1_file,
-    '-o',
-    output_dir
-  ]
-  return subprocess.call(download_cmd)
-
-
 def UpdateClangFormatOnWindows():
   sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'clang-format.exe.sha1')
   output_dir = os.path.join(BUILDTOOLS, 'win', 'clang-format.exe')
@@ -72,9 +44,20 @@ def UpdateClangFormatOnWindows():
   return subprocess.call(download_cmd)
 
 
-# On Mac and Linux we copy clang-format and gn to the place where git cl format
-# expects them to be.
-def CopyClangFormat():
+def CreateSymlink(symlink, link_name):
+  try:
+    os.symlink(symlink, link_name)
+  except OSError, e:
+    if e.errno == errno.EEXIST:
+      os.remove(link_name)
+      os.symlink(symlink, link_name)
+    else:
+      raise e
+
+
+# On Mac and Linux we symlink clang-format and gn to the place where
+# 'git cl format' expects them to be.
+def LinksForGitCLFormat():
   if sys.platform == 'darwin':
     platform = 'darwin'
     tools = 'mac'
@@ -89,14 +72,14 @@ def CopyClangFormat():
 
   clang_format = os.path.join(
       BUILDTOOLS, toolchain, 'clang', 'bin', 'clang-format')
-  gn = os.path.join(BUILDTOOLS, toolchain, 'gn')
+  gn = os.path.join(BUILDTOOLS, 'gn')
   dest_dir = os.path.join(BUILDTOOLS, tools)
   if not os.path.exists(dest_dir):
     os.makedirs(dest_dir)
   clang_format_dest = os.path.join(dest_dir, 'clang-format')
   gn_dest = os.path.join(dest_dir, 'gn')
-  shutil.copy2(clang_format, clang_format_dest)
-  shutil.copy2(gn, gn_dest)
+  CreateSymlink(clang_format, clang_format_dest)
+  CreateSymlink(gn, gn_dest)
   return 0
 
 
@@ -107,13 +90,8 @@ def main(argv):
     print('Not downloading buildtools binaries for ' + arch_id)
     return 0
   if sys.platform.startswith('win'):
-    result = UpdateGNOnWindows()
-    if result != 0:
-      return result
     return UpdateClangFormatOnWindows()
-  if Update() != 0:
-    return 1
-  return CopyClangFormat()
+  return LinksForGitCLFormat()
 
 
 if __name__ == '__main__':

tools/gn.py

@@ -496,16 +496,11 @@ def Main(argv):
   starttime = time.time()
   args = parse_args(argv)
 
-  if sys.platform.startswith(('cygwin', 'win')):
-    subdir = 'win'
-  elif sys.platform == 'darwin':
-    subdir = 'mac-x64'
-  elif sys.platform.startswith('linux'):
-    subdir = 'linux-x64'
-  else:
-    print 'Unknown platform: ' + sys.platform
+  gn = os.path.join(DART_ROOT, 'buildtools',
+      'gn.exe' if utils.IsWindows() else 'gn')
+  if not os.path.isfile(gn):
+    print "Couldn't find the gn binary at path: " + gn
     return 1
-  gn = os.path.join(DART_ROOT, 'buildtools', subdir, 'gn')
 
   commands = []
   for target_os in args.os: