From 031e77eea11a16a5e486a3673549adbd68862c8e Mon Sep 17 00:00:00 2001 From: Zach Anderson Date: Thu, 29 Nov 2018 15:37:40 +0000 Subject: [PATCH] [infra] Update Clang toolchain and GN We no longer rely on Fuchsia's buildtools, instead we directly fetch Clang and GN packages from CIPD using gclient support. Change-Id: I8995d89babf9c3c6b30214a1133e7b0a1cb9a796 Reviewed-on: https://dart-review.googlesource.com/c/85416 Reviewed-by: Ryan Macnak Commit-Queue: Zach Anderson --- DEPS | 35 +++++++++++++++---- build/config/android/BUILD.gn | 4 ++- build/config/android/config.gni | 28 ++++++++------- build/config/compiler/BUILD.gn | 24 +++++++++---- build/config/sysroot.gni | 12 +------ tools/android/VERSION_LINUX_NDK | 2 +- tools/android/VERSION_MACOSX_NDK | 2 +- tools/buildtools/README.md | 6 ++-- tools/buildtools/update.py | 60 ++++++++++---------------------- tools/gn.py | 13 +++---- 10 files changed, 92 insertions(+), 94 deletions(-) diff --git a/DEPS b/DEPS index bccc28f68bf..7ea7791e89b 100644 --- a/DEPS +++ b/DEPS @@ -38,11 +38,13 @@ vars = { "co19_2_rev": "92eb86a60b26089eaffc4fa9703895f71a251a76", - # As Flutter does, we pull buildtools, including the clang toolchain, from - # Fuchsia. This revision should be kept up to date with the revision pulled - # by the Flutter engine. If there are problems with the toolchain, contact - # fuchsia-toolchain@. - "buildtools_revision": "446d5b1019dcbe7835236dc85261e91cf29a9239", + # As Flutter does, we use Fuchsia's GN and Clang toolchain. These revision + # should be kept up to date with the revisions pulled by the Flutter engine. + # The list of revisions for these tools comes from Fuchsia, here: + # https://fuchsia.googlesource.com/buildtools/+/master/fuchsia.ensure + # If there are problems with the toolchain, contact fuchsia-toolchain@. + "clang_revision": "de39621f0f03f20633bdfa50bde97a3908bf6e98", + "gn_revision": "bdb0fd02324b120cacde634a9235405061c8ea06", # Scripts that make 'git cl format' work. "clang_format_scripts_rev": "c09c8deeac31f05bd801995c475e7c8070f9ecda", @@ -152,8 +154,6 @@ vars = { deps = { # Stuff needed for GN build. - Var("dart_root") + "/buildtools": - Var("fuchsia_git") + "/buildtools" + "@" + Var("buildtools_revision"), Var("dart_root") + "/buildtools/clang_format/script": Var("chromium_git") + "/chromium/llvm-project/cfe/tools/clang-format.git" + "@" + Var("clang_format_scripts_rev"), @@ -379,6 +379,27 @@ deps = { "@" + Var("web_socket_channel_tag"), Var("dart_root") + "/third_party/pkg/yaml": Var("dart_git") + "yaml.git" + "@" + Var("yaml_tag"), + + Var("dart_root") + "/buildtools/" + Var("host_os") + "-" + Var("host_cpu") + "/clang": { + "packages": [ + { + "package": "fuchsia/clang/${{platform}}", + "version": "git_revision:" + Var("clang_revision"), + }, + ], + "condition": "(host_os == 'linux' or host_os == 'mac') and (host_cpu == 'x64' or host_cpu == 'arm64')", + "dep_type": "cipd", + }, + + Var("dart_root") + "/buildtools": { + "packages": [ + { + "package": "gn/gn/${{platform}}", + "version": "git_revision:" + Var("gn_revision"), + }, + ], + "dep_type": "cipd", + }, } deps_os = { diff --git a/build/config/android/BUILD.gn b/build/config/android/BUILD.gn index 0cc38b7e3c8..c8c978a175d 100644 --- a/build/config/android/BUILD.gn +++ b/build/config/android/BUILD.gn @@ -8,7 +8,9 @@ import("//build/config/sysroot.gni") config("sdk") { if (sysroot != "") { cflags = [ "--sysroot=" + sysroot ] - ldflags = [ "--sysroot=" + sysroot ] + ldflags = [ + "--sysroot=" + rebase_path("$android_ndk_root/$android_sysroot_subdir"), + ] # Need to get some linker flags out of the sysroot. sysroot_ld_path = rebase_path("//build/config/linux/sysroot_ld_path.py") diff --git a/build/config/android/config.gni b/build/config/android/config.gni index a6dcf789aab..38e3f8e218f 100644 --- a/build/config/android/config.gni +++ b/build/config/android/config.gni @@ -63,18 +63,16 @@ if (is_android) { # Subdirectories inside android_ndk_root that contain the sysroot for the # associated platform. if (current_cpu == "x64" || current_cpu == "arm64") { - _android_api_level = 22 + android_api_level = 22 } else { - _android_api_level = 16 + android_api_level = 16 } - x86_android_sysroot_subdir = - "platforms/android-${_android_api_level}/arch-x86" - arm_android_sysroot_subdir = - "platforms/android-${_android_api_level}/arch-arm" + x86_android_sysroot_subdir = "platforms/android-${android_api_level}/arch-x86" + arm_android_sysroot_subdir = "platforms/android-${android_api_level}/arch-arm" x86_64_android_sysroot_subdir = - "platforms/android-${_android_api_level}/arch-x86_64" + "platforms/android-${android_api_level}/arch-x86_64" arm64_android_sysroot_subdir = - "platforms/android-${_android_api_level}/arch-arm64" + "platforms/android-${android_api_level}/arch-arm64" # Toolchain root directory for each build. The actual binaries are inside # a "bin" directory inside of these. @@ -90,25 +88,29 @@ if (is_android) { # like the toolchain roots. if (current_cpu == "x86") { android_prebuilt_arch = "android-x86" - _binary_prefix = "i686-linux-android" + android_target_triple = "i686-linux-android" android_toolchain_root = "$x86_android_toolchain_root" + android_sysroot_subdir = "$x86_android_sysroot_subdir" } else if (current_cpu == "arm") { android_prebuilt_arch = "android-arm" - _binary_prefix = "arm-linux-androideabi" + android_target_triple = "arm-linux-androideabi" android_toolchain_root = "$arm_android_toolchain_root" + android_sysroot_subdir = "$arm_android_sysroot_subdir" } else if (current_cpu == "x64") { android_prebuilt_arch = "android-x86_64" - _binary_prefix = "x86_64-linux-android" + android_target_triple = "x86_64-linux-android" android_toolchain_root = "$x86_64_android_toolchain_root" + android_sysroot_subdir = "$x86_64_android_sysroot_subdir" } else if (current_cpu == "arm64") { android_prebuilt_arch = "android-arm64" - _binary_prefix = "aarch64-linux-android" + android_target_triple = "aarch64-linux-android" android_toolchain_root = "$arm64_android_toolchain_root" + android_sysroot_subdir = "$arm64_android_sysroot_subdir" } else { assert(false, "Need android libgcc support for your target arch.") } - android_tool_prefix = "$android_toolchain_root/bin/$_binary_prefix-" + android_tool_prefix = "$android_toolchain_root/bin/$android_target_triple-" android_readelf = "${android_tool_prefix}readelf" android_objcopy = "${android_tool_prefix}objcopy" android_gdbserver = diff --git a/build/config/compiler/BUILD.gn b/build/config/compiler/BUILD.gn index 4bdcda6f75d..1f2ebd474f5 100644 --- a/build/config/compiler/BUILD.gn +++ b/build/config/compiler/BUILD.gn @@ -246,11 +246,14 @@ config("compiler") { ldflags += [ "-pthread" ] if (is_clang) { if (current_cpu == "arm") { - cflags += [ "--target=arm-linux-gnueabihf" ] - ldflags += [ "--target=arm-linux-gnueabihf" ] + cflags += [ "--target=armv7-linux-gnueabihf" ] + ldflags += [ "--target=armv7-linux-gnueabihf" ] } else if (current_cpu == "arm64") { cflags += [ "--target=aarch64-linux-gnu" ] ldflags += [ "--target=aarch64-linux-gnu" ] + } else if (current_cpu == "x86") { + cflags += [ "--target=i386-linux-gnu" ] + ldflags += [ "--target=i386-linux-gnu" ] } } } @@ -278,6 +281,7 @@ config("compiler") { "-ffunction-sections", "-funwind-tables", "-fno-short-enums", + "-nostdinc++", ] if (!is_clang) { # Clang doesn't support these flags. @@ -444,14 +448,17 @@ config("runtime_library") { # strange errors. The include ordering here is important; change with # caution. cflags += [ - "-isystem" + - rebase_path("$android_libcpp_root/libcxx/include", root_build_dir), + "-isystem" + rebase_path("$android_libcpp_root/include", root_build_dir), "-isystem" + rebase_path( - "$android_ndk_root/sources/cxx-stl/llvm-libc++abi/libcxxabi/include", + "$android_ndk_root/sources/cxx-stl/llvm-libc++abi/include", root_build_dir), "-isystem" + rebase_path("$android_ndk_root/sources/android/support/include", root_build_dir), + "-isystem" + rebase_path( + "$android_ndk_root/sysroot/usr/include/$android_target_triple", + root_build_dir), + "-D__ANDROID_API__=$android_api_level", ] lib_dirs += [ "$android_libcpp_root/libs/$android_app_abi" ] @@ -459,9 +466,12 @@ config("runtime_library") { libs += [ "$android_libcpp_library", "c++abi", - "android_support", ] + if (android_api_level < 21) { + libs += [ "android_support" ] + } + if (current_cpu == "arm") { libs += [ "unwind" ] } @@ -692,7 +702,7 @@ if (is_win) { "-Wl,--gc-sections", ] - if (is_clang) { + if (is_clang && !using_sanitizer) { # Identical code folding to reduce size. # Warning: This changes C/C++ semantics of function pointer comparison. common_optimize_on_ldflags += [ "-Wl,--icf=all" ] diff --git a/build/config/sysroot.gni b/build/config/sysroot.gni index a4364ac7b7a..52a1b83d7dd 100644 --- a/build/config/sysroot.gni +++ b/build/config/sysroot.gni @@ -33,17 +33,7 @@ if (current_toolchain == default_toolchain && target_sysroot != "") { sysroot = target_sysroot } else if (is_android) { import("//build/config/android/config.gni") - if (current_cpu == "x86") { - sysroot = rebase_path("$android_ndk_root/$x86_android_sysroot_subdir", root_build_dir) - } else if (current_cpu == "arm") { - sysroot = rebase_path("$android_ndk_root/$arm_android_sysroot_subdir", root_build_dir) - } else if (current_cpu == "x64") { - sysroot = rebase_path("$android_ndk_root/$x86_64_android_sysroot_subdir", root_build_dir) - } else if (current_cpu == "arm64") { - sysroot = rebase_path("$android_ndk_root/$arm64_android_sysroot_subdir", root_build_dir) - } else { - sysroot = "" - } + sysroot = rebase_path("$android_ndk_root/sysroot") } else if (is_mac) { import("//build/config/mac/mac_sdk.gni") sysroot = mac_sdk_path diff --git a/tools/android/VERSION_LINUX_NDK b/tools/android/VERSION_LINUX_NDK index 9fff8ff1660..4fac77be7b6 100644 --- a/tools/android/VERSION_LINUX_NDK +++ b/tools/android/VERSION_LINUX_NDK @@ -1 +1 @@ -50bb526ee77fee88b9382d2bf48e7399751b98ae +e626c47cb82a7439b0eda03ac6e0e9e1e41c6093 diff --git a/tools/android/VERSION_MACOSX_NDK b/tools/android/VERSION_MACOSX_NDK index aac8005d75f..bd19b05e9b3 100644 --- a/tools/android/VERSION_MACOSX_NDK +++ b/tools/android/VERSION_MACOSX_NDK @@ -1 +1 @@ -ed883dfaad6f27350eb3de426ccbf9abdc2943e8 +e8b6ecb5d15c4c4018a62b52aabc13e41b17df8f diff --git a/tools/buildtools/README.md b/tools/buildtools/README.md index 8cd1cfb23de..7d79285e94f 100644 --- a/tools/buildtools/README.md +++ b/tools/buildtools/README.md @@ -1,7 +1,7 @@ To build Dart for Mac and Linux, we pull Fuchsia's buildtools, which is also -used by Flutter. Fuchsia's buildtools includes gn, ninja, and the clang -toolchain. Fuchsia buildtools vends clang-format as part of the clang toolchain. -Since Fuchsia's buildtools doesn't vend a clang toolchain for Windows, we can't +used by Flutter. Fuchsia's buildtools includes gn, and the clang toolchain. +Fuchsia buildtools vends clang-format as part of the clang toolchain. Since +Fuchsia's buildtools doesn't vend a clang toolchain for Windows, we can't get a Windows clang-format binary from it. Therefore, from Chromium's buildtools here: diff --git a/tools/buildtools/update.py b/tools/buildtools/update.py index 5cae6a82f53..936e729b1a6 100755 --- a/tools/buildtools/update.py +++ b/tools/buildtools/update.py @@ -5,10 +5,10 @@ """Pulls down tools required to build Dart.""" +import errno import os import platform import subprocess -import shutil import sys THIS_DIR = os.path.abspath(os.path.dirname(__file__)) @@ -22,34 +22,6 @@ import find_depot_tools DEPOT_PATH = find_depot_tools.add_depot_tools_to_path() -def Update(): - path = os.path.join(BUILDTOOLS, 'update.sh') - command = ['/bin/bash', path, '--clang', '--gn'] - return subprocess.call(command, cwd=DART_ROOT) - - -def UpdateGNOnWindows(): - sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'gn.exe.sha1') - output_dir = os.path.join(BUILDTOOLS, 'win', 'gn.exe') - downloader_script = os.path.join( - DEPOT_PATH, 'download_from_google_storage.py') - download_cmd = [ - 'python', - downloader_script, - '--no_auth', - '--no_resume', - '--quiet', - '--platform=win*', - '--bucket', - 'chromium-gn', - '-s', - sha1_file, - '-o', - output_dir - ] - return subprocess.call(download_cmd) - - def UpdateClangFormatOnWindows(): sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'clang-format.exe.sha1') output_dir = os.path.join(BUILDTOOLS, 'win', 'clang-format.exe') @@ -72,9 +44,20 @@ def UpdateClangFormatOnWindows(): return subprocess.call(download_cmd) -# On Mac and Linux we copy clang-format and gn to the place where git cl format -# expects them to be. -def CopyClangFormat(): +def CreateSymlink(symlink, link_name): + try: + os.symlink(symlink, link_name) + except OSError, e: + if e.errno == errno.EEXIST: + os.remove(link_name) + os.symlink(symlink, link_name) + else: + raise e + + +# On Mac and Linux we symlink clang-format and gn to the place where +# 'git cl format' expects them to be. +def LinksForGitCLFormat(): if sys.platform == 'darwin': platform = 'darwin' tools = 'mac' @@ -89,14 +72,14 @@ def CopyClangFormat(): clang_format = os.path.join( BUILDTOOLS, toolchain, 'clang', 'bin', 'clang-format') - gn = os.path.join(BUILDTOOLS, toolchain, 'gn') + gn = os.path.join(BUILDTOOLS, 'gn') dest_dir = os.path.join(BUILDTOOLS, tools) if not os.path.exists(dest_dir): os.makedirs(dest_dir) clang_format_dest = os.path.join(dest_dir, 'clang-format') gn_dest = os.path.join(dest_dir, 'gn') - shutil.copy2(clang_format, clang_format_dest) - shutil.copy2(gn, gn_dest) + CreateSymlink(clang_format, clang_format_dest) + CreateSymlink(gn, gn_dest) return 0 @@ -107,13 +90,8 @@ def main(argv): print('Not downloading buildtools binaries for ' + arch_id) return 0 if sys.platform.startswith('win'): - result = UpdateGNOnWindows() - if result != 0: - return result return UpdateClangFormatOnWindows() - if Update() != 0: - return 1 - return CopyClangFormat() + return LinksForGitCLFormat() if __name__ == '__main__': diff --git a/tools/gn.py b/tools/gn.py index 9b348628b88..bc05a61847b 100755 --- a/tools/gn.py +++ b/tools/gn.py @@ -496,16 +496,11 @@ def Main(argv): starttime = time.time() args = parse_args(argv) - if sys.platform.startswith(('cygwin', 'win')): - subdir = 'win' - elif sys.platform == 'darwin': - subdir = 'mac-x64' - elif sys.platform.startswith('linux'): - subdir = 'linux-x64' - else: - print 'Unknown platform: ' + sys.platform + gn = os.path.join(DART_ROOT, 'buildtools', + 'gn.exe' if utils.IsWindows() else 'gn') + if not os.path.isfile(gn): + print "Couldn't find the gn binary at path: " + gn return 1 - gn = os.path.join(DART_ROOT, 'buildtools', subdir, 'gn') commands = [] for target_os in args.os: