From 726732dd6b02aa4f8117707071c54004a7c6e815 Mon Sep 17 00:00:00 2001 From: Zach Anderson Date: Sat, 6 May 2017 21:44:59 -0700 Subject: [PATCH] [infra] Roll clang to match the version used by Flutter Flutter uses the same clang toolchain as Fuchsia. This CL puts Dart on that toolchain, as well. This roll should entail no changes to glibc version requirements on Linux. The Fuchsia buildtools distribute clang-format with the toolchain rather than separately, so this CL introduces forwarding scripts that are copied to the right place under //buildtools. This CL sets the default for the number of workers in //tools/gn.py to 1, as the newer GN version fails occasionally when run concurrently with other invocations. As Flutter does, this CL uses the clang toolchain to build for Android. Thus, it is now possible to build for Android on MacOS. This change is a prerequisite for assembling the SDK in GN rather than with a python script: https://codereview.chromium.org/2848943003/ and will be nicer for making prebuilt SDKs for Fuchsia. Resubmitting this with the following fixes: 1. Rolls further forward to get past a GN bug 2. Fixes the Android build. 3. Style cleanups in gn.py Review-Url: https://codereview.chromium.org/2858873005 . --- DEPS | 137 +--- build/config/android/config.gni | 24 +- build/config/compiler/BUILD.gn | 54 +- build/toolchain/android/BUILD.gn | 14 +- build/toolchain/linux/BUILD.gn | 4 +- build/toolchain/mac/BUILD.gn | 6 +- tools/android/VERSION_LINUX_NDK | 1 + tools/android/VERSION_LINUX_SDK | 1 + tools/android/VERSION_MACOSX_NDK | 1 + tools/android/VERSION_MACOSX_SDK | 1 + tools/android/download_android_tools.py | 104 +++ tools/build.py | 2 +- tools/buildtools/linux64/clang-format | 20 + tools/buildtools/mac/clang-format | 20 + tools/buildtools/update.py | 88 ++ tools/buildtools/win/clang-format.exe.sha1 | 2 + tools/clang/scripts/update.py | 909 --------------------- tools/clang/scripts/update.sh | 724 ---------------- tools/gn.py | 91 +-- tools/ninja.py | 2 +- 20 files changed, 350 insertions(+), 1855 deletions(-) create mode 100644 tools/android/VERSION_LINUX_NDK create mode 100644 tools/android/VERSION_LINUX_SDK create mode 100644 tools/android/VERSION_MACOSX_NDK create mode 100644 tools/android/VERSION_MACOSX_SDK create mode 100644 tools/android/download_android_tools.py create mode 100755 tools/buildtools/linux64/clang-format create mode 100755 tools/buildtools/mac/clang-format create mode 100755 tools/buildtools/update.py create mode 100644 tools/buildtools/win/clang-format.exe.sha1 delete mode 100755 tools/clang/scripts/update.py delete mode 100755 tools/clang/scripts/update.sh diff --git a/DEPS b/DEPS index 6e0b46ed666..0624a196df5 100644 --- a/DEPS +++ b/DEPS @@ -23,6 +23,7 @@ vars = { # Chromium git "chromium_git": "https://chromium.googlesource.com", + "fuchsia_git": "https://fuchsia.googlesource.com", # Only use this temporarily while waiting for a mirror for a new package. "github_dartlang": "https://github.com/dart-lang/%s.git", @@ -30,9 +31,12 @@ vars = { "gyp_rev": "@6ee91ad8659871916f9aa840d42e1513befdf638", "co19_rev": "@4af9ef149be554216c5bb16cbac8e50d4c28cdf1", - # Revisions of GN related dependencies. - "buildtools_revision": "@39b1db2ab4aa4b2ccaa263c29bdf63e7c1ee28aa", - "clang_format_rev": "@0ed791d1387a3c9146ea6c453c646f3c0fc97784", + # Revisions of GN related dependencies. This should match the revision + # pulled by Flutter. + "buildtools_revision": "@057ef89874e3c622248cf99259434fdc683c4e30", + + # Scripts that make 'git cl format' work. + "clang_format_scripts_rev": "@c09c8deeac31f05bd801995c475e7c8070f9ecda", "gperftools_revision": "@02eeed29df112728564a5dde6417fa4622b57a06", @@ -132,11 +136,10 @@ deps = { # Stuff needed for GN build. Var("dart_root") + "/buildtools": - Var('chromium_git') + '/chromium/buildtools.git' + - Var('buildtools_revision'), + Var("fuchsia_git") + "/buildtools" + Var("buildtools_revision"), Var("dart_root") + "/buildtools/clang_format/script": Var("chromium_git") + "/chromium/llvm-project/cfe/tools/clang-format.git" + - Var("clang_format_rev"), + Var("clang_format_scripts_rev"), Var("dart_root") + "/tests/co19/src": (Var("github_mirror") % "co19") + Var("co19_rev"), @@ -346,11 +349,6 @@ deps = { } deps_os = { - "android": { - Var("dart_root") + "/third_party/android_tools": - Var("chromium_git") + "/android_tools.git" + - "@aaeda3d69df4b4352e3cac7c16bea7f16bd1ec12", - }, "win": { Var("dart_root") + "/third_party/cygwin": Var("chromium_git") + "/chromium/deps/cygwin.git" + @@ -361,98 +359,6 @@ deps_os = { # TODO(iposva): Move the necessary tools so that hooks can be run # without the runtime being available. hooks = [ - # Pull GN binaries. This needs to be before running GYP below. - { - 'name': 'gn_linux64', - 'pattern': '.', - 'action': [ - 'download_from_google_storage', - '--no_auth', - '--no_resume', - '--quiet', - '--platform=linux*', - '--bucket', - 'chromium-gn', - '-s', - Var('dart_root') + '/buildtools/linux64/gn.sha1', - ], - }, - { - 'name': 'gn_mac', - 'pattern': '.', - 'action': [ - 'download_from_google_storage', - '--no_auth', - '--no_resume', - '--quiet', - '--platform=darwin', - '--bucket', - 'chromium-gn', - '-s', - Var('dart_root') + '/buildtools/mac/gn.sha1', - ], - }, - { - 'name': 'gn_win', - 'pattern': '.', - 'action': [ - 'download_from_google_storage', - '--no_auth', - '--no_resume', - '--quiet', - '--platform=win*', - '--bucket', - 'chromium-gn', - '-s', - Var('dart_root') + '/buildtools/win/gn.exe.sha1', - ], - }, - # Pull clang-format binaries using checked-in hashes. - { - 'name': 'clang_format_win', - 'pattern': '.', - 'action': [ - 'download_from_google_storage', - '--no_auth', - '--no_resume', - '--quiet', - '--platform=win32', - '--bucket', - 'chromium-clang-format', - '-s', - Var('dart_root') + '/buildtools/win/clang-format.exe.sha1', - ], - }, - { - 'name': 'clang_format_linux', - 'pattern': '.', - 'action': [ - 'download_from_google_storage', - '--no_auth', - '--no_resume', - '--quiet', - '--platform=linux*', - '--bucket', - 'chromium-clang-format', - '-s', - Var('dart_root') + '/buildtools/linux64/clang-format.sha1', - ], - }, - { - 'name': 'clang_format_mac', - 'pattern': '.', - 'action': [ - 'download_from_google_storage', - '--no_auth', - '--no_resume', - '--quiet', - '--platform=darwin', - '--bucket', - 'chromium-clang-format', - '-s', - Var('dart_root') + '/buildtools/mac/clang-format.sha1', - ], - }, { 'name': 'd8_testing_binaries', 'pattern': '.', @@ -558,21 +464,6 @@ hooks = [ Var('dart_root') + "/third_party/gsutil.tar.gz.sha1", ], }, - { - "name": "clang", - "pattern": ".", - "action": [ - "download_from_google_storage", - "--no_auth", - "--no_resume", - "--bucket", - "dart-dependencies", - "--platform=linux*", - "--extract", - "-s", - Var('dart_root') + "/third_party/clang.tar.gz.sha1", - ], - }, { # Pull Debian wheezy sysroot for i386 Linux 'name': 'sysroot_i386', @@ -588,10 +479,14 @@ hooks = [ '--running-as-hook', '--arch', 'amd64'], }, { - # Pull clang if needed or requested via GYP_DEFINES. - 'name': 'gn_clang', + 'name': 'download_android_tools', 'pattern': '.', - 'action': ['python', 'sdk/tools/clang/scripts/update.py', '--if-needed'], + 'action': ['python', 'sdk/tools/android/download_android_tools.py'], + }, + { + 'name': 'buildtools', + 'pattern': '.', + 'action': ['python', 'sdk/tools/buildtools/update.py'], }, { # Update the Windows toolchain if necessary. diff --git a/build/config/android/config.gni b/build/config/android/config.gni index 33469666d33..38f81e830b1 100644 --- a/build/config/android/config.gni +++ b/build/config/android/config.gni @@ -62,30 +62,34 @@ if (is_android) { # Subdirectories inside android_ndk_root that contain the sysroot for the # associated platform. - _android_api_level = 16 + if (current_cpu == "x64" || current_cpu == "arm64") { + _android_api_level = 22 + } else { + _android_api_level = 16 + } x86_android_sysroot_subdir = "platforms/android-${_android_api_level}/arch-x86" arm_android_sysroot_subdir = "platforms/android-${_android_api_level}/arch-arm" mips_android_sysroot_subdir = "platforms/android-${_android_api_level}/arch-mips" - _android64_api_level = 21 x86_64_android_sysroot_subdir = - "platforms/android-${_android64_api_level}/arch-x86_64" + "platforms/android-${_android_api_level}/arch-x86_64" arm64_android_sysroot_subdir = - "platforms/android-${_android64_api_level}/arch-arm64" + "platforms/android-${_android_api_level}/arch-arm64" mips64_android_sysroot_subdir = - "platforms/android-${_android64_api_level}/arch-mips64" + "platforms/android-${_android_api_level}/arch-mips64" # Toolchain root directory for each build. The actual binaries are inside # a "bin" directory inside of these. _android_toolchain_version = "4.9" + _android_toolchain_detailed_version = "4.9.x" x86_android_toolchain_root = "$android_ndk_root/toolchains/x86-${_android_toolchain_version}/prebuilt/${android_host_os}-${android_host_arch}" arm_android_toolchain_root = "$android_ndk_root/toolchains/arm-linux-androideabi-${_android_toolchain_version}/prebuilt/${android_host_os}-${android_host_arch}" mips_android_toolchain_root = "$android_ndk_root/toolchains/mipsel-linux-android-${_android_toolchain_version}/prebuilt/${android_host_os}-${android_host_arch}" x86_64_android_toolchain_root = "$android_ndk_root/toolchains/x86_64-${_android_toolchain_version}/prebuilt/${android_host_os}-${android_host_arch}" arm64_android_toolchain_root = "$android_ndk_root/toolchains/aarch64-linux-android-${_android_toolchain_version}/prebuilt/${android_host_os}-${android_host_arch}" - mips64_android_toolchain_root = "$android_ndk_root/toolchains/mips64el-${_android_toolchain_version}/prebuilt/${android_host_os}-${android_host_arch}" + mips64_android_toolchain_root = "$android_ndk_root/toolchains/mips64el-linux-android-${_android_toolchain_version}/prebuilt/${android_host_os}-${android_host_arch}" # Location of libgcc. This is only needed for the current GN toolchain, so we # only need to define the current one, rather than one for every platform @@ -94,32 +98,26 @@ if (is_android) { android_prebuilt_arch = "android-x86" _binary_prefix = "i686-linux-android" android_toolchain_root = "$x86_android_toolchain_root" - android_libgcc_file = "$android_toolchain_root/lib/gcc/i686-linux-android/${_android_toolchain_version}/libgcc.a" } else if (current_cpu == "arm") { android_prebuilt_arch = "android-arm" _binary_prefix = "arm-linux-androideabi" android_toolchain_root = "$arm_android_toolchain_root" - android_libgcc_file = "$android_toolchain_root/lib/gcc/arm-linux-androideabi/${_android_toolchain_version}/libgcc.a" } else if (current_cpu == "mipsel") { android_prebuilt_arch = "android-mips" _binary_prefix = "mipsel-linux-android" android_toolchain_root = "$mips_android_toolchain_root" - android_libgcc_file = "$android_toolchain_root/lib/gcc/mipsel-linux-android/${_android_toolchain_version}/libgcc.a" } else if (current_cpu == "x64") { android_prebuilt_arch = "android-x86_64" _binary_prefix = "x86_64-linux-android" android_toolchain_root = "$x86_64_android_toolchain_root" - android_libgcc_file = "$android_toolchain_root/lib/gcc/x86_64-linux-android/${_android_toolchain_version}/libgcc.a" } else if (current_cpu == "arm64") { android_prebuilt_arch = "android-arm64" _binary_prefix = "aarch64-linux-android" android_toolchain_root = "$arm64_android_toolchain_root" - android_libgcc_file = "$android_toolchain_root/lib/gcc/aarch64-linux-android/${_android_toolchain_version}/libgcc.a" } else if (current_cpu == "mips64el") { android_prebuilt_arch = "android-mips64" _binary_prefix = "mips64el-linux-android" android_toolchain_root = "$mips64_android_toolchain_root" - android_libgcc_file = "$android_toolchain_root/lib/gcc/mips64el-linux-android/${_android_toolchain_version}/libgcc.a" } else { assert(false, "Need android libgcc support for your target arch.") } @@ -170,5 +168,5 @@ if (is_android) { assert(false, "Unknown Android ABI: " + current_cpu) } - android_log_tag = "\"flutter\"" + android_log_tag = "\"dart\"" } diff --git a/build/config/compiler/BUILD.gn b/build/config/compiler/BUILD.gn index 8c1818b2484..b33351bc992 100644 --- a/build/config/compiler/BUILD.gn +++ b/build/config/compiler/BUILD.gn @@ -130,17 +130,6 @@ config("compiler") { cflags_objcc += extra_flags } - if (is_clang) { - # This is here so that all files get recompiled after a clang roll and - # when turning clang on or off. (defines are passed via the command line, - # and build system rebuild things when their commandline changes). Nothing - # should ever read this define. - defines += - [ "CR_CLANG_REVISION=" + exec_script("//tools/clang/scripts/update.py", - [ "--print-revision" ], - "trim string") ] - } - # Mac-specific compiler flags setup. # ---------------------------------- if (is_mac) { @@ -365,12 +354,14 @@ config("compiler") { defines += [ "HAVE_SYS_UIO_H" ] # Use gold for Android for most CPU architectures. - if (current_cpu == "x86" || current_cpu == "x64" || current_cpu == "arm") { + if (current_cpu == "x86" || current_cpu == "x64" || current_cpu == "arm" || + current_cpu == "arm64") { ldflags += [ "-fuse-ld=gold" ] if (is_clang) { # Let clang find the ld.gold in the NDK. - ldflags += [ "--gcc-toolchain=" + - rebase_path(android_toolchain_root, root_build_dir) ] + toolchain_root = rebase_path(android_toolchain_root, root_build_dir) + ldflags += [ "--gcc-toolchain=$toolchain_root" ] + ldflags += [ "-B${toolchain_root}/bin" ] } } @@ -391,9 +382,15 @@ config("compiler") { if (current_cpu == "arm") { cflags += [ "--target=arm-linux-androideabi" ] ldflags += [ "--target=arm-linux-androideabi" ] + } else if (current_cpu == "arm64") { + cflags += [ "--target=aarch64-linux-android" ] + ldflags += [ "--target=aarch64-linux-android" ] } else if (current_cpu == "x86") { - cflags += [ "--target=x86-linux-androideabi" ] - ldflags += [ "--target=x86-linux-androideabi" ] + cflags += [ "--target=i686-linux-androideabi" ] + ldflags += [ "--target=i686-linux-androideabi" ] + } else if (current_cpu == "x64") { + cflags += [ "--target=x86_64-linux-androideabi" ] + ldflags += [ "--target=x86_64-linux-androideabi" ] } } } @@ -452,7 +449,6 @@ config("runtime_library") { # Work around incompatibilities between bionic and clang headers. defines += [ "__compiler_offsetof=__builtin_offsetof", - "nan=__builtin_nan", ] } @@ -483,25 +479,19 @@ config("runtime_library") { ] lib_dirs += [ "$android_libcpp_root/libs/$android_app_abi" ] - libs += [ "$android_libcpp_library" ] - if (current_cpu == "mipsel") { - libs += [ - # ld linker is used for mips Android, and ld does not accept library - # absolute path prefixed by "-l"; Since libgcc does not exist in mips - # sysroot the proper library will be linked. - # TODO(gordanac): Remove once gold linker is used for mips Android. - "gcc", - ] - } else { - libs += [ - # Manually link the libgcc.a that the cross compiler uses. This is - # absolute because the linker will look inside the sysroot if it's not. - rebase_path(android_libgcc_file), - ] + libs += [ + "$android_libcpp_library", + "c++abi", + "android_support", + ] + + if (current_cpu == "arm") { + libs += [ "unwind" ] } libs += [ + "gcc", "c", "dl", "m", diff --git a/build/toolchain/android/BUILD.gn b/build/toolchain/android/BUILD.gn index 5e4847349d1..51f03206088 100644 --- a/build/toolchain/android/BUILD.gn +++ b/build/toolchain/android/BUILD.gn @@ -47,17 +47,27 @@ template("android_gcc_toolchain") { is_clang = invoker.is_clang if (is_clang) { - prefix = rebase_path("//third_party/llvm-build/Release+Asserts/bin", + host_suffix = "" + if (host_os == "linux") { + host_suffix = "x86_64-linux" + } else if (host_os == "mac") { + host_suffix = "x86_64-darwin" + } else { + assert(false, "Unknown host") + } + + prefix = rebase_path("//buildtools/toolchain/clang+llvm-$host_suffix/bin", root_build_dir) cc = compiler_prefix + prefix + "/clang" cxx = compiler_prefix + prefix + "/clang++" + ar = prefix + "/llvm-ar" } else { cc = compiler_prefix + tool_prefix + "gcc" cxx = compiler_prefix + tool_prefix + "g++" + ar = tool_prefix + "ar" } - ar = tool_prefix + "ar" ld = cxx readelf = compiler_prefix + tool_prefix + "readelf" nm = compiler_prefix + tool_prefix + "nm" diff --git a/build/toolchain/linux/BUILD.gn b/build/toolchain/linux/BUILD.gn index 4925e15895c..f92005dfe50 100644 --- a/build/toolchain/linux/BUILD.gn +++ b/build/toolchain/linux/BUILD.gn @@ -60,7 +60,7 @@ gcc_toolchain("arm64") { } gcc_toolchain("clang_x86") { - prefix = rebase_path("//third_party/llvm-build/Release+Asserts/bin", + prefix = rebase_path("//buildtools/toolchain/clang+llvm-x86_64-linux/bin", root_build_dir) cc = "${compiler_prefix}$prefix/clang" cxx = "${compiler_prefix}$prefix/clang++" @@ -89,7 +89,7 @@ gcc_toolchain("x86") { } gcc_toolchain("clang_x64") { - prefix = rebase_path("//third_party/llvm-build/Release+Asserts/bin", + prefix = rebase_path("//buildtools/toolchain/clang+llvm-x86_64-linux/bin", root_build_dir) cc = "${compiler_prefix}$prefix/clang" cxx = "${compiler_prefix}$prefix/clang++" diff --git a/build/toolchain/mac/BUILD.gn b/build/toolchain/mac/BUILD.gn index b291f0050b6..811ab467f87 100644 --- a/build/toolchain/mac/BUILD.gn +++ b/build/toolchain/mac/BUILD.gn @@ -211,7 +211,7 @@ template("mac_toolchain") { mac_toolchain("clang_x64") { toolchain_cpu = "x64" toolchain_os = "mac" - prefix = rebase_path("//third_party/llvm-build/Release+Asserts/bin", + prefix = rebase_path("//buildtools/toolchain/clang+llvm-x86_64-darwin/bin", root_build_dir) cc = "${goma_prefix}$prefix/clang" cxx = "${goma_prefix}$prefix/clang++" @@ -221,10 +221,10 @@ mac_toolchain("clang_x64") { } # Toolchain used for Mac host (i386) targets. -mac_toolchain("clang_i386") { +mac_toolchain("clang_x86") { toolchain_cpu = "i386" toolchain_os = "mac" - prefix = rebase_path("//third_party/llvm-build/Release+Asserts/bin", + prefix = rebase_path("//buildtools/toolchain/clang+llvm-x86_64-darwin/bin", root_build_dir) cc = "${goma_prefix}$prefix/clang" cxx = "${goma_prefix}$prefix/clang++" diff --git a/tools/android/VERSION_LINUX_NDK b/tools/android/VERSION_LINUX_NDK new file mode 100644 index 00000000000..9fff8ff1660 --- /dev/null +++ b/tools/android/VERSION_LINUX_NDK @@ -0,0 +1 @@ +50bb526ee77fee88b9382d2bf48e7399751b98ae diff --git a/tools/android/VERSION_LINUX_SDK b/tools/android/VERSION_LINUX_SDK new file mode 100644 index 00000000000..d24f5c1f6ea --- /dev/null +++ b/tools/android/VERSION_LINUX_SDK @@ -0,0 +1 @@ +0d320c50b0ed188c7e1182388e2beb623a1d307d diff --git a/tools/android/VERSION_MACOSX_NDK b/tools/android/VERSION_MACOSX_NDK new file mode 100644 index 00000000000..aac8005d75f --- /dev/null +++ b/tools/android/VERSION_MACOSX_NDK @@ -0,0 +1 @@ +ed883dfaad6f27350eb3de426ccbf9abdc2943e8 diff --git a/tools/android/VERSION_MACOSX_SDK b/tools/android/VERSION_MACOSX_SDK new file mode 100644 index 00000000000..2441d725226 --- /dev/null +++ b/tools/android/VERSION_MACOSX_SDK @@ -0,0 +1 @@ +fa5ea0ca1e0c7c2e40914f3202c7545de4dbca9c diff --git a/tools/android/download_android_tools.py b/tools/android/download_android_tools.py new file mode 100644 index 00000000000..e16989f3146 --- /dev/null +++ b/tools/android/download_android_tools.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# Copyright 2017 The Dart project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# Downloads trimmed-down Android Tools from Google Cloud Storage and extracts +# them to INSTALL_DIR, updating INSTALL_DIR/VERSION_* stamp files with current +# version. Does nothing if INSTALL_DIR/VERSION_* are already up to date. + +import os +import shutil +import subprocess +import sys +import tarfile + +# Path constants. (All of these should be absolute paths.) +THIS_DIR = os.path.abspath(os.path.dirname(__file__)) +DART_ROOT = os.path.abspath(os.path.join(THIS_DIR, '..', '..')) +# Should be the same as in upload.py. +INSTALL_DIR = os.path.join(DART_ROOT, 'third_party', 'android_tools') + +sys.path.insert(0, os.path.join(DART_ROOT, 'tools')) +import find_depot_tools + +DEPOT_PATH = find_depot_tools.add_depot_tools_to_path() +GSUTIL_PATH = os.path.join(DEPOT_PATH, 'gsutil.py') + +def RunCommand(command): + """Run command and return success (True) or failure.""" + + print 'Running %s' % (str(command)) + if subprocess.call(command, shell=False) == 0: + return True + print 'Failed.' + return False + + +def GetInstalledVersion(version_stamp): + version_file = os.path.join(INSTALL_DIR, version_stamp) + if not os.path.exists(version_file): + return None + with open(version_file) as f: + return f.read().strip() + + +def VersionStampName(tools_name): + if sys.platform.startswith('linux'): + return 'VERSION_LINUX_' + tools_name.upper() + elif sys.platform == 'darwin': + return 'VERSION_MACOSX_' + tools_name.upper() + else: + print('NOTE: Will not download android tools. Unsupported platform: ' + sys.platform) + sys.exit(0) + + +def UpdateTools(tools_name): + """Downloads zipped tools from Google Cloud Storage and extracts them, + stamping current version.""" + + # Read latest version. + version_stamp = VersionStampName(tools_name) + version = '' + with open(os.path.join(THIS_DIR, version_stamp)) as f: + version = f.read().strip() + # Return if installed binaries are up to date. + if version == GetInstalledVersion(version_stamp): + return + + # Remove the old install directory checked out from git. + if os.path.exists(os.path.join(INSTALL_DIR, '.git')): + shutil.rmtree(INSTALL_DIR) + # Make sure that the install directory exists. + if not os.path.exists(INSTALL_DIR): + os.mkdir(INSTALL_DIR) + # Remove current installation. + tools_root = os.path.join(INSTALL_DIR, tools_name) + if os.path.exists(tools_root): + shutil.rmtree(tools_root) + + # Download tools from GCS. + archive_path = os.path.join(INSTALL_DIR, tools_name + '.tar.gz') + download_cmd = ['python', GSUTIL_PATH, 'cp', + 'gs://mojo/android/tool/%s.tar.gz' % version, + archive_path] + if not RunCommand(download_cmd): + print ('WARNING: Failed to download Android tools.') + return + + print "Extracting Android tools (" + tools_name + ")" + with tarfile.open(archive_path) as arch: + arch.extractall(INSTALL_DIR) + os.remove(archive_path) + # Write version as the last step. + with open(os.path.join(INSTALL_DIR, version_stamp), 'w+') as f: + f.write('%s\n' % version) + + +def main(): + UpdateTools('sdk') + UpdateTools('ndk') + + +if __name__ == '__main__': + sys.exit(main()) \ No newline at end of file diff --git a/tools/build.py b/tools/build.py index 90dfcd7e9f2..29c081f3899 100755 --- a/tools/build.py +++ b/tools/build.py @@ -131,7 +131,7 @@ def ProcessOptions(options, args): if os_name != 'android': print "Unsupported target os %s" % os_name return False - if not HOST_OS in ['linux']: + if not HOST_OS in ['linux', 'macos']: print ("Cross-compilation to %s is not supported on host os %s." % (os_name, HOST_OS)) return False diff --git a/tools/buildtools/linux64/clang-format b/tools/buildtools/linux64/clang-format new file mode 100755 index 00000000000..715ff745bb6 --- /dev/null +++ b/tools/buildtools/linux64/clang-format @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# Copyright 2017 The Dart project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Forwards to the clang-format for the Linux toolchain.""" + +import os +import subprocess +import sys + +DART_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TOOLCHAIN = os.path.join(DART_ROOT, 'buildtools', 'toolchain') +CLANG_FORMAT = os.path.join(TOOLCHAIN, 'clang+llvm-x86_64-linux', 'bin', 'clang-format') + +def main(argv): + return subprocess.call([CLANG_FORMAT] + argv[1:]) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/tools/buildtools/mac/clang-format b/tools/buildtools/mac/clang-format new file mode 100755 index 00000000000..39549a91daa --- /dev/null +++ b/tools/buildtools/mac/clang-format @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# Copyright 2017 The Dart project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Forwards to the clang-format for the MacOS toolchain.""" + +import os +import subprocess +import sys + +DART_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +TOOLCHAIN = os.path.join(DART_ROOT, 'buildtools', 'toolchain') +CLANG_FORMAT = os.path.join(TOOLCHAIN, 'clang+llvm-x86_64-darwin', 'bin', 'clang-format') + +def main(argv): + return subprocess.call([CLANG_FORMAT] + argv[1:]) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/tools/buildtools/update.py b/tools/buildtools/update.py new file mode 100755 index 00000000000..6d676293aa4 --- /dev/null +++ b/tools/buildtools/update.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +# Copyright 2017 The Dart project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Pulls down tools required to build Dart.""" + +import os +import subprocess +import shutil +import sys + +THIS_DIR = os.path.abspath(os.path.dirname(__file__)) +DART_ROOT = os.path.abspath(os.path.join(THIS_DIR, '..', '..')) +BUILDTOOLS = os.path.join(DART_ROOT, 'buildtools') +TOOLS_BUILDTOOLS = os.path.join(DART_ROOT, 'tools', 'buildtools') + +sys.path.insert(0, os.path.join(DART_ROOT, 'tools')) +import find_depot_tools + +DEPOT_PATH = find_depot_tools.add_depot_tools_to_path() + + +def Update(): + path = os.path.join(BUILDTOOLS, 'update.sh') + return subprocess.call(['/bin/bash', path, '--toolchain', '--gn'], cwd=DART_ROOT) + + +def UpdateGNOnWindows(): + sha1_file = os.path.join(BUILDTOOLS, 'win', 'gn.exe.sha1') + downloader_script = os.path.join(DEPOT_PATH, 'download_from_google_storage.py') + download_cmd = [ + 'python', + downloader_script, + '--no_auth', + '--no_resume', + '--quiet', + '--platform=win*', + '--bucket', + 'chromium-gn', + '-s', + sha1_file + ] + return subprocess.call(download_cmd) + + +def UpdateClangFormatOnWindows(): + sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'clang-format.exe.sha1') + output_dir = os.path.join(BUILDTOOLS, 'win', 'clang-format.exe') + downloader_script = os.path.join(DEPOT_PATH, 'download_from_google_storage.py') + download_cmd = [ + 'python', + downloader_script, + '--no_auth', + '--no_resume', + '--quiet', + '--platform=win', + '--bucket', + 'chromium-clang-format', + '-s', + sha1_file, + '-o', + output_dir + ] + return subprocess.call(download_cmd) + + +def CopyClangFormatScripts(): + linux_script = os.path.join(TOOLS_BUILDTOOLS, 'linux64', 'clang-format') + mac_script = os.path.join(TOOLS_BUILDTOOLS, 'mac', 'clang-format') + linux_dest = os.path.join(BUILDTOOLS, 'linux64', 'clang-format') + mac_dest = os.path.join(BUILDTOOLS, 'mac', 'clang-format') + shutil.copy2(linux_script, linux_dest) + shutil.copy2(mac_script, mac_dest) + + +def main(argv): + if sys.platform.startswith('win'): + result = UpdateGNOnWindows() + if result != 0: + return result + return UpdateClangFormatOnWindows() + CopyClangFormatScripts() + return Update() + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/tools/buildtools/win/clang-format.exe.sha1 b/tools/buildtools/win/clang-format.exe.sha1 new file mode 100644 index 00000000000..a78543133b6 --- /dev/null +++ b/tools/buildtools/win/clang-format.exe.sha1 @@ -0,0 +1,2 @@ +6ddedd571c56b8c184f30a3c1fc36984e8c10ccd + diff --git a/tools/clang/scripts/update.py b/tools/clang/scripts/update.py deleted file mode 100755 index 100bdec5cc8..00000000000 --- a/tools/clang/scripts/update.py +++ /dev/null @@ -1,909 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""This script is used to download prebuilt clang binaries. - -It is also used by package.py to build the prebuilt clang binaries.""" - -import argparse -import distutils.spawn -import glob -import os -import pipes -import re -import shutil -import subprocess -import stat -import sys -import tarfile -import tempfile -import time -import urllib2 -import zipfile - - -# Do NOT CHANGE this if you don't know what you're doing -- see -# https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md -# Reverting problematic clang rolls is safe, though. -CLANG_REVISION = '282487' - -use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ -if use_head_revision: - CLANG_REVISION = 'HEAD' - -# This is incremented when pushing a new build of Clang at the same revision. -CLANG_SUB_REVISION=1 - -PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION) - -# Path constants. (All of these should be absolute paths.) -THIS_DIR = os.path.abspath(os.path.dirname(__file__)) -CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) -THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party') -LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm') -LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap') -LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, - 'llvm-bootstrap-install') -LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin') -CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools') -LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build', - 'Release+Asserts') -COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'compiler-rt') -CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang') -LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld') -# compiler-rt is built as part of the regular LLVM build on Windows to get -# the 64-bit runtime, and out-of-tree elsewhere. -# TODO(thakis): Try to unify this. -if sys.platform == 'win32': - COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt') -else: - COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'compiler-rt') -LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx') -LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi') -LLVM_BUILD_TOOLS_DIR = os.path.abspath( - os.path.join(LLVM_DIR, '..', 'llvm-build-tools')) -STAMP_FILE = os.path.normpath( - os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision')) -BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils') -BINUTILS_BIN_DIR = os.path.join(BINUTILS_DIR, BINUTILS_DIR, - 'Linux_x64', 'Release', 'bin') -BFD_PLUGINS_DIR = os.path.join(BINUTILS_DIR, 'Linux_x64', 'Release', - 'lib', 'bfd-plugins') -VERSION = '4.0.0' -ANDROID_NDK_DIR = os.path.join( - CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk') - -# URL for pre-built binaries. -CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE', - 'https://commondatastorage.googleapis.com/chromium-browser-clang') - -LLVM_REPO_URL='https://llvm.org/svn/llvm-project' -if 'LLVM_REPO_URL' in os.environ: - LLVM_REPO_URL = os.environ['LLVM_REPO_URL'] - -# Bump after VC updates. -DIA_DLL = { - '2013': 'msdia120.dll', - '2015': 'msdia140.dll', -} - - -def DownloadUrl(url, output_file): - """Download url into output_file.""" - CHUNK_SIZE = 4096 - TOTAL_DOTS = 10 - num_retries = 3 - retry_wait_s = 5 # Doubled at each retry. - - while True: - try: - sys.stdout.write('Downloading %s ' % url) - sys.stdout.flush() - response = urllib2.urlopen(url) - total_size = int(response.info().getheader('Content-Length').strip()) - bytes_done = 0 - dots_printed = 0 - while True: - chunk = response.read(CHUNK_SIZE) - if not chunk: - break - output_file.write(chunk) - bytes_done += len(chunk) - num_dots = TOTAL_DOTS * bytes_done / total_size - sys.stdout.write('.' * (num_dots - dots_printed)) - sys.stdout.flush() - dots_printed = num_dots - if bytes_done != total_size: - raise urllib2.URLError("only got %d of %d bytes" % - (bytes_done, total_size)) - print ' Done.' - return - except urllib2.URLError as e: - sys.stdout.write('\n') - print e - if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404: - raise e - num_retries -= 1 - print 'Retrying in %d s ...' % retry_wait_s - time.sleep(retry_wait_s) - retry_wait_s *= 2 - - -def EnsureDirExists(path): - if not os.path.exists(path): - print "Creating directory %s" % path - os.makedirs(path) - - -def DownloadAndUnpack(url, output_dir): - with tempfile.TemporaryFile() as f: - DownloadUrl(url, f) - f.seek(0) - EnsureDirExists(output_dir) - if url.endswith('.zip'): - zipfile.ZipFile(f).extractall(path=output_dir) - else: - tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir) - - -def ReadStampFile(path=STAMP_FILE): - """Return the contents of the stamp file, or '' if it doesn't exist.""" - try: - with open(path, 'r') as f: - return f.read().rstrip() - except IOError: - return '' - - -def WriteStampFile(s, path=STAMP_FILE): - """Write s to the stamp file.""" - EnsureDirExists(os.path.dirname(path)) - with open(path, 'w') as f: - f.write(s) - f.write('\n') - - -def GetSvnRevision(svn_repo): - """Returns current revision of the svn repo at svn_repo.""" - if sys.platform == 'darwin': - # mac_files toolchain must be set for hermetic builds. - root = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(__file__)))) - sys.path.append(os.path.join(root, 'build')) - import mac_toolchain - - mac_toolchain.SetToolchainEnvironment() - svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True) - m = re.search(r'Revision: (\d+)', svn_info) - return m.group(1) - - -def RmTree(dir): - """Delete dir.""" - def ChmodAndRetry(func, path, _): - # Subversion can leave read-only files around. - if not os.access(path, os.W_OK): - os.chmod(path, stat.S_IWUSR) - return func(path) - raise - - shutil.rmtree(dir, onerror=ChmodAndRetry) - - -def RmCmakeCache(dir): - """Delete CMake cache related files from dir.""" - for dirpath, dirs, files in os.walk(dir): - if 'CMakeCache.txt' in files: - os.remove(os.path.join(dirpath, 'CMakeCache.txt')) - if 'CMakeFiles' in dirs: - RmTree(os.path.join(dirpath, 'CMakeFiles')) - - -def RunCommand(command, msvc_arch=None, env=None, fail_hard=True): - """Run command and return success (True) or failure; or if fail_hard is - True, exit on failure. If msvc_arch is set, runs the command in a - shell with the msvc tools for that architecture.""" - - if msvc_arch and sys.platform == 'win32': - command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command - - # https://docs.python.org/2/library/subprocess.html: - # "On Unix with shell=True [...] if args is a sequence, the first item - # specifies the command string, and any additional items will be treated as - # additional arguments to the shell itself. That is to say, Popen does the - # equivalent of: - # Popen(['/bin/sh', '-c', args[0], args[1], ...])" - # - # We want to pass additional arguments to command[0], not to the shell, - # so manually join everything into a single string. - # Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should - # quote c:\path but svn can't handle quoted paths on Windows. Since on - # Windows follow-on args are passed to args[0] instead of the shell, don't - # do the single-string transformation there. - if sys.platform != 'win32': - command = ' '.join([pipes.quote(c) for c in command]) - print 'Running', command - if subprocess.call(command, env=env, shell=True) == 0: - return True - print 'Failed.' - if fail_hard: - sys.exit(1) - return False - - -def CopyFile(src, dst): - """Copy a file from src to dst.""" - print "Copying %s to %s" % (src, dst) - shutil.copy(src, dst) - - -def CopyDirectoryContents(src, dst, filename_filter=None): - """Copy the files from directory src to dst - with an optional filename filter.""" - dst = os.path.realpath(dst) # realpath() in case dst ends in /.. - EnsureDirExists(dst) - for root, _, files in os.walk(src): - for f in files: - if filename_filter and not re.match(filename_filter, f): - continue - CopyFile(os.path.join(root, f), dst) - - -def Checkout(name, url, dir): - """Checkout the SVN module at url into dir. Use name for the log message.""" - print "Checking out %s r%s into '%s'" % (name, CLANG_REVISION, dir) - - command = ['svn', 'checkout', '--force', url + '@' + CLANG_REVISION, dir] - if RunCommand(command, fail_hard=False): - return - - if os.path.isdir(dir): - print "Removing %s." % (dir) - RmTree(dir) - - print "Retrying." - RunCommand(command) - - -def DeleteChromeToolsShim(): - OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools') - shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True) - shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True) - - -def CreateChromeToolsShim(): - """Hooks the Chrome tools into the LLVM build. - - Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build - detects implicit tools in the tools subdirectory, so this helper install a - shim CMakeLists.txt that forwards to the real directory for the Chrome tools. - - Note that the shim directory name intentionally has no - or _. The implicit - tool detection logic munges them in a weird way.""" - assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_') - os.mkdir(CHROME_TOOLS_SHIM_DIR) - with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f: - f.write('# Automatically generated by tools/clang/scripts/update.py. ' + - 'Do not edit.\n') - f.write('# Since tools/clang is located in another directory, use the \n') - f.write('# two arg version to specify where build artifacts go. CMake\n') - f.write('# disallows reuse of the same binary dir for multiple source\n') - f.write('# dirs, so the build artifacts need to go into a subdirectory.\n') - f.write('# dirs, so the build artifacts need to go into a subdirectory.\n') - f.write('if (CHROMIUM_TOOLS_SRC)\n') - f.write(' add_subdirectory(${CHROMIUM_TOOLS_SRC} ' + - '${CMAKE_CURRENT_BINARY_DIR}/a)\n') - f.write('endif (CHROMIUM_TOOLS_SRC)\n') - - -def DownloadHostGcc(args): - """Downloads gcc 4.8.5 and makes sure args.gcc_toolchain is set.""" - if not sys.platform.startswith('linux') or args.gcc_toolchain: - return - # Unconditionally download a prebuilt gcc to guarantee the included libstdc++ - # works on Ubuntu Precise. - gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc485precise') - if not os.path.exists(gcc_dir): - print 'Downloading pre-built GCC 4.8.5...' - DownloadAndUnpack( - CDS_URL + '/tools/gcc485precise.tgz', LLVM_BUILD_TOOLS_DIR) - args.gcc_toolchain = gcc_dir - - -def AddCMakeToPath(): - """Download CMake and add it to PATH.""" - if sys.platform == 'win32': - zip_name = 'cmake-3.4.3-win32-x86.zip' - cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, - 'cmake-3.4.3-win32-x86', 'bin') - else: - suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux' - zip_name = 'cmake343_%s.tgz' % suffix - cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake343', 'bin') - if not os.path.exists(cmake_dir): - DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR) - os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '') - - -def AddGnuWinToPath(): - """Download some GNU win tools and add them to PATH.""" - if sys.platform != 'win32': - return - - gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin') - GNUWIN_VERSION = '5' - GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp') - if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION: - print 'GNU Win tools already up to date.' - else: - zip_name = 'gnuwin-%s.zip' % GNUWIN_VERSION - DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR) - WriteStampFile(GNUWIN_VERSION, GNUWIN_STAMP) - - os.environ['PATH'] = gnuwin_dir + os.pathsep + os.environ.get('PATH', '') - - -vs_version = None -def GetVSVersion(): - global vs_version - if vs_version: - return vs_version - - # Try using the toolchain in depot_tools. - # This sets environment variables used by SelectVisualStudioVersion below. - sys.path.append(os.path.join(CHROMIUM_DIR, 'build')) - import vs_toolchain - vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs() - - # Use gyp to find the MSVS installation, either in depot_tools as per above, - # or a system-wide installation otherwise. - sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib')) - import gyp.MSVSVersion - vs_version = gyp.MSVSVersion.SelectVisualStudioVersion( - vs_toolchain.GetVisualStudioVersion()) - return vs_version - - -def CopyDiaDllTo(target_dir): - # This script always wants to use the 64-bit msdia*.dll. - dia_path = os.path.join(GetVSVersion().Path(), 'DIA SDK', 'bin', 'amd64') - dia_dll = os.path.join(dia_path, DIA_DLL[GetVSVersion().ShortName()]) - CopyFile(dia_dll, target_dir) - - -def VeryifyVersionOfBuiltClangMatchesVERSION(): - """Checks that `clang --version` outputs VERSION. If this fails, VERSION - in this file is out-of-date and needs to be updated (possibly in an - `if use_head_revision:` block in main() first).""" - clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang') - if sys.platform == 'win32': - # TODO: Parse `clang-cl /?` output for built clang's version and check that - # to check the binary we're actually shipping? But clang-cl.exe is just - # a copy of clang.exe, so this does check the same thing. - clang += '.exe' - version_out = subprocess.check_output([clang, '--version']) - version_out = re.match(r'clang version ([0-9.]+)', version_out).group(1) - if version_out != VERSION: - print ('unexpected clang version %s (not %s), update VERSION in update.py' - % (version_out, VERSION)) - sys.exit(1) - - -def UpdateClang(args): - print 'Updating Clang to %s...' % PACKAGE_VERSION - - need_gold_plugin = 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ or ( - sys.platform.startswith('linux') and - 'buildtype=Official' in os.environ.get('GYP_DEFINES', '') and - 'branding=Chrome' in os.environ.get('GYP_DEFINES', '')) - - if ReadStampFile() == PACKAGE_VERSION and not args.force_local_build: - print 'Clang is already up to date.' - if not need_gold_plugin or os.path.exists( - os.path.join(LLVM_BUILD_DIR, "lib/LLVMgold.so")): - return 0 - - # Reset the stamp file in case the build is unsuccessful. - WriteStampFile('') - - if not args.force_local_build: - cds_file = "clang-%s.tgz" % PACKAGE_VERSION - if sys.platform == 'win32' or sys.platform == 'cygwin': - cds_full_url = CDS_URL + '/Win/' + cds_file - elif sys.platform == 'darwin': - cds_full_url = CDS_URL + '/Mac/' + cds_file - else: - assert sys.platform.startswith('linux') - cds_full_url = CDS_URL + '/Linux_x64/' + cds_file - - print 'Downloading prebuilt clang' - if os.path.exists(LLVM_BUILD_DIR): - RmTree(LLVM_BUILD_DIR) - try: - DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR) - print 'clang %s unpacked' % PACKAGE_VERSION - if sys.platform == 'win32': - CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin')) - # Download the gold plugin if requested to by an environment variable. - # This is used by the CFI ClusterFuzz bot, and it's required for official - # builds on linux. - if need_gold_plugin: - RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py']) - WriteStampFile(PACKAGE_VERSION) - return 0 - except urllib2.URLError: - print 'Failed to download prebuilt clang %s' % cds_file - print 'Use --force-local-build if you want to build locally.' - print 'Exiting.' - return 1 - - if args.with_android and not os.path.exists(ANDROID_NDK_DIR): - print 'Android NDK not found at ' + ANDROID_NDK_DIR - print 'The Android NDK is needed to build a Clang whose -fsanitize=address' - print 'works on Android. See ' - print 'https://www.chromium.org/developers/how-tos/android-build-instructions' - print 'for how to install the NDK, or pass --without-android.' - return 1 - - DownloadHostGcc(args) - AddCMakeToPath() - AddGnuWinToPath() - - DeleteChromeToolsShim() - - Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR) - Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR) - if sys.platform == 'win32' or use_head_revision: - Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR) - Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR) - if sys.platform == 'darwin': - # clang needs a libc++ checkout, else -stdlib=libc++ won't find includes - # (i.e. this is needed for bootstrap builds). - Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR) - # We used to check out libcxxabi on OS X; we no longer need that. - if os.path.exists(LIBCXXABI_DIR): - RmTree(LIBCXXABI_DIR) - - cc, cxx = None, None - libstdcpp = None - if args.gcc_toolchain: # This option is only used on Linux. - # Use the specified gcc installation for building. - cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc') - cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++') - - if not os.access(cc, os.X_OK): - print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain - print '"%s" does not appear to be valid.' % cc - return 1 - - # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap - # compiler, etc.) find the .so. - libstdcpp = subprocess.check_output( - [cxx, '-print-file-name=libstdc++.so.6']).rstrip() - os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp) - - cflags = [] - cxxflags = [] - ldflags = [] - - base_cmake_args = ['-GNinja', - '-DCMAKE_BUILD_TYPE=Release', - '-DLLVM_ENABLE_ASSERTIONS=ON', - '-DLLVM_ENABLE_THREADS=OFF', - '-DLLVM_ENABLE_TIMESTAMPS=OFF', - # Statically link MSVCRT to avoid DLL dependencies. - '-DLLVM_USE_CRT_RELEASE=MT', - ] - - binutils_incdir = '' - if sys.platform.startswith('linux'): - binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include') - - if args.bootstrap: - print 'Building bootstrap compiler' - EnsureDirExists(LLVM_BOOTSTRAP_DIR) - os.chdir(LLVM_BOOTSTRAP_DIR) - bootstrap_args = base_cmake_args + [ - '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, - '-DLLVM_TARGETS_TO_BUILD=host', - '-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR, - '-DCMAKE_C_FLAGS=' + ' '.join(cflags), - '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), - ] - if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc) - if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx) - RmCmakeCache('.') - RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64') - RunCommand(['ninja'], msvc_arch='x64') - if args.run_tests: - if sys.platform == 'win32': - CopyDiaDllTo(os.path.join(LLVM_BOOTSTRAP_DIR, 'bin')) - RunCommand(['ninja', 'check-all'], msvc_arch='x64') - RunCommand(['ninja', 'install'], msvc_arch='x64') - if args.gcc_toolchain: - # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap - # compiler can start. - CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib')) - - if sys.platform == 'win32': - cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe') - cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe') - # CMake has a hard time with backslashes in compiler paths: - # https://stackoverflow.com/questions/13050827 - cc = cc.replace('\\', '/') - cxx = cxx.replace('\\', '/') - else: - cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang') - cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++') - - if args.gcc_toolchain: - # Tell the bootstrap compiler to use a specific gcc prefix to search - # for standard library headers and shared object files. - cflags = ['--gcc-toolchain=' + args.gcc_toolchain] - cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain] - print 'Building final compiler' - - # Build LLVM gold plugin with LTO. That speeds up the linker by ~10%. - # We only use LTO for Linux now. - if args.bootstrap and args.lto_gold_plugin: - print 'Building LTO LLVM Gold plugin' - if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR): - RmTree(LLVM_LTO_GOLD_PLUGIN_DIR) - EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR) - os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR) - - # Create a symlink to LLVMgold.so build in the previous step so that ar - # and ranlib could find it while linking LLVMgold.so with LTO. - EnsureDirExists(BFD_PLUGINS_DIR) - RunCommand(['ln', '-sf', - os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'), - os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')]) - - lto_cflags = ['-flto'] - lto_ldflags = ['-fuse-ld=gold'] - if args.gcc_toolchain: - # Tell the bootstrap compiler to use a specific gcc prefix to search - # for standard library headers and shared object files. - lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain] - lto_cmake_args = base_cmake_args + [ - '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, - '-DCMAKE_C_COMPILER=' + cc, - '-DCMAKE_CXX_COMPILER=' + cxx, - '-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags), - '-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags), - '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags), - '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags), - '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)] - - # We need to use the proper binutils which support LLVM Gold plugin. - lto_env = os.environ.copy() - lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '') - - RmCmakeCache('.') - RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env) - RunCommand(['ninja', 'LLVMgold'], env=lto_env) - - - # LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is - # needed, on OS X it requires libc++. clang only automatically links to libc++ - # when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run - # on OS X versions as old as 10.7. - deployment_target = '' - - if sys.platform == 'darwin' and args.bootstrap: - # When building on 10.9, /usr/include usually doesn't exist, and while - # Xcode's clang automatically sets a sysroot, self-built clangs don't. - cflags = ['-isysroot', subprocess.check_output( - ['xcrun', '--show-sdk-path']).rstrip()] - cxxflags = ['-stdlib=libc++'] + cflags - ldflags += ['-stdlib=libc++'] - deployment_target = '10.7' - # Running libc++ tests takes a long time. Since it was only needed for - # the install step above, don't build it as part of the main build. - # This makes running package.py over 10% faster (30 min instead of 34 min) - RmTree(LIBCXX_DIR) - - # Build clang. - - # If building at head, define a macro that plugins can use for #ifdefing - # out code that builds at head, but not at CLANG_REVISION or vice versa. - if use_head_revision: - cflags += ['-DLLVM_FORCE_HEAD_REVISION'] - cxxflags += ['-DLLVM_FORCE_HEAD_REVISION'] - - CreateChromeToolsShim() - - deployment_env = None - if deployment_target: - deployment_env = os.environ.copy() - deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target - - cmake_args = [] - # TODO(thakis): Unconditionally append this to base_cmake_args instead once - # compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698) - cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args - if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc) - if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx) - cmake_args += base_cmake_args + [ - '-DLLVM_BINUTILS_INCDIR=' + binutils_incdir, - '-DCMAKE_C_FLAGS=' + ' '.join(cflags), - '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags), - '-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags), - '-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags), - '-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags), - '-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR, - # TODO(thakis): Remove this once official builds pass -Wl,--build-id - # explicitly, https://crbug.com/622775 - '-DENABLE_LINKER_BUILD_ID=ON', - '-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'), - '-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)] - - EnsureDirExists(LLVM_BUILD_DIR) - os.chdir(LLVM_BUILD_DIR) - RmCmakeCache('.') - RunCommand(['cmake'] + cmake_args + [LLVM_DIR], - msvc_arch='x64', env=deployment_env) - - if args.gcc_toolchain: - # Copy in the right stdlibc++.so.6 so clang can start. - if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')): - os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib')) - libstdcpp = subprocess.check_output( - [cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip() - CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib')) - - RunCommand(['ninja'], msvc_arch='x64') - - if args.tools: - # If any Chromium tools were built, install those now. - RunCommand(['ninja', 'cr-install'], msvc_arch='x64') - - if sys.platform == 'darwin': - # See http://crbug.com/256342 - RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')]) - elif sys.platform.startswith('linux'): - RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')]) - - VeryifyVersionOfBuiltClangMatchesVERSION() - - # Do an out-of-tree build of compiler-rt. - # On Windows, this is used to get the 32-bit ASan run-time. - # TODO(hans): Remove once the regular build above produces this. - # On Mac and Linux, this is used to get the regular 64-bit run-time. - # Do a clobbered build due to cmake changes. - if os.path.isdir(COMPILER_RT_BUILD_DIR): - RmTree(COMPILER_RT_BUILD_DIR) - os.makedirs(COMPILER_RT_BUILD_DIR) - os.chdir(COMPILER_RT_BUILD_DIR) - # TODO(thakis): Add this once compiler-rt can build with clang-cl (see - # above). - #if args.bootstrap and sys.platform == 'win32': - # The bootstrap compiler produces 64-bit binaries by default. - #cflags += ['-m32'] - #cxxflags += ['-m32'] - compiler_rt_args = base_cmake_args + [ - '-DCMAKE_C_FLAGS=' + ' '.join(cflags), - '-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)] - if sys.platform == 'darwin': - compiler_rt_args += ['-DCOMPILER_RT_ENABLE_IOS=ON'] - if sys.platform != 'win32': - compiler_rt_args += ['-DLLVM_CONFIG_PATH=' + - os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'), - '-DSANITIZER_MIN_OSX_VERSION="10.7"'] - # compiler-rt is part of the llvm checkout on Windows but a stand-alone - # directory elsewhere, see the TODO above COMPILER_RT_DIR. - RmCmakeCache('.') - RunCommand(['cmake'] + compiler_rt_args + - [LLVM_DIR if sys.platform == 'win32' else COMPILER_RT_DIR], - msvc_arch='x86', env=deployment_env) - RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86') - - # Copy select output to the main tree. - # TODO(hans): Make this (and the .gypi and .isolate files) version number - # independent. - if sys.platform == 'win32': - platform = 'windows' - elif sys.platform == 'darwin': - platform = 'darwin' - else: - assert sys.platform.startswith('linux') - platform = 'linux' - asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', platform) - if sys.platform == 'win32': - # TODO(thakis): This too is due to compiler-rt being part of the checkout - # on Windows, see TODO above COMPILER_RT_DIR. - asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang', - VERSION, 'lib', platform) - asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', - VERSION, 'lib', platform) - # Blacklists: - CopyDirectoryContents(os.path.join(asan_rt_lib_src_dir, '..', '..'), - os.path.join(asan_rt_lib_dst_dir, '..', '..'), - r'^.*blacklist\.txt$') - # Headers: - if sys.platform != 'win32': - CopyDirectoryContents( - os.path.join(COMPILER_RT_BUILD_DIR, 'include/sanitizer'), - os.path.join(LLVM_BUILD_DIR, 'lib/clang', VERSION, 'include/sanitizer')) - # Static and dynamic libraries: - CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir) - if sys.platform == 'darwin': - for dylib in glob.glob(os.path.join(asan_rt_lib_dst_dir, '*.dylib')): - # Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to - # @executable_path. - # TODO(glider): this is transitional. We'll need to fix the dylib - # name either in our build system, or in Clang. See also - # http://crbug.com/344836. - subprocess.call(['install_name_tool', '-id', - '@executable_path/' + os.path.basename(dylib), dylib]) - - - if sys.platform == 'win32': - # Make an extra copy of the sanitizer headers, to be put on the include path - # of the fallback compiler. - sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', - VERSION, 'include', 'sanitizer') - aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang', - VERSION, 'include_sanitizer', - 'sanitizer') - EnsureDirExists(aux_sanitizer_include_dir) - for _, _, files in os.walk(sanitizer_include_dir): - for f in files: - CopyFile(os.path.join(sanitizer_include_dir, f), - aux_sanitizer_include_dir) - - if args.with_android: - make_toolchain = os.path.join( - ANDROID_NDK_DIR, 'build', 'tools', 'make_standalone_toolchain.py') - for target_arch in ['aarch64', 'arm', 'i686']: - # Make standalone Android toolchain for target_arch. - toolchain_dir = os.path.join( - LLVM_BUILD_DIR, 'android-toolchain-' + target_arch) - RunCommand([ - make_toolchain, - '--api=' + ('21' if target_arch == 'aarch64' else '19'), - '--force', - '--install-dir=%s' % toolchain_dir, - '--stl=stlport', - '--arch=' + { - 'aarch64': 'arm64', - 'arm': 'arm', - 'i686': 'x86', - }[target_arch]]) - # Android NDK r9d copies a broken unwind.h into the toolchain, see - # http://crbug.com/357890 - for f in glob.glob(os.path.join(toolchain_dir, 'include/c++/*/unwind.h')): - os.remove(f) - - # Build ASan runtime for Android in a separate build tree. - build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch) - if not os.path.exists(build_dir): - os.mkdir(os.path.join(build_dir)) - os.chdir(build_dir) - cflags = ['--target=%s-linux-androideabi' % target_arch, - '--sysroot=%s/sysroot' % toolchain_dir, - '-B%s' % toolchain_dir] - android_args = base_cmake_args + [ - '-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'), - '-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'), - '-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'), - '-DCMAKE_C_FLAGS=' + ' '.join(cflags), - '-DCMAKE_CXX_FLAGS=' + ' '.join(cflags), - '-DANDROID=1'] - RmCmakeCache('.') - RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR]) - RunCommand(['ninja', 'libclang_rt.asan-%s-android.so' % target_arch]) - - # And copy it into the main build tree. - runtime = 'libclang_rt.asan-%s-android.so' % target_arch - for root, _, files in os.walk(build_dir): - if runtime in files: - shutil.copy(os.path.join(root, runtime), asan_rt_lib_dst_dir) - - # Run tests. - if args.run_tests or use_head_revision: - os.chdir(LLVM_BUILD_DIR) - RunCommand(['ninja', 'cr-check-all'], msvc_arch='x64') - if args.run_tests: - if sys.platform == 'win32': - CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin')) - os.chdir(LLVM_BUILD_DIR) - RunCommand(['ninja', 'check-all'], msvc_arch='x64') - - WriteStampFile(PACKAGE_VERSION) - print 'Clang update was successful.' - return 0 - - -def main(): - parser = argparse.ArgumentParser(description='Build Clang.') - parser.add_argument('--bootstrap', action='store_true', - help='first build clang with CC, then with itself.') - parser.add_argument('--if-needed', action='store_true', - help="run only if the script thinks clang is needed") - parser.add_argument('--force-local-build', action='store_true', - help="don't try to download prebuild binaries") - parser.add_argument('--gcc-toolchain', help='set the version for which gcc ' - 'version be used for building; --gcc-toolchain=/opt/foo ' - 'picks /opt/foo/bin/gcc') - parser.add_argument('--lto-gold-plugin', action='store_true', - help='build LLVM Gold plugin with LTO') - parser.add_argument('--llvm-force-head-revision', action='store_true', - help=('use the revision in the repo when printing ' - 'the revision')) - parser.add_argument('--print-revision', action='store_true', - help='print current clang revision and exit.') - parser.add_argument('--print-clang-version', action='store_true', - help='print current clang version (e.g. x.y.z) and exit.') - parser.add_argument('--run-tests', action='store_true', - help='run tests after building; only for local builds') - parser.add_argument('--tools', nargs='*', - help='select which chrome tools to build', - default=['plugins', 'blink_gc_plugin']) - parser.add_argument('--without-android', action='store_false', - help='don\'t build Android ASan runtime (linux only)', - dest='with_android', - default=sys.platform.startswith('linux')) - args = parser.parse_args() - - if args.lto_gold_plugin and not args.bootstrap: - print '--lto-gold-plugin requires --bootstrap' - return 1 - if args.lto_gold_plugin and not sys.platform.startswith('linux'): - print '--lto-gold-plugin is only effective on Linux. Ignoring the option.' - args.lto_gold_plugin = False - - if args.if_needed: - is_clang_required = False - # clang is always used on Mac and Linux. - if sys.platform == 'darwin' or sys.platform.startswith('linux'): - is_clang_required = True - # clang requested via $GYP_DEFINES. - if re.search(r'\b(clang|asan|lsan|msan|tsan)=1', - os.environ.get('GYP_DEFINES', '')): - is_clang_required = True - # clang previously downloaded, keep it up to date. - # If you don't want this, delete third_party/llvm-build on your machine. - if os.path.isdir(LLVM_BUILD_DIR): - is_clang_required = True - if not is_clang_required: - return 0 - if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')): - print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).' - return 0 - - global CLANG_REVISION, PACKAGE_VERSION - if args.print_revision: - if use_head_revision or args.llvm_force_head_revision: - print GetSvnRevision(LLVM_DIR) - else: - print PACKAGE_VERSION - return 0 - - if args.print_clang_version: - sys.stdout.write(VERSION) - return 0 - - # Don't buffer stdout, so that print statements are immediately flushed. - # Do this only after --print-revision has been handled, else we'll get - # an error message when this script is run from gn for some reason. - sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) - - if use_head_revision: - # Use a real revision number rather than HEAD to make sure that the stamp - # file logic works. - CLANG_REVISION = GetSvnRevision(LLVM_REPO_URL) - PACKAGE_VERSION = CLANG_REVISION + '-0' - - args.force_local_build = True - if 'OS=android' not in os.environ.get('GYP_DEFINES', ''): - # Only build the Android ASan rt on ToT bots when targetting Android. - args.with_android = False - - return UpdateClang(args) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tools/clang/scripts/update.sh b/tools/clang/scripts/update.sh deleted file mode 100755 index d85deae23c3..00000000000 --- a/tools/clang/scripts/update.sh +++ /dev/null @@ -1,724 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# This script will check out llvm and clang into third_party/llvm and build it. - -# Do NOT CHANGE this if you don't know what you're doing -- see -# https://code.google.com/p/chromium/wiki/UpdatingClang -# Reverting problematic clang rolls is safe, though. -CLANG_REVISION=241602 - -# This is incremented when pushing a new build of Clang at the same revision. -CLANG_SUB_REVISION=3 - -PACKAGE_VERSION="${CLANG_REVISION}-${CLANG_SUB_REVISION}" - -THIS_DIR="$(dirname "${0}")" -LLVM_DIR="${THIS_DIR}/../../../third_party/llvm" -LLVM_BUILD_DIR="${LLVM_DIR}/../llvm-build/Release+Asserts" -COMPILER_RT_BUILD_DIR="${LLVM_DIR}/../llvm-build/compiler-rt" -LLVM_BOOTSTRAP_DIR="${LLVM_DIR}/../llvm-bootstrap" -LLVM_BOOTSTRAP_INSTALL_DIR="${LLVM_DIR}/../llvm-bootstrap-install" -CLANG_DIR="${LLVM_DIR}/tools/clang" -COMPILER_RT_DIR="${LLVM_DIR}/compiler-rt" -LIBCXX_DIR="${LLVM_DIR}/projects/libcxx" -LIBCXXABI_DIR="${LLVM_DIR}/projects/libcxxabi" -ANDROID_NDK_DIR="${THIS_DIR}/../../../third_party/android_tools/ndk" -STAMP_FILE="${LLVM_DIR}/../llvm-build/cr_build_revision" -CHROMIUM_TOOLS_DIR="${THIS_DIR}/.." -BINUTILS_DIR="${THIS_DIR}/../../../third_party/binutils" - -ABS_CHROMIUM_TOOLS_DIR="${PWD}/${CHROMIUM_TOOLS_DIR}" -ABS_LIBCXX_DIR="${PWD}/${LIBCXX_DIR}" -ABS_LIBCXXABI_DIR="${PWD}/${LIBCXXABI_DIR}" -ABS_LLVM_DIR="${PWD}/${LLVM_DIR}" -ABS_LLVM_BUILD_DIR="${PWD}/${LLVM_BUILD_DIR}" -ABS_COMPILER_RT_DIR="${PWD}/${COMPILER_RT_DIR}" -ABS_BINUTILS_DIR="${PWD}/${BINUTILS_DIR}" - -# ${A:-a} returns $A if it's set, a else. -LLVM_REPO_URL=${LLVM_URL:-https://llvm.org/svn/llvm-project} - -CDS_URL=https://commondatastorage.googleapis.com/chromium-browser-clang - -if [[ -z "$GYP_DEFINES" ]]; then - GYP_DEFINES= -fi -if [[ -z "$GYP_GENERATORS" ]]; then - GYP_GENERATORS= -fi -if [[ -z "$LLVM_DOWNLOAD_GOLD_PLUGIN" ]]; then - LLVM_DOWNLOAD_GOLD_PLUGIN= -fi - - -# Die if any command dies, error on undefined variable expansions. -set -eu - - -if [[ -n ${LLVM_FORCE_HEAD_REVISION:-''} ]]; then - # Use a real revision number rather than HEAD to make sure that the stamp file - # logic works. - CLANG_REVISION=$(svn info "$LLVM_REPO_URL" \ - | grep 'Revision:' | awk '{ printf $2; }') - PACKAGE_VERSION="${CLANG_REVISION}-0" -fi - -OS="$(uname -s)" - -# Parse command line options. -if_needed= -force_local_build= -run_tests= -bootstrap= -with_android=yes -chrome_tools="plugins;blink_gc_plugin" -gcc_toolchain= -with_patches=yes - -if [[ "${OS}" = "Darwin" ]]; then - with_android= -fi - -while [[ $# > 0 ]]; do - case $1 in - --bootstrap) - bootstrap=yes - ;; - --if-needed) - if_needed=yes - ;; - --force-local-build) - force_local_build=yes - ;; - --print-revision) - if [[ -n ${LLVM_FORCE_HEAD_REVISION:-''} ]]; then - svn info "$LLVM_DIR" | grep 'Revision:' | awk '{ printf $2; }' - else - echo $PACKAGE_VERSION - fi - exit 0 - ;; - --run-tests) - run_tests=yes - ;; - --without-android) - with_android= - ;; - --without-patches) - with_patches= - ;; - --with-chrome-tools) - shift - if [[ $# == 0 ]]; then - echo "--with-chrome-tools requires an argument." - exit 1 - fi - chrome_tools=$1 - ;; - --gcc-toolchain) - shift - if [[ $# == 0 ]]; then - echo "--gcc-toolchain requires an argument." - exit 1 - fi - if [[ -x "$1/bin/gcc" ]]; then - gcc_toolchain=$1 - else - echo "Invalid --gcc-toolchain: '$1'." - echo "'$1/bin/gcc' does not appear to be valid." - exit 1 - fi - ;; - - --help) - echo "usage: $0 [--force-local-build] [--if-needed] [--run-tests] " - echo "--bootstrap: First build clang with CC, then with itself." - echo "--force-local-build: Don't try to download prebuilt binaries." - echo "--if-needed: Download clang only if the script thinks it is needed." - echo "--run-tests: Run tests after building. Only for local builds." - echo "--print-revision: Print current clang revision and exit." - echo "--without-android: Don't build ASan Android runtime library." - echo "--with-chrome-tools: Select which chrome tools to build." \ - "Defaults to plugins;blink_gc_plugin." - echo " Example: --with-chrome-tools plugins;empty-string" - echo "--gcc-toolchain: Set the prefix for which GCC version should" - echo " be used for building. For example, to use gcc in" - echo " /opt/foo/bin/gcc, use '--gcc-toolchain '/opt/foo" - echo "--without-patches: Don't apply local patches." - echo - exit 1 - ;; - *) - echo "Unknown argument: '$1'." - echo "Use --help for help." - exit 1 - ;; - esac - shift -done - -if [[ -n ${LLVM_FORCE_HEAD_REVISION:-''} ]]; then - force_local_build=yes - - # Skip local patches when using HEAD: they probably don't apply anymore. - with_patches= - - if ! [[ "$GYP_DEFINES" =~ .*OS=android.* ]]; then - # Only build the Android ASan rt when targetting Android. - with_android= - fi - - LLVM_BUILD_TOOLS_DIR="${ABS_LLVM_DIR}/../llvm-build-tools" - - if [[ "${OS}" == "Linux" ]] && [[ -z "${gcc_toolchain}" ]]; then - if [[ $(gcc -dumpversion) < "4.7.0" ]]; then - # We need a newer GCC version. - if [[ ! -e "${LLVM_BUILD_TOOLS_DIR}/gcc482" ]]; then - echo "Downloading pre-built GCC 4.8.2..." - mkdir -p "${LLVM_BUILD_TOOLS_DIR}" - curl --fail -L "${CDS_URL}/tools/gcc482.tgz" | \ - tar zxf - -C "${LLVM_BUILD_TOOLS_DIR}" - echo Done - fi - gcc_toolchain="${LLVM_BUILD_TOOLS_DIR}/gcc482" - else - # Always set gcc_toolchain; llvm-symbolizer needs the bundled libstdc++. - gcc_toolchain="$(dirname $(dirname $(which gcc)))" - fi - fi - - if [[ "${OS}" == "Linux" || "${OS}" == "Darwin" ]]; then - if [[ $(cmake --version | grep -Eo '[0-9.]+') < "3.0" ]]; then - # We need a newer CMake version. - if [[ ! -e "${LLVM_BUILD_TOOLS_DIR}/cmake310" ]]; then - echo "Downloading pre-built CMake 3.10..." - mkdir -p "${LLVM_BUILD_TOOLS_DIR}" - curl --fail -L "${CDS_URL}/tools/cmake310_${OS}.tgz" | \ - tar zxf - -C "${LLVM_BUILD_TOOLS_DIR}" - echo Done - fi - export PATH="${LLVM_BUILD_TOOLS_DIR}/cmake310/bin:${PATH}" - fi - fi - - echo "LLVM_FORCE_HEAD_REVISION was set; using r${CLANG_REVISION}" -fi - -if [[ -n "$if_needed" ]]; then - if [[ "${OS}" == "Darwin" ]]; then - # clang is always used on Mac. - true - elif [[ "${OS}" == "Linux" ]]; then - # clang is also aways used on Linux. - true - elif [[ "$GYP_DEFINES" =~ .*(clang|tsan|asan|lsan|msan)=1.* ]]; then - # clang requested via $GYP_DEFINES. - true - elif [[ -d "${LLVM_BUILD_DIR}" ]]; then - # clang previously downloaded, keep it up-to-date. - # If you don't want this, delete third_party/llvm-build on your machine. - true - else - # clang wasn't needed, not doing anything. - exit 0 - fi -fi - - -# Check if there's anything to be done, exit early if not. -if [[ -f "${STAMP_FILE}" ]]; then - PREVIOUSLY_BUILT_REVISON=$(cat "${STAMP_FILE}") - if [[ -z "$force_local_build" ]] && \ - [[ "${PREVIOUSLY_BUILT_REVISON}" = \ - "${PACKAGE_VERSION}" ]]; then - echo "Clang already at ${PACKAGE_VERSION}" - exit 0 - fi -fi -# To always force a new build if someone interrupts their build half way. -rm -f "${STAMP_FILE}" - - -if [[ -z "$force_local_build" ]]; then - # Check if there's a prebuilt binary and if so just fetch that. That's faster, - # and goma relies on having matching binary hashes on client and server too. - CDS_FILE="clang-${PACKAGE_VERSION}.tgz" - CDS_OUT_DIR=$(mktemp -d -t clang_download.XXXXXX) - CDS_OUTPUT="${CDS_OUT_DIR}/${CDS_FILE}" - if [ "${OS}" = "Linux" ]; then - CDS_FULL_URL="${CDS_URL}/Linux_x64/${CDS_FILE}" - elif [ "${OS}" = "Darwin" ]; then - CDS_FULL_URL="${CDS_URL}/Mac/${CDS_FILE}" - fi - echo Trying to download prebuilt clang - if which curl > /dev/null; then - curl -L --fail "${CDS_FULL_URL}" -o "${CDS_OUTPUT}" || \ - rm -rf "${CDS_OUT_DIR}" - elif which wget > /dev/null; then - wget "${CDS_FULL_URL}" -O "${CDS_OUTPUT}" || rm -rf "${CDS_OUT_DIR}" - else - echo "Neither curl nor wget found. Please install one of these." - exit 1 - fi - if [ -f "${CDS_OUTPUT}" ]; then - rm -rf "${LLVM_BUILD_DIR}" - mkdir -p "${LLVM_BUILD_DIR}" - tar -xzf "${CDS_OUTPUT}" -C "${LLVM_BUILD_DIR}" - echo clang "${PACKAGE_VERSION}" unpacked - echo "${PACKAGE_VERSION}" > "${STAMP_FILE}" - rm -rf "${CDS_OUT_DIR}" - # Download the gold plugin if requested to by an environment variable. - # This is used by the CFI ClusterFuzz bot. - if [[ -n "${LLVM_DOWNLOAD_GOLD_PLUGIN}" ]]; then - ${THIS_DIR}/../../../build/download_gold_plugin.py - fi - exit 0 - else - echo Did not find prebuilt clang "${PACKAGE_VERSION}", building - fi -fi - -if [[ -n "${with_android}" ]] && ! [[ -d "${ANDROID_NDK_DIR}" ]]; then - echo "Android NDK not found at ${ANDROID_NDK_DIR}" - echo "The Android NDK is needed to build a Clang whose -fsanitize=address" - echo "works on Android. See " - echo "http://code.google.com/p/chromium/wiki/AndroidBuildInstructions for how" - echo "to install the NDK, or pass --without-android." - exit 1 -fi - -# Check that cmake and ninja are available. -if ! which cmake > /dev/null; then - echo "CMake needed to build clang; please install" - exit 1 -fi -if ! which ninja > /dev/null; then - echo "ninja needed to build clang, please install" - exit 1 -fi - -echo Reverting previously patched files -for i in \ - "${CLANG_DIR}/test/Index/crash-recovery-modules.m" \ - "${CLANG_DIR}/unittests/libclang/LibclangTest.cpp" \ - "${COMPILER_RT_DIR}/lib/asan/asan_rtl.cc" \ - "${COMPILER_RT_DIR}/test/asan/TestCases/Linux/new_array_cookie_test.cc" \ - "${LLVM_DIR}/test/DebugInfo/gmlt.ll" \ - "${LLVM_DIR}/lib/CodeGen/SpillPlacement.cpp" \ - "${LLVM_DIR}/lib/CodeGen/SpillPlacement.h" \ - "${LLVM_DIR}/lib/Transforms/Instrumentation/MemorySanitizer.cpp" \ - "${CLANG_DIR}/test/Driver/env.c" \ - "${CLANG_DIR}/lib/Frontend/InitPreprocessor.cpp" \ - "${CLANG_DIR}/test/Frontend/exceptions.c" \ - "${CLANG_DIR}/test/Preprocessor/predefined-exceptions.m" \ - "${LLVM_DIR}/test/Bindings/Go/go.test" \ - "${CLANG_DIR}/lib/Parse/ParseExpr.cpp" \ - "${CLANG_DIR}/lib/Parse/ParseTemplate.cpp" \ - "${CLANG_DIR}/lib/Sema/SemaDeclCXX.cpp" \ - "${CLANG_DIR}/lib/Sema/SemaExprCXX.cpp" \ - "${CLANG_DIR}/test/SemaCXX/default2.cpp" \ - "${CLANG_DIR}/test/SemaCXX/typo-correction-delayed.cpp" \ - "${COMPILER_RT_DIR}/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc" \ - "${COMPILER_RT_DIR}/test/tsan/signal_segv_handler.cc" \ - "${COMPILER_RT_DIR}/lib/sanitizer_common/sanitizer_coverage_libcdep.cc" \ - "${COMPILER_RT_DIR}/cmake/config-ix.cmake" \ - "${COMPILER_RT_DIR}/CMakeLists.txt" \ - "${COMPILER_RT_DIR}/lib/ubsan/ubsan_platform.h" \ - ; do - if [[ -e "${i}" ]]; then - rm -f "${i}" # For unversioned files. - svn revert "${i}" - fi; -done - -echo Remove the Clang tools shim dir -CHROME_TOOLS_SHIM_DIR=${ABS_LLVM_DIR}/tools/chrometools -rm -rfv ${CHROME_TOOLS_SHIM_DIR} - -echo Getting LLVM r"${CLANG_REVISION}" in "${LLVM_DIR}" -if ! svn co --force "${LLVM_REPO_URL}/llvm/trunk@${CLANG_REVISION}" \ - "${LLVM_DIR}"; then - echo Checkout failed, retrying - rm -rf "${LLVM_DIR}" - svn co --force "${LLVM_REPO_URL}/llvm/trunk@${CLANG_REVISION}" "${LLVM_DIR}" -fi - -echo Getting clang r"${CLANG_REVISION}" in "${CLANG_DIR}" -svn co --force "${LLVM_REPO_URL}/cfe/trunk@${CLANG_REVISION}" "${CLANG_DIR}" - -# We have moved from building compiler-rt in the LLVM tree, to a separate -# directory. Nuke any previous checkout to avoid building it. -rm -rf "${LLVM_DIR}/projects/compiler-rt" - -echo Getting compiler-rt r"${CLANG_REVISION}" in "${COMPILER_RT_DIR}" -svn co --force "${LLVM_REPO_URL}/compiler-rt/trunk@${CLANG_REVISION}" \ - "${COMPILER_RT_DIR}" - -# clang needs a libc++ checkout, else -stdlib=libc++ won't find includes -# (i.e. this is needed for bootstrap builds). -if [ "${OS}" = "Darwin" ]; then - echo Getting libc++ r"${CLANG_REVISION}" in "${LIBCXX_DIR}" - svn co --force "${LLVM_REPO_URL}/libcxx/trunk@${CLANG_REVISION}" \ - "${LIBCXX_DIR}" -fi - -# While we're bundling our own libc++ on OS X, we need to compile libc++abi -# into it too (since OS X 10.6 doesn't have libc++abi.dylib either). -if [ "${OS}" = "Darwin" ]; then - echo Getting libc++abi r"${CLANG_REVISION}" in "${LIBCXXABI_DIR}" - svn co --force "${LLVM_REPO_URL}/libcxxabi/trunk@${CLANG_REVISION}" \ - "${LIBCXXABI_DIR}" -fi - -if [[ -n "$with_patches" ]]; then - - # Apply patch for tests failing with --disable-pthreads (llvm.org/PR11974) - pushd "${CLANG_DIR}" - cat << 'EOF' | ---- test/Index/crash-recovery-modules.m (revision 202554) -+++ test/Index/crash-recovery-modules.m (working copy) -@@ -12,6 +12,8 @@ - - // REQUIRES: crash-recovery - // REQUIRES: shell -+// XFAIL: * -+// (PR11974) - - @import Crash; -EOF -patch -p0 -popd - -pushd "${CLANG_DIR}" -cat << 'EOF' | ---- unittests/libclang/LibclangTest.cpp (revision 215949) -+++ unittests/libclang/LibclangTest.cpp (working copy) -@@ -431,7 +431,7 @@ - EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU)); - } - --TEST_F(LibclangReparseTest, ReparseWithModule) { -+TEST_F(LibclangReparseTest, DISABLED_ReparseWithModule) { - const char *HeaderTop = "#ifndef H\n#define H\nstruct Foo { int bar;"; - const char *HeaderBottom = "\n};\n#endif\n"; - const char *MFile = "#include \"HeaderFile.h\"\nint main() {" -EOF - patch -p0 - popd - - # This Go bindings test doesn't work after the bootstrap build on Linux. (PR21552) - pushd "${LLVM_DIR}" - cat << 'EOF' | ---- test/Bindings/Go/go.test (revision 223109) -+++ test/Bindings/Go/go.test (working copy) -@@ -1,3 +1,3 @@ --; RUN: llvm-go test llvm.org/llvm/bindings/go/llvm -+; RUN: true - - ; REQUIRES: shell -EOF - patch -p0 - popd - - # The UBSan run-time, which is now bundled with the ASan run-time, doesn't work - # on Mac OS X 10.8 (PR23539). - pushd "${COMPILER_RT_DIR}" - cat << 'EOF' | -Index: CMakeLists.txt -=================================================================== ---- CMakeLists.txt (revision 241602) -+++ CMakeLists.txt (working copy) -@@ -305,6 +305,7 @@ - list(APPEND SANITIZER_COMMON_SUPPORTED_OS iossim) - endif() - endif() -+ set(SANITIZER_MIN_OSX_VERSION "10.7") - if(SANITIZER_MIN_OSX_VERSION VERSION_LESS "10.7") - message(FATAL_ERROR "Too old OS X version: ${SANITIZER_MIN_OSX_VERSION}") - endif() -EOF - patch -p0 - popd - -fi - -# Echo all commands. -set -x - -# Set default values for CC and CXX if they're not set in the environment. -CC=${CC:-cc} -CXX=${CXX:-c++} - -if [[ -n "${gcc_toolchain}" ]]; then - # Use the specified gcc installation for building. - CC="$gcc_toolchain/bin/gcc" - CXX="$gcc_toolchain/bin/g++" - # Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap compiler, - # etc.) find the .so. - export LD_LIBRARY_PATH="$(dirname $(${CXX} -print-file-name=libstdc++.so.6))" -fi - -CFLAGS="" -CXXFLAGS="" -LDFLAGS="" - -# LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is -# needed, on OS X it requires libc++. clang only automatically links to libc++ -# when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run on -# OS X versions as old as 10.7. -# TODO(thakis): Some bots are still on 10.6 (nacl...), so for now bundle -# libc++.dylib. Remove this once all bots are on 10.7+, then use -# -DLLVM_ENABLE_LIBCXX=ON and change deployment_target to 10.7. -deployment_target="" - -if [ "${OS}" = "Darwin" ]; then - # When building on 10.9, /usr/include usually doesn't exist, and while - # Xcode's clang automatically sets a sysroot, self-built clangs don't. - CFLAGS="-isysroot $(xcrun --show-sdk-path)" - CXXFLAGS="-stdlib=libc++ -nostdinc++ -I${ABS_LIBCXX_DIR}/include ${CFLAGS}" - - if [[ -n "${bootstrap}" ]]; then - deployment_target=10.6 - fi -fi - -# Build bootstrap clang if requested. -if [[ -n "${bootstrap}" ]]; then - ABS_INSTALL_DIR="${PWD}/${LLVM_BOOTSTRAP_INSTALL_DIR}" - echo "Building bootstrap compiler" - mkdir -p "${LLVM_BOOTSTRAP_DIR}" - pushd "${LLVM_BOOTSTRAP_DIR}" - - cmake -GNinja \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_TARGETS_TO_BUILD=host \ - -DLLVM_ENABLE_THREADS=OFF \ - -DCMAKE_INSTALL_PREFIX="${ABS_INSTALL_DIR}" \ - -DCMAKE_C_COMPILER="${CC}" \ - -DCMAKE_CXX_COMPILER="${CXX}" \ - -DCMAKE_C_FLAGS="${CFLAGS}" \ - -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ - ../llvm - - ninja - if [[ -n "${run_tests}" ]]; then - ninja check-all - fi - - ninja install - if [[ -n "${gcc_toolchain}" ]]; then - # Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap - # compiler can start. - cp -v "$(${CXX} -print-file-name=libstdc++.so.6)" \ - "${ABS_INSTALL_DIR}/lib/" - fi - - popd - CC="${ABS_INSTALL_DIR}/bin/clang" - CXX="${ABS_INSTALL_DIR}/bin/clang++" - - if [[ -n "${gcc_toolchain}" ]]; then - # Tell the bootstrap compiler to use a specific gcc prefix to search - # for standard library headers and shared object file. - CFLAGS="--gcc-toolchain=${gcc_toolchain}" - CXXFLAGS="--gcc-toolchain=${gcc_toolchain}" - fi - - echo "Building final compiler" -fi - -# Build clang (in a separate directory). -# The clang bots have this path hardcoded in built/scripts/slave/compile.py, -# so if you change it you also need to change these links. -mkdir -p "${LLVM_BUILD_DIR}" -pushd "${LLVM_BUILD_DIR}" - -# Build libc++.dylib while some bots are still on OS X 10.6. -if [ "${OS}" = "Darwin" ]; then - rm -rf libcxxbuild - LIBCXXFLAGS="-O3 -std=c++11 -fstrict-aliasing" - - # libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files - # into different subdirectories. - mkdir -p libcxxbuild/libcxx - pushd libcxxbuild/libcxx - ${CXX:-c++} -c ${CXXFLAGS} ${LIBCXXFLAGS} "${ABS_LIBCXX_DIR}"/src/*.cpp - popd - - mkdir -p libcxxbuild/libcxxabi - pushd libcxxbuild/libcxxabi - ${CXX:-c++} -c ${CXXFLAGS} ${LIBCXXFLAGS} "${ABS_LIBCXXABI_DIR}"/src/*.cpp -I"${ABS_LIBCXXABI_DIR}/include" - popd - - pushd libcxxbuild - ${CC:-cc} libcxx/*.o libcxxabi/*.o -o libc++.1.dylib -dynamiclib \ - -nodefaultlibs -current_version 1 -compatibility_version 1 \ - -lSystem -install_name @executable_path/libc++.dylib \ - -Wl,-unexported_symbols_list,${ABS_LIBCXX_DIR}/lib/libc++unexp.exp \ - -Wl,-force_symbols_not_weak_list,${ABS_LIBCXX_DIR}/lib/notweak.exp \ - -Wl,-force_symbols_weak_list,${ABS_LIBCXX_DIR}/lib/weak.exp - ln -sf libc++.1.dylib libc++.dylib - popd - LDFLAGS+="-stdlib=libc++ -L${PWD}/libcxxbuild" - - if [[ -n "${bootstrap}" ]]; then - # Now that the libc++ headers have been installed and libc++.dylib is built, - # delete the libc++ checkout again so that it's not part of the main - # build below -- the libc++(abi) tests don't pass on OS X in bootstrap - # builds (http://llvm.org/PR24068) - rm -rf "${ABS_LIBCXX_DIR}" - rm -rf "${ABS_LIBCXXABI_DIR}" - CXXFLAGS="-stdlib=libc++ -nostdinc++ -I${ABS_INSTALL_DIR}/include/c++/v1 ${CFLAGS}" - fi -fi - -# Find the binutils include dir for the gold plugin. -BINUTILS_INCDIR="" -if [ "${OS}" = "Linux" ]; then - BINUTILS_INCDIR="${ABS_BINUTILS_DIR}/Linux_x64/Release/include" -fi - - -# If building at head, define a macro that plugins can use for #ifdefing -# out code that builds at head, but not at CLANG_REVISION or vice versa. -if [[ -n ${LLVM_FORCE_HEAD_REVISION:-''} ]]; then - CFLAGS="${CFLAGS} -DLLVM_FORCE_HEAD_REVISION" - CXXFLAGS="${CXXFLAGS} -DLLVM_FORCE_HEAD_REVISION" -fi - -# Hook the Chromium tools into the LLVM build. Several Chromium tools have -# dependencies on LLVM/Clang libraries. The LLVM build detects implicit tools -# in the tools subdirectory, so install a shim CMakeLists.txt that forwards to -# the real directory for the Chromium tools. -# Note that the shim directory name intentionally has no _ or _. The implicit -# tool detection logic munges them in a weird way. -mkdir -v ${CHROME_TOOLS_SHIM_DIR} -cat > ${CHROME_TOOLS_SHIM_DIR}/CMakeLists.txt << EOF -# Since tools/clang isn't actually a subdirectory, use the two argument version -# to specify where build artifacts go. CMake doesn't allow reusing the same -# binary dir for multiple source dirs, so the build artifacts have to go into a -# subdirectory... -add_subdirectory(\${CHROMIUM_TOOLS_SRC} \${CMAKE_CURRENT_BINARY_DIR}/a) -EOF -rm -fv CMakeCache.txt -MACOSX_DEPLOYMENT_TARGET=${deployment_target} cmake -GNinja \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_ENABLE_THREADS=OFF \ - -DLLVM_BINUTILS_INCDIR="${BINUTILS_INCDIR}" \ - -DCMAKE_C_COMPILER="${CC}" \ - -DCMAKE_CXX_COMPILER="${CXX}" \ - -DCMAKE_C_FLAGS="${CFLAGS}" \ - -DCMAKE_CXX_FLAGS="${CXXFLAGS}" \ - -DCMAKE_EXE_LINKER_FLAGS="${LDFLAGS}" \ - -DCMAKE_SHARED_LINKER_FLAGS="${LDFLAGS}" \ - -DCMAKE_MODULE_LINKER_FLAGS="${LDFLAGS}" \ - -DCMAKE_INSTALL_PREFIX="${ABS_LLVM_BUILD_DIR}" \ - -DCHROMIUM_TOOLS_SRC="${ABS_CHROMIUM_TOOLS_DIR}" \ - -DCHROMIUM_TOOLS="${chrome_tools}" \ - "${ABS_LLVM_DIR}" -env - -if [[ -n "${gcc_toolchain}" ]]; then - # Copy in the right stdlibc++.so.6 so clang can start. - mkdir -p lib - cp -v "$(${CXX} ${CXXFLAGS} -print-file-name=libstdc++.so.6)" lib/ -fi - -ninja -# If any Chromium tools were built, install those now. -if [[ -n "${chrome_tools}" ]]; then - ninja cr-install -fi - -STRIP_FLAGS= -if [ "${OS}" = "Darwin" ]; then - # See http://crbug.com/256342 - STRIP_FLAGS=-x - - cp libcxxbuild/libc++.1.dylib bin/ -fi -strip ${STRIP_FLAGS} bin/clang -popd - -# Build compiler-rt out-of-tree. -mkdir -p "${COMPILER_RT_BUILD_DIR}" -pushd "${COMPILER_RT_BUILD_DIR}" - -rm -fv CMakeCache.txt -MACOSX_DEPLOYMENT_TARGET=${deployment_target} cmake -GNinja \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_ENABLE_THREADS=OFF \ - -DCMAKE_C_COMPILER="${CC}" \ - -DCMAKE_CXX_COMPILER="${CXX}" \ - -DLLVM_CONFIG_PATH="${ABS_LLVM_BUILD_DIR}/bin/llvm-config" \ - "${ABS_COMPILER_RT_DIR}" - -ninja - -# Copy selected output to the main tree. -# Darwin doesn't support cp --parents, so pipe through tar instead. -CLANG_VERSION=$("${ABS_LLVM_BUILD_DIR}/bin/clang" --version | \ - sed -ne 's/clang version \([0-9]\.[0-9]\.[0-9]\).*/\1/p') -ABS_LLVM_CLANG_LIB_DIR="${ABS_LLVM_BUILD_DIR}/lib/clang/${CLANG_VERSION}" -tar -c *blacklist.txt | tar -C ${ABS_LLVM_CLANG_LIB_DIR} -xv -tar -c include/sanitizer | tar -C ${ABS_LLVM_CLANG_LIB_DIR} -xv -if [[ "${OS}" = "Darwin" ]]; then - tar -c lib/darwin | tar -C ${ABS_LLVM_CLANG_LIB_DIR} -xv -else - tar -c lib/linux | tar -C ${ABS_LLVM_CLANG_LIB_DIR} -xv -fi - -popd - -if [[ -n "${with_android}" ]]; then - # Make a standalone Android toolchain. - ${ANDROID_NDK_DIR}/build/tools/make-standalone-toolchain.sh \ - --platform=android-19 \ - --install-dir="${LLVM_BUILD_DIR}/android-toolchain" \ - --system=linux-x86_64 \ - --stl=stlport \ - --toolchain=arm-linux-androideabi-4.9 - - # Android NDK r9d copies a broken unwind.h into the toolchain, see - # http://crbug.com/357890 - rm -v "${LLVM_BUILD_DIR}"/android-toolchain/include/c++/*/unwind.h - - # Build ASan runtime for Android in a separate build tree. - mkdir -p ${LLVM_BUILD_DIR}/android - pushd ${LLVM_BUILD_DIR}/android - rm -fv CMakeCache.txt - MACOSX_DEPLOYMENT_TARGET=${deployment_target} cmake -GNinja \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_ENABLE_THREADS=OFF \ - -DCMAKE_C_COMPILER=${PWD}/../bin/clang \ - -DCMAKE_CXX_COMPILER=${PWD}/../bin/clang++ \ - -DLLVM_CONFIG_PATH=${PWD}/../bin/llvm-config \ - -DCMAKE_C_FLAGS="--target=arm-linux-androideabi --sysroot=${PWD}/../android-toolchain/sysroot -B${PWD}/../android-toolchain" \ - -DCMAKE_CXX_FLAGS="--target=arm-linux-androideabi --sysroot=${PWD}/../android-toolchain/sysroot -B${PWD}/../android-toolchain" \ - -DANDROID=1 \ - "${ABS_COMPILER_RT_DIR}" - ninja libclang_rt.asan-arm-android.so - - # And copy it into the main build tree. - cp "$(find -name libclang_rt.asan-arm-android.so)" "${ABS_LLVM_CLANG_LIB_DIR}/lib/linux/" - popd -fi - -if [[ -n "$run_tests" || -n "${LLVM_FORCE_HEAD_REVISION:-''}" ]]; then - # Run Chrome tool tests. - ninja -C "${LLVM_BUILD_DIR}" cr-check-all -fi -if [[ -n "$run_tests" ]]; then - # Run the LLVM and Clang tests. - ninja -C "${LLVM_BUILD_DIR}" check-all -fi - -# After everything is done, log success for this revision. -echo "${PACKAGE_VERSION}" > "${STAMP_FILE}" diff --git a/tools/gn.py b/tools/gn.py index 3f4345ef172..32884b072f9 100755 --- a/tools/gn.py +++ b/tools/gn.py @@ -6,6 +6,7 @@ import argparse import multiprocessing import os +import shutil import subprocess import sys import time @@ -15,6 +16,7 @@ HOST_OS = utils.GuessOS() HOST_ARCH = utils.GuessArchitecture() SCRIPT_DIR = os.path.dirname(sys.argv[0]) DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..')) +GN = os.path.join(DART_ROOT, 'buildtools', 'gn') # Environment variables for default settings. DART_USE_ASAN = "DART_USE_ASAN" # Use instead of --asan @@ -24,39 +26,39 @@ DART_USE_WHEEZY = "DART_USE_WHEEZY" # Use instread of --wheezy DART_USE_TOOLCHAIN = "DART_USE_TOOLCHAIN" # Use instread of --toolchain-prefix DART_USE_SYSROOT = "DART_USE_SYSROOT" # Use instead of --target-sysroot -def use_asan(): +def UseASAN(): return DART_USE_ASAN in os.environ -def use_msan(): +def UseMSAN(): return DART_USE_MSAN in os.environ -def use_tsan(): +def UseTSAN(): return DART_USE_TSAN in os.environ -def use_wheezy(): +def UseWheezy(): return DART_USE_WHEEZY in os.environ -def toolchain_prefix(args): +def ToolchainPrefix(args): if args.toolchain_prefix: return args.toolchain_prefix return os.environ.get(DART_USE_TOOLCHAIN) -def target_sysroot(args): +def TargetSysroot(args): if args.target_sysroot: return args.target_sysroot return os.environ.get(DART_USE_SYSROOT) -def get_out_dir(mode, arch, target_os): +def GetOutDir(mode, arch, target_os): return utils.GetBuildRoot(HOST_OS, mode, arch, target_os) -def to_command_line(gn_args): +def ToCommandLine(gn_args): def merge(key, value): if type(value) is bool: return '%s=%s' % (key, 'true' if value else 'false') @@ -64,7 +66,7 @@ def to_command_line(gn_args): return [merge(x, y) for x, y in gn_args.iteritems()] -def host_cpu_for_arch(arch): +def HostCpuForArch(arch): if arch in ['ia32', 'arm', 'armv6', 'armv5te', 'mips', 'simarm', 'simarmv6', 'simarmv5te', 'simmips', 'simdbc', 'armsimdbc']: @@ -73,7 +75,7 @@ def host_cpu_for_arch(arch): return 'x64' -def target_cpu_for_arch(arch, target_os): +def TargetCpuForArch(arch, target_os): if arch in ['ia32', 'simarm', 'simarmv6', 'simarmv5te', 'simmips']: return 'x86' if arch in ['simarm64']: @@ -91,7 +93,7 @@ def target_cpu_for_arch(arch, target_os): return arch -def host_os_for_gn(host_os): +def HostOsForGn(host_os): if host_os.startswith('macos'): return 'mac' if host_os.startswith('win'): @@ -101,7 +103,7 @@ def host_os_for_gn(host_os): # Where string_map is formatted as X1=Y1,X2=Y2 etc. # If key is X1, returns Y1. -def parse_string_map(key, string_map): +def ParseStringMap(key, string_map): for m in string_map.split(','): l = m.split('=') if l[0] == key: @@ -109,18 +111,18 @@ def parse_string_map(key, string_map): return None -def to_gn_args(args, mode, arch, target_os): +def ToGnArgs(args, mode, arch, target_os): gn_args = {} - host_os = host_os_for_gn(HOST_OS) + host_os = HostOsForGn(HOST_OS) if target_os == 'host': gn_args['target_os'] = host_os else: gn_args['target_os'] = target_os gn_args['dart_target_arch'] = arch - gn_args['target_cpu'] = target_cpu_for_arch(arch, target_os) - gn_args['host_cpu'] = host_cpu_for_arch(arch) + gn_args['target_cpu'] = TargetCpuForArch(arch, target_os) + gn_args['host_cpu'] = HostCpuForArch(arch) # See: runtime/observatory/BUILD.gn. # This allows the standalone build of the observatory to fall back on @@ -165,10 +167,8 @@ def to_gn_args(args, mode, arch, target_os): # TODO(zra): Investigate using clang with these configurations. # Clang compiles tcmalloc's inline assembly for ia32 on Linux wrong, so we # don't use clang in that configuration. Thus, we use gcc for ia32 *unless* - # asan or tsan is specified. + # a clang-based sanitizer is specified. has_clang = (host_os != 'win' - and args.os not in ['android'] - and not gn_args['target_cpu'].startswith('arm') and not gn_args['target_cpu'].startswith('mips') and not ((gn_args['target_os'] == 'linux') and (gn_args['host_cpu'] == 'x86') @@ -185,13 +185,13 @@ def to_gn_args(args, mode, arch, target_os): if gn_args['target_os'] == 'linux' and args.wheezy: gn_args['dart_use_wheezy_sysroot'] = True else: - sysroot = target_sysroot(args) + sysroot = TargetSysroot(args) if sysroot: - gn_args['target_sysroot'] = parse_string_map(arch, sysroot) + gn_args['target_sysroot'] = ParseStringMap(arch, sysroot) - toolchain = toolchain_prefix(args) + toolchain = ToolchainPrefix(args) if toolchain: - gn_args['toolchain_prefix'] = parse_string_map(arch, toolchain) + gn_args['toolchain_prefix'] = ParseStringMap(arch, toolchain) goma_dir = os.environ.get('GOMA_DIR') goma_home_dir = os.path.join(os.getenv('HOME', ''), 'goma') @@ -212,13 +212,13 @@ def to_gn_args(args, mode, arch, target_os): return gn_args -def process_os_option(os_name): +def ProcessOsOption(os_name): if os_name == 'host': return HOST_OS return os_name -def process_options(args): +def ProcessOptions(args): if args.arch == 'all': args.arch = 'ia32,x64,simarm,simarm64,simmips,simdbc64' if args.mode == 'all': @@ -239,7 +239,7 @@ def process_options(args): if not arch in archs: print "Unknown arch %s" % arch return False - oses = [process_os_option(os_name) for os_name in args.os] + oses = [ProcessOsOption(os_name) for os_name in args.os] for os_name in oses: if not os_name in ['android', 'freebsd', 'linux', 'macos', 'win32']: print "Unknown os %s" % os_name @@ -248,7 +248,7 @@ def process_options(args): if os_name != 'android': print "Unsupported target os %s" % os_name return False - if not HOST_OS in ['linux']: + if not HOST_OS in ['linux', 'macos']: print ("Cross-compilation to %s is not supported on host os %s." % (os_name, HOST_OS)) return False @@ -303,7 +303,7 @@ def parse_args(args): other_group.add_argument('--asan', help='Build with ASAN', - default=use_asan(), + default=UseASAN(), action='store_true') other_group.add_argument('--no-asan', help='Disable ASAN', @@ -335,7 +335,7 @@ def parse_args(args): action='store_true') other_group.add_argument('--msan', help='Build with MSAN', - default=use_msan(), + default=UseMSAN(), action='store_true') other_group.add_argument('--no-msan', help='Disable MSAN', @@ -349,7 +349,7 @@ def parse_args(args): help='Comma-separated list of arch=/path/to/toolchain-prefix mappings') other_group.add_argument('--tsan', help='Build with TSAN', - default=use_tsan(), + default=UseTSAN(), action='store_true') other_group.add_argument('--no-tsan', help='Disable TSAN', @@ -357,7 +357,7 @@ def parse_args(args): action='store_false') other_group.add_argument('--wheezy', help='Use the Debian wheezy sysroot on Linux', - default=use_wheezy(), + default=UseWheezy(), action='store_true') other_group.add_argument('--no-wheezy', help='Disable the Debian wheezy sysroot on Linux', @@ -367,17 +367,18 @@ def parse_args(args): type=int, help='Number of simultaneous GN invocations', dest='workers', - # Set to multiprocessing.cpu_count() when GN can be run in parallel. - default=1) + default=multiprocessing.cpu_count()) options = parser.parse_args(args) - if not process_options(options): + if not ProcessOptions(options): parser.print_help() return None return options -def run_command(command): +# Run the command, if it succeeds returns 0, if it fails, returns the commands +# output as a string. +def RunCommand(command): try: subprocess.check_output( command, cwd=DART_ROOT, stderr=subprocess.STDOUT) @@ -387,7 +388,7 @@ def run_command(command): "output: " + e.output) -def main(argv): +def Main(argv): starttime = time.time() args = parse_args(argv) @@ -396,32 +397,28 @@ def main(argv): elif sys.platform == 'darwin': subdir = 'mac' elif sys.platform.startswith('linux'): - subdir = 'linux64' + subdir = 'linux64' else: print 'Unknown platform: ' + sys.platform return 1 + gn = os.path.join(DART_ROOT, 'buildtools', subdir, 'gn') commands = [] for target_os in args.os: for mode in args.mode: for arch in args.arch: - command = [ - '%s/buildtools/%s/gn' % (DART_ROOT, subdir), - 'gen', - '--check' - ] - gn_args = to_command_line(to_gn_args(args, mode, arch, target_os)) - out_dir = get_out_dir(mode, arch, target_os) + out_dir = GetOutDir(mode, arch, target_os) + command = [gn, 'gen', out_dir, '--check'] + gn_args = ToCommandLine(ToGnArgs(args, mode, arch, target_os)) if args.verbose: print "gn gen --check in %s" % out_dir if args.ide: command.append(ide_switch(HOST_OS)) - command.append(out_dir) command.append('--args=%s' % ' '.join(gn_args)) commands.append(command) pool = multiprocessing.Pool(args.workers) - results = pool.map(run_command, commands, chunksize=1) + results = pool.map(RunCommand, commands, chunksize=1) for r in results: if r != 0: print r.strip() @@ -434,4 +431,4 @@ def main(argv): if __name__ == '__main__': - sys.exit(main(sys.argv)) + sys.exit(Main(sys.argv)) diff --git a/tools/ninja.py b/tools/ninja.py index 55d8d2ad8e4..7a2db651a7f 100755 --- a/tools/ninja.py +++ b/tools/ninja.py @@ -90,7 +90,7 @@ def ProcessOptions(options, args): if os_name != 'android': print "Unsupported target os %s" % os_name return False - if not HOST_OS in ['linux']: + if not HOST_OS in ['linux', 'macos']: print ("Cross-compilation to %s is not supported on host os %s." % (os_name, HOST_OS)) return False