Revert "[infra] Roll clang to match the version used by Flutter"

This reverts commit cc968ca45b.

This CL will have to be relanded in two pieces so that the
update to tools/gn.py lands before the unmodified version
can invoke the updated GN binary.

Review-Url: https://codereview.chromium.org/2855973006 .
This commit is contained in:
Zachary Anderson 2017-05-03 15:16:20 -07:00
parent cc968ca45b
commit 958c25c1db
20 changed files with 1801 additions and 301 deletions

137
DEPS
View file

@ -23,7 +23,6 @@ vars = {
# Chromium git
"chromium_git": "https://chromium.googlesource.com",
"fuchsia_git": "https://fuchsia.googlesource.com",
# Only use this temporarily while waiting for a mirror for a new package.
"github_dartlang": "https://github.com/dart-lang/%s.git",
@ -31,12 +30,9 @@ vars = {
"gyp_rev": "@6ee91ad8659871916f9aa840d42e1513befdf638",
"co19_rev": "@4af9ef149be554216c5bb16cbac8e50d4c28cdf1",
# Revisions of GN related dependencies. This should match the revision
# pulled by Flutter.
"buildtools_revision": "@c8db819853bcf8ce1635a8b7a395820f39b5a9fc",
# Scripts that make 'git cl format' work.
"clang_format_scripts_rev": "@c09c8deeac31f05bd801995c475e7c8070f9ecda",
# Revisions of GN related dependencies.
"buildtools_revision": "@39b1db2ab4aa4b2ccaa263c29bdf63e7c1ee28aa",
"clang_format_rev": "@0ed791d1387a3c9146ea6c453c646f3c0fc97784",
"gperftools_revision": "@02eeed29df112728564a5dde6417fa4622b57a06",
@ -136,10 +132,11 @@ deps = {
# Stuff needed for GN build.
Var("dart_root") + "/buildtools":
Var("fuchsia_git") + "/buildtools" + Var("buildtools_revision"),
Var('chromium_git') + '/chromium/buildtools.git' +
Var('buildtools_revision'),
Var("dart_root") + "/buildtools/clang_format/script":
Var("chromium_git") + "/chromium/llvm-project/cfe/tools/clang-format.git" +
Var("clang_format_scripts_rev"),
Var("clang_format_rev"),
Var("dart_root") + "/tests/co19/src":
(Var("github_mirror") % "co19") + Var("co19_rev"),
@ -349,6 +346,11 @@ deps = {
}
deps_os = {
"android": {
Var("dart_root") + "/third_party/android_tools":
Var("chromium_git") + "/android_tools.git" +
"@aaeda3d69df4b4352e3cac7c16bea7f16bd1ec12",
},
"win": {
Var("dart_root") + "/third_party/cygwin":
Var("chromium_git") + "/chromium/deps/cygwin.git" +
@ -359,6 +361,98 @@ deps_os = {
# TODO(iposva): Move the necessary tools so that hooks can be run
# without the runtime being available.
hooks = [
# Pull GN binaries. This needs to be before running GYP below.
{
'name': 'gn_linux64',
'pattern': '.',
'action': [
'download_from_google_storage',
'--no_auth',
'--no_resume',
'--quiet',
'--platform=linux*',
'--bucket',
'chromium-gn',
'-s',
Var('dart_root') + '/buildtools/linux64/gn.sha1',
],
},
{
'name': 'gn_mac',
'pattern': '.',
'action': [
'download_from_google_storage',
'--no_auth',
'--no_resume',
'--quiet',
'--platform=darwin',
'--bucket',
'chromium-gn',
'-s',
Var('dart_root') + '/buildtools/mac/gn.sha1',
],
},
{
'name': 'gn_win',
'pattern': '.',
'action': [
'download_from_google_storage',
'--no_auth',
'--no_resume',
'--quiet',
'--platform=win*',
'--bucket',
'chromium-gn',
'-s',
Var('dart_root') + '/buildtools/win/gn.exe.sha1',
],
},
# Pull clang-format binaries using checked-in hashes.
{
'name': 'clang_format_win',
'pattern': '.',
'action': [
'download_from_google_storage',
'--no_auth',
'--no_resume',
'--quiet',
'--platform=win32',
'--bucket',
'chromium-clang-format',
'-s',
Var('dart_root') + '/buildtools/win/clang-format.exe.sha1',
],
},
{
'name': 'clang_format_linux',
'pattern': '.',
'action': [
'download_from_google_storage',
'--no_auth',
'--no_resume',
'--quiet',
'--platform=linux*',
'--bucket',
'chromium-clang-format',
'-s',
Var('dart_root') + '/buildtools/linux64/clang-format.sha1',
],
},
{
'name': 'clang_format_mac',
'pattern': '.',
'action': [
'download_from_google_storage',
'--no_auth',
'--no_resume',
'--quiet',
'--platform=darwin',
'--bucket',
'chromium-clang-format',
'-s',
Var('dart_root') + '/buildtools/mac/clang-format.sha1',
],
},
{
'name': 'd8_testing_binaries',
'pattern': '.',
@ -464,6 +558,21 @@ hooks = [
Var('dart_root') + "/third_party/gsutil.tar.gz.sha1",
],
},
{
"name": "clang",
"pattern": ".",
"action": [
"download_from_google_storage",
"--no_auth",
"--no_resume",
"--bucket",
"dart-dependencies",
"--platform=linux*",
"--extract",
"-s",
Var('dart_root') + "/third_party/clang.tar.gz.sha1",
],
},
{
# Pull Debian wheezy sysroot for i386 Linux
'name': 'sysroot_i386',
@ -479,14 +588,10 @@ hooks = [
'--running-as-hook', '--arch', 'amd64'],
},
{
'name': 'download_android_tools',
# Pull clang if needed or requested via GYP_DEFINES.
'name': 'gn_clang',
'pattern': '.',
'action': ['python', 'sdk/tools/android/download_android_tools.py'],
},
{
'name': 'buildtools',
'pattern': '.',
'action': ['python', 'sdk/tools/buildtools/update.py'],
'action': ['python', 'sdk/tools/clang/scripts/update.py', '--if-needed'],
},
{
# Update the Windows toolchain if necessary.

View file

@ -62,11 +62,7 @@ if (is_android) {
# Subdirectories inside android_ndk_root that contain the sysroot for the
# associated platform.
if (current_cpu == "x64" || current_cpu == "arm64") {
_android_api_level = 22
} else {
_android_api_level = 16
}
_android_api_level = 16
x86_android_sysroot_subdir =
"platforms/android-${_android_api_level}/arch-x86"
arm_android_sysroot_subdir =
@ -174,5 +170,5 @@ if (is_android) {
assert(false, "Unknown Android ABI: " + current_cpu)
}
android_log_tag = "\"dart\""
android_log_tag = "\"flutter\""
}

View file

@ -130,6 +130,17 @@ config("compiler") {
cflags_objcc += extra_flags
}
if (is_clang) {
# This is here so that all files get recompiled after a clang roll and
# when turning clang on or off. (defines are passed via the command line,
# and build system rebuild things when their commandline changes). Nothing
# should ever read this define.
defines +=
[ "CR_CLANG_REVISION=" + exec_script("//tools/clang/scripts/update.py",
[ "--print-revision" ],
"trim string") ]
}
# Mac-specific compiler flags setup.
# ----------------------------------
if (is_mac) {
@ -354,8 +365,7 @@ config("compiler") {
defines += [ "HAVE_SYS_UIO_H" ]
# Use gold for Android for most CPU architectures.
if (current_cpu == "x86" || current_cpu == "x64" || current_cpu == "arm" ||
current_cpu == "arm64") {
if (current_cpu == "x86" || current_cpu == "x64" || current_cpu == "arm") {
ldflags += [ "-fuse-ld=gold" ]
if (is_clang) {
# Let clang find the ld.gold in the NDK.
@ -365,8 +375,10 @@ config("compiler") {
}
ldflags += [
# Don't re-export visible symbols from libraries.
"-Wl,--exclude-libs,ALL",
# Don't allow visible symbols from libgcc or libc++ to be
# re-exported.
"-Wl,--exclude-libs=libgcc.a",
"-Wl,--exclude-libs=libc++_static.a",
]
if (current_cpu == "arm") {
ldflags += [
@ -379,15 +391,9 @@ config("compiler") {
if (current_cpu == "arm") {
cflags += [ "--target=arm-linux-androideabi" ]
ldflags += [ "--target=arm-linux-androideabi" ]
} else if (current_cpu == "arm64") {
cflags += [ "--target=aarch64-linux-android" ]
ldflags += [ "--target=aarch64-linux-android" ]
} else if (current_cpu == "x86") {
cflags += [ "--target=i686-linux-androideabi" ]
ldflags += [ "--target=i686-linux-androideabi" ]
} else if (current_cpu == "x64") {
cflags += [ "--target=x86_64-linux-androideabi" ]
ldflags += [ "--target=x86_64-linux-androideabi" ]
cflags += [ "--target=x86-linux-androideabi" ]
ldflags += [ "--target=x86-linux-androideabi" ]
}
}
}
@ -477,19 +483,25 @@ config("runtime_library") {
]
lib_dirs += [ "$android_libcpp_root/libs/$android_app_abi" ]
libs += [ "$android_libcpp_library" ]
libs += [
"$android_libcpp_library",
"c++abi",
"android_support",
]
if (current_cpu == "arm") {
libs += [ "unwind" ]
if (current_cpu == "mipsel") {
libs += [
# ld linker is used for mips Android, and ld does not accept library
# absolute path prefixed by "-l"; Since libgcc does not exist in mips
# sysroot the proper library will be linked.
# TODO(gordanac): Remove once gold linker is used for mips Android.
"gcc",
]
} else {
libs += [
# Manually link the libgcc.a that the cross compiler uses. This is
# absolute because the linker will look inside the sysroot if it's not.
rebase_path(android_libgcc_file),
]
}
libs += [
"gcc",
"c",
"dl",
"m",

View file

@ -47,16 +47,7 @@ template("android_gcc_toolchain") {
is_clang = invoker.is_clang
if (is_clang) {
host_suffix = ""
if (host_os == "linux") {
host_suffix = "x86_64-linux"
} else if (host_os == "mac") {
host_suffix = "x86_64-darwin"
} else {
assert(false, "Unknown host")
}
prefix = rebase_path("//buildtools/toolchain/clang+llvm-$host_suffix/bin",
prefix = rebase_path("//third_party/llvm-build/Release+Asserts/bin",
root_build_dir)
cc = compiler_prefix + prefix + "/clang"

View file

@ -60,7 +60,7 @@ gcc_toolchain("arm64") {
}
gcc_toolchain("clang_x86") {
prefix = rebase_path("//buildtools/toolchain/clang+llvm-x86_64-linux/bin",
prefix = rebase_path("//third_party/llvm-build/Release+Asserts/bin",
root_build_dir)
cc = "${compiler_prefix}$prefix/clang"
cxx = "${compiler_prefix}$prefix/clang++"
@ -89,7 +89,7 @@ gcc_toolchain("x86") {
}
gcc_toolchain("clang_x64") {
prefix = rebase_path("//buildtools/toolchain/clang+llvm-x86_64-linux/bin",
prefix = rebase_path("//third_party/llvm-build/Release+Asserts/bin",
root_build_dir)
cc = "${compiler_prefix}$prefix/clang"
cxx = "${compiler_prefix}$prefix/clang++"

View file

@ -211,7 +211,7 @@ template("mac_toolchain") {
mac_toolchain("clang_x64") {
toolchain_cpu = "x64"
toolchain_os = "mac"
prefix = rebase_path("//buildtools/toolchain/clang+llvm-x86_64-darwin/bin",
prefix = rebase_path("//third_party/llvm-build/Release+Asserts/bin",
root_build_dir)
cc = "${goma_prefix}$prefix/clang"
cxx = "${goma_prefix}$prefix/clang++"
@ -224,7 +224,7 @@ mac_toolchain("clang_x64") {
mac_toolchain("clang_i386") {
toolchain_cpu = "i386"
toolchain_os = "mac"
prefix = rebase_path("//buildtools/toolchain/clang+llvm-x86_64-darwin/bin",
prefix = rebase_path("//third_party/llvm-build/Release+Asserts/bin",
root_build_dir)
cc = "${goma_prefix}$prefix/clang"
cxx = "${goma_prefix}$prefix/clang++"

View file

@ -1 +0,0 @@
50bb526ee77fee88b9382d2bf48e7399751b98ae

View file

@ -1 +0,0 @@
0d320c50b0ed188c7e1182388e2beb623a1d307d

View file

@ -1 +0,0 @@
ed883dfaad6f27350eb3de426ccbf9abdc2943e8

View file

@ -1 +0,0 @@
fa5ea0ca1e0c7c2e40914f3202c7545de4dbca9c

View file

@ -1,104 +0,0 @@
#!/usr/bin/env python
# Copyright 2017 The Dart project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# Downloads trimmed-down Android Tools from Google Cloud Storage and extracts
# them to INSTALL_DIR, updating INSTALL_DIR/VERSION_* stamp files with current
# version. Does nothing if INSTALL_DIR/VERSION_* are already up to date.
import os
import shutil
import subprocess
import sys
import tarfile
# Path constants. (All of these should be absolute paths.)
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
DART_ROOT = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
# Should be the same as in upload.py.
INSTALL_DIR = os.path.join(DART_ROOT, 'third_party', 'android_tools')
sys.path.insert(0, os.path.join(DART_ROOT, 'tools'))
import find_depot_tools
DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
GSUTIL_PATH = os.path.join(DEPOT_PATH, 'gsutil.py')
def RunCommand(command):
"""Run command and return success (True) or failure."""
print 'Running %s' % (str(command))
if subprocess.call(command, shell=False) == 0:
return True
print 'Failed.'
return False
def GetInstalledVersion(version_stamp):
version_file = os.path.join(INSTALL_DIR, version_stamp)
if not os.path.exists(version_file):
return None
with open(version_file) as f:
return f.read().strip()
def VersionStampName(tools_name):
if sys.platform.startswith('linux'):
return 'VERSION_LINUX_' + tools_name.upper()
elif sys.platform == 'darwin':
return 'VERSION_MACOSX_' + tools_name.upper()
else:
print('NOTE: Will not download android tools. Unsupported platform: ' + sys.platform)
sys.exit(0)
def UpdateTools(tools_name):
"""Downloads zipped tools from Google Cloud Storage and extracts them,
stamping current version."""
# Read latest version.
version_stamp = VersionStampName(tools_name)
version = ''
with open(os.path.join(THIS_DIR, version_stamp)) as f:
version = f.read().strip()
# Return if installed binaries are up to date.
if version == GetInstalledVersion(version_stamp):
return
# Remove the old install directory checked out from git.
if os.path.exists(os.path.join(INSTALL_DIR, '.git')):
shutil.rmtree(INSTALL_DIR)
# Make sure that the install directory exists.
if not os.path.exists(INSTALL_DIR):
os.mkdir(INSTALL_DIR)
# Remove current installation.
tools_root = os.path.join(INSTALL_DIR, tools_name)
if os.path.exists(tools_root):
shutil.rmtree(tools_root)
# Download tools from GCS.
archive_path = os.path.join(INSTALL_DIR, tools_name + '.tar.gz')
download_cmd = ['python', GSUTIL_PATH, 'cp',
'gs://mojo/android/tool/%s.tar.gz' % version,
archive_path]
if not RunCommand(download_cmd):
print ('WARNING: Failed to download Android tools.')
return
print "Extracting Android tools (" + tools_name + ")"
with tarfile.open(archive_path) as arch:
arch.extractall(INSTALL_DIR)
os.remove(archive_path)
# Write version as the last step.
with open(os.path.join(INSTALL_DIR, version_stamp), 'w+') as f:
f.write('%s\n' % version)
def main():
UpdateTools('sdk')
UpdateTools('ndk')
if __name__ == '__main__':
sys.exit(main())

View file

@ -131,7 +131,7 @@ def ProcessOptions(options, args):
if os_name != 'android':
print "Unsupported target os %s" % os_name
return False
if not HOST_OS in ['linux', 'macos']:
if not HOST_OS in ['linux']:
print ("Cross-compilation to %s is not supported on host os %s."
% (os_name, HOST_OS))
return False

View file

@ -1,20 +0,0 @@
#!/usr/bin/env python
# Copyright 2017 The Dart project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Forwards to the clang-format for the Linux toolchain."""
import os
import subprocess
import sys
DART_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TOOLCHAIN = os.path.join(DART_ROOT, 'buildtools', 'toolchain')
CLANG_FORMAT = os.path.join(TOOLCHAIN, 'clang+llvm-x86_64-linux', 'bin', 'clang-format')
def main(argv):
return subprocess.call([CLANG_FORMAT] + argv[1:])
if __name__ == '__main__':
sys.exit(main(sys.argv))

View file

@ -1,20 +0,0 @@
#!/usr/bin/env python
# Copyright 2017 The Dart project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Forwards to the clang-format for the MacOS toolchain."""
import os
import subprocess
import sys
DART_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TOOLCHAIN = os.path.join(DART_ROOT, 'buildtools', 'toolchain')
CLANG_FORMAT = os.path.join(TOOLCHAIN, 'clang+llvm-x86_64-darwin', 'bin', 'clang-format')
def main(argv):
return subprocess.call([CLANG_FORMAT] + argv[1:])
if __name__ == '__main__':
sys.exit(main(sys.argv))

View file

@ -1,88 +0,0 @@
#!/usr/bin/env python
# Copyright 2017 The Dart project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Pulls down tools required to build Dart."""
import os
import subprocess
import shutil
import sys
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
DART_ROOT = os.path.abspath(os.path.join(THIS_DIR, '..', '..'))
BUILDTOOLS = os.path.join(DART_ROOT, 'buildtools')
TOOLS_BUILDTOOLS = os.path.join(DART_ROOT, 'tools', 'buildtools')
sys.path.insert(0, os.path.join(DART_ROOT, 'tools'))
import find_depot_tools
DEPOT_PATH = find_depot_tools.add_depot_tools_to_path()
def Update():
path = os.path.join(BUILDTOOLS, 'update.sh')
return subprocess.call(['/bin/bash', path, '--toolchain', '--gn'], cwd=DART_ROOT)
def UpdateGNOnWindows():
sha1_file = os.path.join(BUILDTOOLS, 'win', 'gn.exe.sha1')
downloader_script = os.path.join(DEPOT_PATH, 'download_from_google_storage.py')
download_cmd = [
'python',
downloader_script,
'--no_auth',
'--no_resume',
'--quiet',
'--platform=win*',
'--bucket',
'chromium-gn',
'-s',
sha1_file
]
return subprocess.call(download_cmd)
def UpdateClangFormatOnWindows():
sha1_file = os.path.join(TOOLS_BUILDTOOLS, 'win', 'clang-format.exe.sha1')
output_dir = os.path.join(BUILDTOOLS, 'win', 'clang-format.exe')
downloader_script = os.path.join(DEPOT_PATH, 'download_from_google_storage.py')
download_cmd = [
'python',
downloader_script,
'--no_auth',
'--no_resume',
'--quiet',
'--platform=win',
'--bucket',
'chromium-clang-format',
'-s',
sha1_file,
'-o',
output_dir
]
return subprocess.call(download_cmd)
def CopyClangFormatScripts():
linux_script = os.path.join(TOOLS_BUILDTOOLS, 'linux64', 'clang-format')
mac_script = os.path.join(TOOLS_BUILDTOOLS, 'mac', 'clang-format')
linux_dest = os.path.join(BUILDTOOLS, 'linux64', 'clang-format')
mac_dest = os.path.join(BUILDTOOLS, 'mac', 'clang-format')
shutil.copy2(linux_script, linux_dest)
shutil.copy2(mac_script, mac_dest)
def main(argv):
if sys.platform.startswith('win'):
result = UpdateGNOnWindows()
if result != 0:
return result
return UpdateClangFormatOnWindows()
CopyClangFormatScripts()
return Update()
if __name__ == '__main__':
sys.exit(main(sys.argv))

View file

@ -1,2 +0,0 @@
6ddedd571c56b8c184f30a3c1fc36984e8c10ccd

909
tools/clang/scripts/update.py Executable file
View file

@ -0,0 +1,909 @@
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""This script is used to download prebuilt clang binaries.
It is also used by package.py to build the prebuilt clang binaries."""
import argparse
import distutils.spawn
import glob
import os
import pipes
import re
import shutil
import subprocess
import stat
import sys
import tarfile
import tempfile
import time
import urllib2
import zipfile
# Do NOT CHANGE this if you don't know what you're doing -- see
# https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md
# Reverting problematic clang rolls is safe, though.
CLANG_REVISION = '282487'
use_head_revision = 'LLVM_FORCE_HEAD_REVISION' in os.environ
if use_head_revision:
CLANG_REVISION = 'HEAD'
# This is incremented when pushing a new build of Clang at the same revision.
CLANG_SUB_REVISION=1
PACKAGE_VERSION = "%s-%s" % (CLANG_REVISION, CLANG_SUB_REVISION)
# Path constants. (All of these should be absolute paths.)
THIS_DIR = os.path.abspath(os.path.dirname(__file__))
CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
THIRD_PARTY_DIR = os.path.join(CHROMIUM_DIR, 'third_party')
LLVM_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm')
LLVM_BOOTSTRAP_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-bootstrap')
LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR,
'llvm-bootstrap-install')
LLVM_LTO_GOLD_PLUGIN_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-lto-gold-plugin')
CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'chrometools')
LLVM_BUILD_DIR = os.path.join(CHROMIUM_DIR, 'third_party', 'llvm-build',
'Release+Asserts')
COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'compiler-rt')
CLANG_DIR = os.path.join(LLVM_DIR, 'tools', 'clang')
LLD_DIR = os.path.join(LLVM_DIR, 'tools', 'lld')
# compiler-rt is built as part of the regular LLVM build on Windows to get
# the 64-bit runtime, and out-of-tree elsewhere.
# TODO(thakis): Try to unify this.
if sys.platform == 'win32':
COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'projects', 'compiler-rt')
else:
COMPILER_RT_DIR = os.path.join(LLVM_DIR, 'compiler-rt')
LIBCXX_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxx')
LIBCXXABI_DIR = os.path.join(LLVM_DIR, 'projects', 'libcxxabi')
LLVM_BUILD_TOOLS_DIR = os.path.abspath(
os.path.join(LLVM_DIR, '..', 'llvm-build-tools'))
STAMP_FILE = os.path.normpath(
os.path.join(LLVM_DIR, '..', 'llvm-build', 'cr_build_revision'))
BINUTILS_DIR = os.path.join(THIRD_PARTY_DIR, 'binutils')
BINUTILS_BIN_DIR = os.path.join(BINUTILS_DIR, BINUTILS_DIR,
'Linux_x64', 'Release', 'bin')
BFD_PLUGINS_DIR = os.path.join(BINUTILS_DIR, 'Linux_x64', 'Release',
'lib', 'bfd-plugins')
VERSION = '4.0.0'
ANDROID_NDK_DIR = os.path.join(
CHROMIUM_DIR, 'third_party', 'android_tools', 'ndk')
# URL for pre-built binaries.
CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
'https://commondatastorage.googleapis.com/chromium-browser-clang')
LLVM_REPO_URL='https://llvm.org/svn/llvm-project'
if 'LLVM_REPO_URL' in os.environ:
LLVM_REPO_URL = os.environ['LLVM_REPO_URL']
# Bump after VC updates.
DIA_DLL = {
'2013': 'msdia120.dll',
'2015': 'msdia140.dll',
}
def DownloadUrl(url, output_file):
"""Download url into output_file."""
CHUNK_SIZE = 4096
TOTAL_DOTS = 10
num_retries = 3
retry_wait_s = 5 # Doubled at each retry.
while True:
try:
sys.stdout.write('Downloading %s ' % url)
sys.stdout.flush()
response = urllib2.urlopen(url)
total_size = int(response.info().getheader('Content-Length').strip())
bytes_done = 0
dots_printed = 0
while True:
chunk = response.read(CHUNK_SIZE)
if not chunk:
break
output_file.write(chunk)
bytes_done += len(chunk)
num_dots = TOTAL_DOTS * bytes_done / total_size
sys.stdout.write('.' * (num_dots - dots_printed))
sys.stdout.flush()
dots_printed = num_dots
if bytes_done != total_size:
raise urllib2.URLError("only got %d of %d bytes" %
(bytes_done, total_size))
print ' Done.'
return
except urllib2.URLError as e:
sys.stdout.write('\n')
print e
if num_retries == 0 or isinstance(e, urllib2.HTTPError) and e.code == 404:
raise e
num_retries -= 1
print 'Retrying in %d s ...' % retry_wait_s
time.sleep(retry_wait_s)
retry_wait_s *= 2
def EnsureDirExists(path):
if not os.path.exists(path):
print "Creating directory %s" % path
os.makedirs(path)
def DownloadAndUnpack(url, output_dir):
with tempfile.TemporaryFile() as f:
DownloadUrl(url, f)
f.seek(0)
EnsureDirExists(output_dir)
if url.endswith('.zip'):
zipfile.ZipFile(f).extractall(path=output_dir)
else:
tarfile.open(mode='r:gz', fileobj=f).extractall(path=output_dir)
def ReadStampFile(path=STAMP_FILE):
"""Return the contents of the stamp file, or '' if it doesn't exist."""
try:
with open(path, 'r') as f:
return f.read().rstrip()
except IOError:
return ''
def WriteStampFile(s, path=STAMP_FILE):
"""Write s to the stamp file."""
EnsureDirExists(os.path.dirname(path))
with open(path, 'w') as f:
f.write(s)
f.write('\n')
def GetSvnRevision(svn_repo):
"""Returns current revision of the svn repo at svn_repo."""
if sys.platform == 'darwin':
# mac_files toolchain must be set for hermetic builds.
root = os.path.dirname(os.path.dirname(os.path.dirname(
os.path.dirname(__file__))))
sys.path.append(os.path.join(root, 'build'))
import mac_toolchain
mac_toolchain.SetToolchainEnvironment()
svn_info = subprocess.check_output('svn info ' + svn_repo, shell=True)
m = re.search(r'Revision: (\d+)', svn_info)
return m.group(1)
def RmTree(dir):
"""Delete dir."""
def ChmodAndRetry(func, path, _):
# Subversion can leave read-only files around.
if not os.access(path, os.W_OK):
os.chmod(path, stat.S_IWUSR)
return func(path)
raise
shutil.rmtree(dir, onerror=ChmodAndRetry)
def RmCmakeCache(dir):
"""Delete CMake cache related files from dir."""
for dirpath, dirs, files in os.walk(dir):
if 'CMakeCache.txt' in files:
os.remove(os.path.join(dirpath, 'CMakeCache.txt'))
if 'CMakeFiles' in dirs:
RmTree(os.path.join(dirpath, 'CMakeFiles'))
def RunCommand(command, msvc_arch=None, env=None, fail_hard=True):
"""Run command and return success (True) or failure; or if fail_hard is
True, exit on failure. If msvc_arch is set, runs the command in a
shell with the msvc tools for that architecture."""
if msvc_arch and sys.platform == 'win32':
command = GetVSVersion().SetupScript(msvc_arch) + ['&&'] + command
# https://docs.python.org/2/library/subprocess.html:
# "On Unix with shell=True [...] if args is a sequence, the first item
# specifies the command string, and any additional items will be treated as
# additional arguments to the shell itself. That is to say, Popen does the
# equivalent of:
# Popen(['/bin/sh', '-c', args[0], args[1], ...])"
#
# We want to pass additional arguments to command[0], not to the shell,
# so manually join everything into a single string.
# Annoyingly, for "svn co url c:\path", pipes.quote() thinks that it should
# quote c:\path but svn can't handle quoted paths on Windows. Since on
# Windows follow-on args are passed to args[0] instead of the shell, don't
# do the single-string transformation there.
if sys.platform != 'win32':
command = ' '.join([pipes.quote(c) for c in command])
print 'Running', command
if subprocess.call(command, env=env, shell=True) == 0:
return True
print 'Failed.'
if fail_hard:
sys.exit(1)
return False
def CopyFile(src, dst):
"""Copy a file from src to dst."""
print "Copying %s to %s" % (src, dst)
shutil.copy(src, dst)
def CopyDirectoryContents(src, dst, filename_filter=None):
"""Copy the files from directory src to dst
with an optional filename filter."""
dst = os.path.realpath(dst) # realpath() in case dst ends in /..
EnsureDirExists(dst)
for root, _, files in os.walk(src):
for f in files:
if filename_filter and not re.match(filename_filter, f):
continue
CopyFile(os.path.join(root, f), dst)
def Checkout(name, url, dir):
"""Checkout the SVN module at url into dir. Use name for the log message."""
print "Checking out %s r%s into '%s'" % (name, CLANG_REVISION, dir)
command = ['svn', 'checkout', '--force', url + '@' + CLANG_REVISION, dir]
if RunCommand(command, fail_hard=False):
return
if os.path.isdir(dir):
print "Removing %s." % (dir)
RmTree(dir)
print "Retrying."
RunCommand(command)
def DeleteChromeToolsShim():
OLD_SHIM_DIR = os.path.join(LLVM_DIR, 'tools', 'zzz-chrometools')
shutil.rmtree(OLD_SHIM_DIR, ignore_errors=True)
shutil.rmtree(CHROME_TOOLS_SHIM_DIR, ignore_errors=True)
def CreateChromeToolsShim():
"""Hooks the Chrome tools into the LLVM build.
Several Chrome tools have dependencies on LLVM/Clang libraries. The LLVM build
detects implicit tools in the tools subdirectory, so this helper install a
shim CMakeLists.txt that forwards to the real directory for the Chrome tools.
Note that the shim directory name intentionally has no - or _. The implicit
tool detection logic munges them in a weird way."""
assert not any(i in os.path.basename(CHROME_TOOLS_SHIM_DIR) for i in '-_')
os.mkdir(CHROME_TOOLS_SHIM_DIR)
with file(os.path.join(CHROME_TOOLS_SHIM_DIR, 'CMakeLists.txt'), 'w') as f:
f.write('# Automatically generated by tools/clang/scripts/update.py. ' +
'Do not edit.\n')
f.write('# Since tools/clang is located in another directory, use the \n')
f.write('# two arg version to specify where build artifacts go. CMake\n')
f.write('# disallows reuse of the same binary dir for multiple source\n')
f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
f.write('# dirs, so the build artifacts need to go into a subdirectory.\n')
f.write('if (CHROMIUM_TOOLS_SRC)\n')
f.write(' add_subdirectory(${CHROMIUM_TOOLS_SRC} ' +
'${CMAKE_CURRENT_BINARY_DIR}/a)\n')
f.write('endif (CHROMIUM_TOOLS_SRC)\n')
def DownloadHostGcc(args):
"""Downloads gcc 4.8.5 and makes sure args.gcc_toolchain is set."""
if not sys.platform.startswith('linux') or args.gcc_toolchain:
return
# Unconditionally download a prebuilt gcc to guarantee the included libstdc++
# works on Ubuntu Precise.
gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc485precise')
if not os.path.exists(gcc_dir):
print 'Downloading pre-built GCC 4.8.5...'
DownloadAndUnpack(
CDS_URL + '/tools/gcc485precise.tgz', LLVM_BUILD_TOOLS_DIR)
args.gcc_toolchain = gcc_dir
def AddCMakeToPath():
"""Download CMake and add it to PATH."""
if sys.platform == 'win32':
zip_name = 'cmake-3.4.3-win32-x86.zip'
cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR,
'cmake-3.4.3-win32-x86', 'bin')
else:
suffix = 'Darwin' if sys.platform == 'darwin' else 'Linux'
zip_name = 'cmake343_%s.tgz' % suffix
cmake_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'cmake343', 'bin')
if not os.path.exists(cmake_dir):
DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
os.environ['PATH'] = cmake_dir + os.pathsep + os.environ.get('PATH', '')
def AddGnuWinToPath():
"""Download some GNU win tools and add them to PATH."""
if sys.platform != 'win32':
return
gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin')
GNUWIN_VERSION = '5'
GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp')
if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION:
print 'GNU Win tools already up to date.'
else:
zip_name = 'gnuwin-%s.zip' % GNUWIN_VERSION
DownloadAndUnpack(CDS_URL + '/tools/' + zip_name, LLVM_BUILD_TOOLS_DIR)
WriteStampFile(GNUWIN_VERSION, GNUWIN_STAMP)
os.environ['PATH'] = gnuwin_dir + os.pathsep + os.environ.get('PATH', '')
vs_version = None
def GetVSVersion():
global vs_version
if vs_version:
return vs_version
# Try using the toolchain in depot_tools.
# This sets environment variables used by SelectVisualStudioVersion below.
sys.path.append(os.path.join(CHROMIUM_DIR, 'build'))
import vs_toolchain
vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
# Use gyp to find the MSVS installation, either in depot_tools as per above,
# or a system-wide installation otherwise.
sys.path.append(os.path.join(CHROMIUM_DIR, 'tools', 'gyp', 'pylib'))
import gyp.MSVSVersion
vs_version = gyp.MSVSVersion.SelectVisualStudioVersion(
vs_toolchain.GetVisualStudioVersion())
return vs_version
def CopyDiaDllTo(target_dir):
# This script always wants to use the 64-bit msdia*.dll.
dia_path = os.path.join(GetVSVersion().Path(), 'DIA SDK', 'bin', 'amd64')
dia_dll = os.path.join(dia_path, DIA_DLL[GetVSVersion().ShortName()])
CopyFile(dia_dll, target_dir)
def VeryifyVersionOfBuiltClangMatchesVERSION():
"""Checks that `clang --version` outputs VERSION. If this fails, VERSION
in this file is out-of-date and needs to be updated (possibly in an
`if use_head_revision:` block in main() first)."""
clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
if sys.platform == 'win32':
# TODO: Parse `clang-cl /?` output for built clang's version and check that
# to check the binary we're actually shipping? But clang-cl.exe is just
# a copy of clang.exe, so this does check the same thing.
clang += '.exe'
version_out = subprocess.check_output([clang, '--version'])
version_out = re.match(r'clang version ([0-9.]+)', version_out).group(1)
if version_out != VERSION:
print ('unexpected clang version %s (not %s), update VERSION in update.py'
% (version_out, VERSION))
sys.exit(1)
def UpdateClang(args):
print 'Updating Clang to %s...' % PACKAGE_VERSION
need_gold_plugin = 'LLVM_DOWNLOAD_GOLD_PLUGIN' in os.environ or (
sys.platform.startswith('linux') and
'buildtype=Official' in os.environ.get('GYP_DEFINES', '') and
'branding=Chrome' in os.environ.get('GYP_DEFINES', ''))
if ReadStampFile() == PACKAGE_VERSION and not args.force_local_build:
print 'Clang is already up to date.'
if not need_gold_plugin or os.path.exists(
os.path.join(LLVM_BUILD_DIR, "lib/LLVMgold.so")):
return 0
# Reset the stamp file in case the build is unsuccessful.
WriteStampFile('')
if not args.force_local_build:
cds_file = "clang-%s.tgz" % PACKAGE_VERSION
if sys.platform == 'win32' or sys.platform == 'cygwin':
cds_full_url = CDS_URL + '/Win/' + cds_file
elif sys.platform == 'darwin':
cds_full_url = CDS_URL + '/Mac/' + cds_file
else:
assert sys.platform.startswith('linux')
cds_full_url = CDS_URL + '/Linux_x64/' + cds_file
print 'Downloading prebuilt clang'
if os.path.exists(LLVM_BUILD_DIR):
RmTree(LLVM_BUILD_DIR)
try:
DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR)
print 'clang %s unpacked' % PACKAGE_VERSION
if sys.platform == 'win32':
CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
# Download the gold plugin if requested to by an environment variable.
# This is used by the CFI ClusterFuzz bot, and it's required for official
# builds on linux.
if need_gold_plugin:
RunCommand(['python', CHROMIUM_DIR+'/build/download_gold_plugin.py'])
WriteStampFile(PACKAGE_VERSION)
return 0
except urllib2.URLError:
print 'Failed to download prebuilt clang %s' % cds_file
print 'Use --force-local-build if you want to build locally.'
print 'Exiting.'
return 1
if args.with_android and not os.path.exists(ANDROID_NDK_DIR):
print 'Android NDK not found at ' + ANDROID_NDK_DIR
print 'The Android NDK is needed to build a Clang whose -fsanitize=address'
print 'works on Android. See '
print 'https://www.chromium.org/developers/how-tos/android-build-instructions'
print 'for how to install the NDK, or pass --without-android.'
return 1
DownloadHostGcc(args)
AddCMakeToPath()
AddGnuWinToPath()
DeleteChromeToolsShim()
Checkout('LLVM', LLVM_REPO_URL + '/llvm/trunk', LLVM_DIR)
Checkout('Clang', LLVM_REPO_URL + '/cfe/trunk', CLANG_DIR)
if sys.platform == 'win32' or use_head_revision:
Checkout('LLD', LLVM_REPO_URL + '/lld/trunk', LLD_DIR)
Checkout('compiler-rt', LLVM_REPO_URL + '/compiler-rt/trunk', COMPILER_RT_DIR)
if sys.platform == 'darwin':
# clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
# (i.e. this is needed for bootstrap builds).
Checkout('libcxx', LLVM_REPO_URL + '/libcxx/trunk', LIBCXX_DIR)
# We used to check out libcxxabi on OS X; we no longer need that.
if os.path.exists(LIBCXXABI_DIR):
RmTree(LIBCXXABI_DIR)
cc, cxx = None, None
libstdcpp = None
if args.gcc_toolchain: # This option is only used on Linux.
# Use the specified gcc installation for building.
cc = os.path.join(args.gcc_toolchain, 'bin', 'gcc')
cxx = os.path.join(args.gcc_toolchain, 'bin', 'g++')
if not os.access(cc, os.X_OK):
print 'Invalid --gcc-toolchain: "%s"' % args.gcc_toolchain
print '"%s" does not appear to be valid.' % cc
return 1
# Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap
# compiler, etc.) find the .so.
libstdcpp = subprocess.check_output(
[cxx, '-print-file-name=libstdc++.so.6']).rstrip()
os.environ['LD_LIBRARY_PATH'] = os.path.dirname(libstdcpp)
cflags = []
cxxflags = []
ldflags = []
base_cmake_args = ['-GNinja',
'-DCMAKE_BUILD_TYPE=Release',
'-DLLVM_ENABLE_ASSERTIONS=ON',
'-DLLVM_ENABLE_THREADS=OFF',
'-DLLVM_ENABLE_TIMESTAMPS=OFF',
# Statically link MSVCRT to avoid DLL dependencies.
'-DLLVM_USE_CRT_RELEASE=MT',
]
binutils_incdir = ''
if sys.platform.startswith('linux'):
binutils_incdir = os.path.join(BINUTILS_DIR, 'Linux_x64/Release/include')
if args.bootstrap:
print 'Building bootstrap compiler'
EnsureDirExists(LLVM_BOOTSTRAP_DIR)
os.chdir(LLVM_BOOTSTRAP_DIR)
bootstrap_args = base_cmake_args + [
'-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
'-DLLVM_TARGETS_TO_BUILD=host',
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BOOTSTRAP_INSTALL_DIR,
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
]
if cc is not None: bootstrap_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None: bootstrap_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
RmCmakeCache('.')
RunCommand(['cmake'] + bootstrap_args + [LLVM_DIR], msvc_arch='x64')
RunCommand(['ninja'], msvc_arch='x64')
if args.run_tests:
if sys.platform == 'win32':
CopyDiaDllTo(os.path.join(LLVM_BOOTSTRAP_DIR, 'bin'))
RunCommand(['ninja', 'check-all'], msvc_arch='x64')
RunCommand(['ninja', 'install'], msvc_arch='x64')
if args.gcc_toolchain:
# Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
# compiler can start.
CopyFile(libstdcpp, os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib'))
if sys.platform == 'win32':
cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang-cl.exe')
# CMake has a hard time with backslashes in compiler paths:
# https://stackoverflow.com/questions/13050827
cc = cc.replace('\\', '/')
cxx = cxx.replace('\\', '/')
else:
cc = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang')
cxx = os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'bin', 'clang++')
if args.gcc_toolchain:
# Tell the bootstrap compiler to use a specific gcc prefix to search
# for standard library headers and shared object files.
cflags = ['--gcc-toolchain=' + args.gcc_toolchain]
cxxflags = ['--gcc-toolchain=' + args.gcc_toolchain]
print 'Building final compiler'
# Build LLVM gold plugin with LTO. That speeds up the linker by ~10%.
# We only use LTO for Linux now.
if args.bootstrap and args.lto_gold_plugin:
print 'Building LTO LLVM Gold plugin'
if os.path.exists(LLVM_LTO_GOLD_PLUGIN_DIR):
RmTree(LLVM_LTO_GOLD_PLUGIN_DIR)
EnsureDirExists(LLVM_LTO_GOLD_PLUGIN_DIR)
os.chdir(LLVM_LTO_GOLD_PLUGIN_DIR)
# Create a symlink to LLVMgold.so build in the previous step so that ar
# and ranlib could find it while linking LLVMgold.so with LTO.
EnsureDirExists(BFD_PLUGINS_DIR)
RunCommand(['ln', '-sf',
os.path.join(LLVM_BOOTSTRAP_INSTALL_DIR, 'lib', 'LLVMgold.so'),
os.path.join(BFD_PLUGINS_DIR, 'LLVMgold.so')])
lto_cflags = ['-flto']
lto_ldflags = ['-fuse-ld=gold']
if args.gcc_toolchain:
# Tell the bootstrap compiler to use a specific gcc prefix to search
# for standard library headers and shared object files.
lto_cflags += ['--gcc-toolchain=' + args.gcc_toolchain]
lto_cmake_args = base_cmake_args + [
'-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
'-DCMAKE_C_COMPILER=' + cc,
'-DCMAKE_CXX_COMPILER=' + cxx,
'-DCMAKE_C_FLAGS=' + ' '.join(lto_cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(lto_cflags),
'-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(lto_ldflags),
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(lto_ldflags),
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(lto_ldflags)]
# We need to use the proper binutils which support LLVM Gold plugin.
lto_env = os.environ.copy()
lto_env['PATH'] = BINUTILS_BIN_DIR + os.pathsep + lto_env.get('PATH', '')
RmCmakeCache('.')
RunCommand(['cmake'] + lto_cmake_args + [LLVM_DIR], env=lto_env)
RunCommand(['ninja', 'LLVMgold'], env=lto_env)
# LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
# needed, on OS X it requires libc++. clang only automatically links to libc++
# when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run
# on OS X versions as old as 10.7.
deployment_target = ''
if sys.platform == 'darwin' and args.bootstrap:
# When building on 10.9, /usr/include usually doesn't exist, and while
# Xcode's clang automatically sets a sysroot, self-built clangs don't.
cflags = ['-isysroot', subprocess.check_output(
['xcrun', '--show-sdk-path']).rstrip()]
cxxflags = ['-stdlib=libc++'] + cflags
ldflags += ['-stdlib=libc++']
deployment_target = '10.7'
# Running libc++ tests takes a long time. Since it was only needed for
# the install step above, don't build it as part of the main build.
# This makes running package.py over 10% faster (30 min instead of 34 min)
RmTree(LIBCXX_DIR)
# Build clang.
# If building at head, define a macro that plugins can use for #ifdefing
# out code that builds at head, but not at CLANG_REVISION or vice versa.
if use_head_revision:
cflags += ['-DLLVM_FORCE_HEAD_REVISION']
cxxflags += ['-DLLVM_FORCE_HEAD_REVISION']
CreateChromeToolsShim()
deployment_env = None
if deployment_target:
deployment_env = os.environ.copy()
deployment_env['MACOSX_DEPLOYMENT_TARGET'] = deployment_target
cmake_args = []
# TODO(thakis): Unconditionally append this to base_cmake_args instead once
# compiler-rt can build with clang-cl on Windows (http://llvm.org/PR23698)
cc_args = base_cmake_args if sys.platform != 'win32' else cmake_args
if cc is not None: cc_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None: cc_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
cmake_args += base_cmake_args + [
'-DLLVM_BINUTILS_INCDIR=' + binutils_incdir,
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags),
'-DCMAKE_EXE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_SHARED_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_MODULE_LINKER_FLAGS=' + ' '.join(ldflags),
'-DCMAKE_INSTALL_PREFIX=' + LLVM_BUILD_DIR,
# TODO(thakis): Remove this once official builds pass -Wl,--build-id
# explicitly, https://crbug.com/622775
'-DENABLE_LINKER_BUILD_ID=ON',
'-DCHROMIUM_TOOLS_SRC=%s' % os.path.join(CHROMIUM_DIR, 'tools', 'clang'),
'-DCHROMIUM_TOOLS=%s' % ';'.join(args.tools)]
EnsureDirExists(LLVM_BUILD_DIR)
os.chdir(LLVM_BUILD_DIR)
RmCmakeCache('.')
RunCommand(['cmake'] + cmake_args + [LLVM_DIR],
msvc_arch='x64', env=deployment_env)
if args.gcc_toolchain:
# Copy in the right stdlibc++.so.6 so clang can start.
if not os.path.exists(os.path.join(LLVM_BUILD_DIR, 'lib')):
os.mkdir(os.path.join(LLVM_BUILD_DIR, 'lib'))
libstdcpp = subprocess.check_output(
[cxx] + cxxflags + ['-print-file-name=libstdc++.so.6']).rstrip()
CopyFile(libstdcpp, os.path.join(LLVM_BUILD_DIR, 'lib'))
RunCommand(['ninja'], msvc_arch='x64')
if args.tools:
# If any Chromium tools were built, install those now.
RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
if sys.platform == 'darwin':
# See http://crbug.com/256342
RunCommand(['strip', '-x', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
elif sys.platform.startswith('linux'):
RunCommand(['strip', os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')])
VeryifyVersionOfBuiltClangMatchesVERSION()
# Do an out-of-tree build of compiler-rt.
# On Windows, this is used to get the 32-bit ASan run-time.
# TODO(hans): Remove once the regular build above produces this.
# On Mac and Linux, this is used to get the regular 64-bit run-time.
# Do a clobbered build due to cmake changes.
if os.path.isdir(COMPILER_RT_BUILD_DIR):
RmTree(COMPILER_RT_BUILD_DIR)
os.makedirs(COMPILER_RT_BUILD_DIR)
os.chdir(COMPILER_RT_BUILD_DIR)
# TODO(thakis): Add this once compiler-rt can build with clang-cl (see
# above).
#if args.bootstrap and sys.platform == 'win32':
# The bootstrap compiler produces 64-bit binaries by default.
#cflags += ['-m32']
#cxxflags += ['-m32']
compiler_rt_args = base_cmake_args + [
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cxxflags)]
if sys.platform == 'darwin':
compiler_rt_args += ['-DCOMPILER_RT_ENABLE_IOS=ON']
if sys.platform != 'win32':
compiler_rt_args += ['-DLLVM_CONFIG_PATH=' +
os.path.join(LLVM_BUILD_DIR, 'bin', 'llvm-config'),
'-DSANITIZER_MIN_OSX_VERSION="10.7"']
# compiler-rt is part of the llvm checkout on Windows but a stand-alone
# directory elsewhere, see the TODO above COMPILER_RT_DIR.
RmCmakeCache('.')
RunCommand(['cmake'] + compiler_rt_args +
[LLVM_DIR if sys.platform == 'win32' else COMPILER_RT_DIR],
msvc_arch='x86', env=deployment_env)
RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86')
# Copy select output to the main tree.
# TODO(hans): Make this (and the .gypi and .isolate files) version number
# independent.
if sys.platform == 'win32':
platform = 'windows'
elif sys.platform == 'darwin':
platform = 'darwin'
else:
assert sys.platform.startswith('linux')
platform = 'linux'
asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', platform)
if sys.platform == 'win32':
# TODO(thakis): This too is due to compiler-rt being part of the checkout
# on Windows, see TODO above COMPILER_RT_DIR.
asan_rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang',
VERSION, 'lib', platform)
asan_rt_lib_dst_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
VERSION, 'lib', platform)
# Blacklists:
CopyDirectoryContents(os.path.join(asan_rt_lib_src_dir, '..', '..'),
os.path.join(asan_rt_lib_dst_dir, '..', '..'),
r'^.*blacklist\.txt$')
# Headers:
if sys.platform != 'win32':
CopyDirectoryContents(
os.path.join(COMPILER_RT_BUILD_DIR, 'include/sanitizer'),
os.path.join(LLVM_BUILD_DIR, 'lib/clang', VERSION, 'include/sanitizer'))
# Static and dynamic libraries:
CopyDirectoryContents(asan_rt_lib_src_dir, asan_rt_lib_dst_dir)
if sys.platform == 'darwin':
for dylib in glob.glob(os.path.join(asan_rt_lib_dst_dir, '*.dylib')):
# Fix LC_ID_DYLIB for the ASan dynamic libraries to be relative to
# @executable_path.
# TODO(glider): this is transitional. We'll need to fix the dylib
# name either in our build system, or in Clang. See also
# http://crbug.com/344836.
subprocess.call(['install_name_tool', '-id',
'@executable_path/' + os.path.basename(dylib), dylib])
if sys.platform == 'win32':
# Make an extra copy of the sanitizer headers, to be put on the include path
# of the fallback compiler.
sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
VERSION, 'include', 'sanitizer')
aux_sanitizer_include_dir = os.path.join(LLVM_BUILD_DIR, 'lib', 'clang',
VERSION, 'include_sanitizer',
'sanitizer')
EnsureDirExists(aux_sanitizer_include_dir)
for _, _, files in os.walk(sanitizer_include_dir):
for f in files:
CopyFile(os.path.join(sanitizer_include_dir, f),
aux_sanitizer_include_dir)
if args.with_android:
make_toolchain = os.path.join(
ANDROID_NDK_DIR, 'build', 'tools', 'make_standalone_toolchain.py')
for target_arch in ['aarch64', 'arm', 'i686']:
# Make standalone Android toolchain for target_arch.
toolchain_dir = os.path.join(
LLVM_BUILD_DIR, 'android-toolchain-' + target_arch)
RunCommand([
make_toolchain,
'--api=' + ('21' if target_arch == 'aarch64' else '19'),
'--force',
'--install-dir=%s' % toolchain_dir,
'--stl=stlport',
'--arch=' + {
'aarch64': 'arm64',
'arm': 'arm',
'i686': 'x86',
}[target_arch]])
# Android NDK r9d copies a broken unwind.h into the toolchain, see
# http://crbug.com/357890
for f in glob.glob(os.path.join(toolchain_dir, 'include/c++/*/unwind.h')):
os.remove(f)
# Build ASan runtime for Android in a separate build tree.
build_dir = os.path.join(LLVM_BUILD_DIR, 'android-' + target_arch)
if not os.path.exists(build_dir):
os.mkdir(os.path.join(build_dir))
os.chdir(build_dir)
cflags = ['--target=%s-linux-androideabi' % target_arch,
'--sysroot=%s/sysroot' % toolchain_dir,
'-B%s' % toolchain_dir]
android_args = base_cmake_args + [
'-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'),
'-DCMAKE_CXX_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang++'),
'-DLLVM_CONFIG_PATH=' + os.path.join(LLVM_BUILD_DIR, 'bin/llvm-config'),
'-DCMAKE_C_FLAGS=' + ' '.join(cflags),
'-DCMAKE_CXX_FLAGS=' + ' '.join(cflags),
'-DANDROID=1']
RmCmakeCache('.')
RunCommand(['cmake'] + android_args + [COMPILER_RT_DIR])
RunCommand(['ninja', 'libclang_rt.asan-%s-android.so' % target_arch])
# And copy it into the main build tree.
runtime = 'libclang_rt.asan-%s-android.so' % target_arch
for root, _, files in os.walk(build_dir):
if runtime in files:
shutil.copy(os.path.join(root, runtime), asan_rt_lib_dst_dir)
# Run tests.
if args.run_tests or use_head_revision:
os.chdir(LLVM_BUILD_DIR)
RunCommand(['ninja', 'cr-check-all'], msvc_arch='x64')
if args.run_tests:
if sys.platform == 'win32':
CopyDiaDllTo(os.path.join(LLVM_BUILD_DIR, 'bin'))
os.chdir(LLVM_BUILD_DIR)
RunCommand(['ninja', 'check-all'], msvc_arch='x64')
WriteStampFile(PACKAGE_VERSION)
print 'Clang update was successful.'
return 0
def main():
parser = argparse.ArgumentParser(description='Build Clang.')
parser.add_argument('--bootstrap', action='store_true',
help='first build clang with CC, then with itself.')
parser.add_argument('--if-needed', action='store_true',
help="run only if the script thinks clang is needed")
parser.add_argument('--force-local-build', action='store_true',
help="don't try to download prebuild binaries")
parser.add_argument('--gcc-toolchain', help='set the version for which gcc '
'version be used for building; --gcc-toolchain=/opt/foo '
'picks /opt/foo/bin/gcc')
parser.add_argument('--lto-gold-plugin', action='store_true',
help='build LLVM Gold plugin with LTO')
parser.add_argument('--llvm-force-head-revision', action='store_true',
help=('use the revision in the repo when printing '
'the revision'))
parser.add_argument('--print-revision', action='store_true',
help='print current clang revision and exit.')
parser.add_argument('--print-clang-version', action='store_true',
help='print current clang version (e.g. x.y.z) and exit.')
parser.add_argument('--run-tests', action='store_true',
help='run tests after building; only for local builds')
parser.add_argument('--tools', nargs='*',
help='select which chrome tools to build',
default=['plugins', 'blink_gc_plugin'])
parser.add_argument('--without-android', action='store_false',
help='don\'t build Android ASan runtime (linux only)',
dest='with_android',
default=sys.platform.startswith('linux'))
args = parser.parse_args()
if args.lto_gold_plugin and not args.bootstrap:
print '--lto-gold-plugin requires --bootstrap'
return 1
if args.lto_gold_plugin and not sys.platform.startswith('linux'):
print '--lto-gold-plugin is only effective on Linux. Ignoring the option.'
args.lto_gold_plugin = False
if args.if_needed:
is_clang_required = False
# clang is always used on Mac and Linux.
if sys.platform == 'darwin' or sys.platform.startswith('linux'):
is_clang_required = True
# clang requested via $GYP_DEFINES.
if re.search(r'\b(clang|asan|lsan|msan|tsan)=1',
os.environ.get('GYP_DEFINES', '')):
is_clang_required = True
# clang previously downloaded, keep it up to date.
# If you don't want this, delete third_party/llvm-build on your machine.
if os.path.isdir(LLVM_BUILD_DIR):
is_clang_required = True
if not is_clang_required:
return 0
if re.search(r'\b(make_clang_dir)=', os.environ.get('GYP_DEFINES', '')):
print 'Skipping Clang update (make_clang_dir= was set in GYP_DEFINES).'
return 0
global CLANG_REVISION, PACKAGE_VERSION
if args.print_revision:
if use_head_revision or args.llvm_force_head_revision:
print GetSvnRevision(LLVM_DIR)
else:
print PACKAGE_VERSION
return 0
if args.print_clang_version:
sys.stdout.write(VERSION)
return 0
# Don't buffer stdout, so that print statements are immediately flushed.
# Do this only after --print-revision has been handled, else we'll get
# an error message when this script is run from gn for some reason.
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0)
if use_head_revision:
# Use a real revision number rather than HEAD to make sure that the stamp
# file logic works.
CLANG_REVISION = GetSvnRevision(LLVM_REPO_URL)
PACKAGE_VERSION = CLANG_REVISION + '-0'
args.force_local_build = True
if 'OS=android' not in os.environ.get('GYP_DEFINES', ''):
# Only build the Android ASan rt on ToT bots when targetting Android.
args.with_android = False
return UpdateClang(args)
if __name__ == '__main__':
sys.exit(main())

724
tools/clang/scripts/update.sh Executable file
View file

@ -0,0 +1,724 @@
#!/usr/bin/env bash
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# This script will check out llvm and clang into third_party/llvm and build it.
# Do NOT CHANGE this if you don't know what you're doing -- see
# https://code.google.com/p/chromium/wiki/UpdatingClang
# Reverting problematic clang rolls is safe, though.
CLANG_REVISION=241602
# This is incremented when pushing a new build of Clang at the same revision.
CLANG_SUB_REVISION=3
PACKAGE_VERSION="${CLANG_REVISION}-${CLANG_SUB_REVISION}"
THIS_DIR="$(dirname "${0}")"
LLVM_DIR="${THIS_DIR}/../../../third_party/llvm"
LLVM_BUILD_DIR="${LLVM_DIR}/../llvm-build/Release+Asserts"
COMPILER_RT_BUILD_DIR="${LLVM_DIR}/../llvm-build/compiler-rt"
LLVM_BOOTSTRAP_DIR="${LLVM_DIR}/../llvm-bootstrap"
LLVM_BOOTSTRAP_INSTALL_DIR="${LLVM_DIR}/../llvm-bootstrap-install"
CLANG_DIR="${LLVM_DIR}/tools/clang"
COMPILER_RT_DIR="${LLVM_DIR}/compiler-rt"
LIBCXX_DIR="${LLVM_DIR}/projects/libcxx"
LIBCXXABI_DIR="${LLVM_DIR}/projects/libcxxabi"
ANDROID_NDK_DIR="${THIS_DIR}/../../../third_party/android_tools/ndk"
STAMP_FILE="${LLVM_DIR}/../llvm-build/cr_build_revision"
CHROMIUM_TOOLS_DIR="${THIS_DIR}/.."
BINUTILS_DIR="${THIS_DIR}/../../../third_party/binutils"
ABS_CHROMIUM_TOOLS_DIR="${PWD}/${CHROMIUM_TOOLS_DIR}"
ABS_LIBCXX_DIR="${PWD}/${LIBCXX_DIR}"
ABS_LIBCXXABI_DIR="${PWD}/${LIBCXXABI_DIR}"
ABS_LLVM_DIR="${PWD}/${LLVM_DIR}"
ABS_LLVM_BUILD_DIR="${PWD}/${LLVM_BUILD_DIR}"
ABS_COMPILER_RT_DIR="${PWD}/${COMPILER_RT_DIR}"
ABS_BINUTILS_DIR="${PWD}/${BINUTILS_DIR}"
# ${A:-a} returns $A if it's set, a else.
LLVM_REPO_URL=${LLVM_URL:-https://llvm.org/svn/llvm-project}
CDS_URL=https://commondatastorage.googleapis.com/chromium-browser-clang
if [[ -z "$GYP_DEFINES" ]]; then
GYP_DEFINES=
fi
if [[ -z "$GYP_GENERATORS" ]]; then
GYP_GENERATORS=
fi
if [[ -z "$LLVM_DOWNLOAD_GOLD_PLUGIN" ]]; then
LLVM_DOWNLOAD_GOLD_PLUGIN=
fi
# Die if any command dies, error on undefined variable expansions.
set -eu
if [[ -n ${LLVM_FORCE_HEAD_REVISION:-''} ]]; then
# Use a real revision number rather than HEAD to make sure that the stamp file
# logic works.
CLANG_REVISION=$(svn info "$LLVM_REPO_URL" \
| grep 'Revision:' | awk '{ printf $2; }')
PACKAGE_VERSION="${CLANG_REVISION}-0"
fi
OS="$(uname -s)"
# Parse command line options.
if_needed=
force_local_build=
run_tests=
bootstrap=
with_android=yes
chrome_tools="plugins;blink_gc_plugin"
gcc_toolchain=
with_patches=yes
if [[ "${OS}" = "Darwin" ]]; then
with_android=
fi
while [[ $# > 0 ]]; do
case $1 in
--bootstrap)
bootstrap=yes
;;
--if-needed)
if_needed=yes
;;
--force-local-build)
force_local_build=yes
;;
--print-revision)
if [[ -n ${LLVM_FORCE_HEAD_REVISION:-''} ]]; then
svn info "$LLVM_DIR" | grep 'Revision:' | awk '{ printf $2; }'
else
echo $PACKAGE_VERSION
fi
exit 0
;;
--run-tests)
run_tests=yes
;;
--without-android)
with_android=
;;
--without-patches)
with_patches=
;;
--with-chrome-tools)
shift
if [[ $# == 0 ]]; then
echo "--with-chrome-tools requires an argument."
exit 1
fi
chrome_tools=$1
;;
--gcc-toolchain)
shift
if [[ $# == 0 ]]; then
echo "--gcc-toolchain requires an argument."
exit 1
fi
if [[ -x "$1/bin/gcc" ]]; then
gcc_toolchain=$1
else
echo "Invalid --gcc-toolchain: '$1'."
echo "'$1/bin/gcc' does not appear to be valid."
exit 1
fi
;;
--help)
echo "usage: $0 [--force-local-build] [--if-needed] [--run-tests] "
echo "--bootstrap: First build clang with CC, then with itself."
echo "--force-local-build: Don't try to download prebuilt binaries."
echo "--if-needed: Download clang only if the script thinks it is needed."
echo "--run-tests: Run tests after building. Only for local builds."
echo "--print-revision: Print current clang revision and exit."
echo "--without-android: Don't build ASan Android runtime library."
echo "--with-chrome-tools: Select which chrome tools to build." \
"Defaults to plugins;blink_gc_plugin."
echo " Example: --with-chrome-tools plugins;empty-string"
echo "--gcc-toolchain: Set the prefix for which GCC version should"
echo " be used for building. For example, to use gcc in"
echo " /opt/foo/bin/gcc, use '--gcc-toolchain '/opt/foo"
echo "--without-patches: Don't apply local patches."
echo
exit 1
;;
*)
echo "Unknown argument: '$1'."
echo "Use --help for help."
exit 1
;;
esac
shift
done
if [[ -n ${LLVM_FORCE_HEAD_REVISION:-''} ]]; then
force_local_build=yes
# Skip local patches when using HEAD: they probably don't apply anymore.
with_patches=
if ! [[ "$GYP_DEFINES" =~ .*OS=android.* ]]; then
# Only build the Android ASan rt when targetting Android.
with_android=
fi
LLVM_BUILD_TOOLS_DIR="${ABS_LLVM_DIR}/../llvm-build-tools"
if [[ "${OS}" == "Linux" ]] && [[ -z "${gcc_toolchain}" ]]; then
if [[ $(gcc -dumpversion) < "4.7.0" ]]; then
# We need a newer GCC version.
if [[ ! -e "${LLVM_BUILD_TOOLS_DIR}/gcc482" ]]; then
echo "Downloading pre-built GCC 4.8.2..."
mkdir -p "${LLVM_BUILD_TOOLS_DIR}"
curl --fail -L "${CDS_URL}/tools/gcc482.tgz" | \
tar zxf - -C "${LLVM_BUILD_TOOLS_DIR}"
echo Done
fi
gcc_toolchain="${LLVM_BUILD_TOOLS_DIR}/gcc482"
else
# Always set gcc_toolchain; llvm-symbolizer needs the bundled libstdc++.
gcc_toolchain="$(dirname $(dirname $(which gcc)))"
fi
fi
if [[ "${OS}" == "Linux" || "${OS}" == "Darwin" ]]; then
if [[ $(cmake --version | grep -Eo '[0-9.]+') < "3.0" ]]; then
# We need a newer CMake version.
if [[ ! -e "${LLVM_BUILD_TOOLS_DIR}/cmake310" ]]; then
echo "Downloading pre-built CMake 3.10..."
mkdir -p "${LLVM_BUILD_TOOLS_DIR}"
curl --fail -L "${CDS_URL}/tools/cmake310_${OS}.tgz" | \
tar zxf - -C "${LLVM_BUILD_TOOLS_DIR}"
echo Done
fi
export PATH="${LLVM_BUILD_TOOLS_DIR}/cmake310/bin:${PATH}"
fi
fi
echo "LLVM_FORCE_HEAD_REVISION was set; using r${CLANG_REVISION}"
fi
if [[ -n "$if_needed" ]]; then
if [[ "${OS}" == "Darwin" ]]; then
# clang is always used on Mac.
true
elif [[ "${OS}" == "Linux" ]]; then
# clang is also aways used on Linux.
true
elif [[ "$GYP_DEFINES" =~ .*(clang|tsan|asan|lsan|msan)=1.* ]]; then
# clang requested via $GYP_DEFINES.
true
elif [[ -d "${LLVM_BUILD_DIR}" ]]; then
# clang previously downloaded, keep it up-to-date.
# If you don't want this, delete third_party/llvm-build on your machine.
true
else
# clang wasn't needed, not doing anything.
exit 0
fi
fi
# Check if there's anything to be done, exit early if not.
if [[ -f "${STAMP_FILE}" ]]; then
PREVIOUSLY_BUILT_REVISON=$(cat "${STAMP_FILE}")
if [[ -z "$force_local_build" ]] && \
[[ "${PREVIOUSLY_BUILT_REVISON}" = \
"${PACKAGE_VERSION}" ]]; then
echo "Clang already at ${PACKAGE_VERSION}"
exit 0
fi
fi
# To always force a new build if someone interrupts their build half way.
rm -f "${STAMP_FILE}"
if [[ -z "$force_local_build" ]]; then
# Check if there's a prebuilt binary and if so just fetch that. That's faster,
# and goma relies on having matching binary hashes on client and server too.
CDS_FILE="clang-${PACKAGE_VERSION}.tgz"
CDS_OUT_DIR=$(mktemp -d -t clang_download.XXXXXX)
CDS_OUTPUT="${CDS_OUT_DIR}/${CDS_FILE}"
if [ "${OS}" = "Linux" ]; then
CDS_FULL_URL="${CDS_URL}/Linux_x64/${CDS_FILE}"
elif [ "${OS}" = "Darwin" ]; then
CDS_FULL_URL="${CDS_URL}/Mac/${CDS_FILE}"
fi
echo Trying to download prebuilt clang
if which curl > /dev/null; then
curl -L --fail "${CDS_FULL_URL}" -o "${CDS_OUTPUT}" || \
rm -rf "${CDS_OUT_DIR}"
elif which wget > /dev/null; then
wget "${CDS_FULL_URL}" -O "${CDS_OUTPUT}" || rm -rf "${CDS_OUT_DIR}"
else
echo "Neither curl nor wget found. Please install one of these."
exit 1
fi
if [ -f "${CDS_OUTPUT}" ]; then
rm -rf "${LLVM_BUILD_DIR}"
mkdir -p "${LLVM_BUILD_DIR}"
tar -xzf "${CDS_OUTPUT}" -C "${LLVM_BUILD_DIR}"
echo clang "${PACKAGE_VERSION}" unpacked
echo "${PACKAGE_VERSION}" > "${STAMP_FILE}"
rm -rf "${CDS_OUT_DIR}"
# Download the gold plugin if requested to by an environment variable.
# This is used by the CFI ClusterFuzz bot.
if [[ -n "${LLVM_DOWNLOAD_GOLD_PLUGIN}" ]]; then
${THIS_DIR}/../../../build/download_gold_plugin.py
fi
exit 0
else
echo Did not find prebuilt clang "${PACKAGE_VERSION}", building
fi
fi
if [[ -n "${with_android}" ]] && ! [[ -d "${ANDROID_NDK_DIR}" ]]; then
echo "Android NDK not found at ${ANDROID_NDK_DIR}"
echo "The Android NDK is needed to build a Clang whose -fsanitize=address"
echo "works on Android. See "
echo "http://code.google.com/p/chromium/wiki/AndroidBuildInstructions for how"
echo "to install the NDK, or pass --without-android."
exit 1
fi
# Check that cmake and ninja are available.
if ! which cmake > /dev/null; then
echo "CMake needed to build clang; please install"
exit 1
fi
if ! which ninja > /dev/null; then
echo "ninja needed to build clang, please install"
exit 1
fi
echo Reverting previously patched files
for i in \
"${CLANG_DIR}/test/Index/crash-recovery-modules.m" \
"${CLANG_DIR}/unittests/libclang/LibclangTest.cpp" \
"${COMPILER_RT_DIR}/lib/asan/asan_rtl.cc" \
"${COMPILER_RT_DIR}/test/asan/TestCases/Linux/new_array_cookie_test.cc" \
"${LLVM_DIR}/test/DebugInfo/gmlt.ll" \
"${LLVM_DIR}/lib/CodeGen/SpillPlacement.cpp" \
"${LLVM_DIR}/lib/CodeGen/SpillPlacement.h" \
"${LLVM_DIR}/lib/Transforms/Instrumentation/MemorySanitizer.cpp" \
"${CLANG_DIR}/test/Driver/env.c" \
"${CLANG_DIR}/lib/Frontend/InitPreprocessor.cpp" \
"${CLANG_DIR}/test/Frontend/exceptions.c" \
"${CLANG_DIR}/test/Preprocessor/predefined-exceptions.m" \
"${LLVM_DIR}/test/Bindings/Go/go.test" \
"${CLANG_DIR}/lib/Parse/ParseExpr.cpp" \
"${CLANG_DIR}/lib/Parse/ParseTemplate.cpp" \
"${CLANG_DIR}/lib/Sema/SemaDeclCXX.cpp" \
"${CLANG_DIR}/lib/Sema/SemaExprCXX.cpp" \
"${CLANG_DIR}/test/SemaCXX/default2.cpp" \
"${CLANG_DIR}/test/SemaCXX/typo-correction-delayed.cpp" \
"${COMPILER_RT_DIR}/lib/sanitizer_common/sanitizer_stoptheworld_linux_libcdep.cc" \
"${COMPILER_RT_DIR}/test/tsan/signal_segv_handler.cc" \
"${COMPILER_RT_DIR}/lib/sanitizer_common/sanitizer_coverage_libcdep.cc" \
"${COMPILER_RT_DIR}/cmake/config-ix.cmake" \
"${COMPILER_RT_DIR}/CMakeLists.txt" \
"${COMPILER_RT_DIR}/lib/ubsan/ubsan_platform.h" \
; do
if [[ -e "${i}" ]]; then
rm -f "${i}" # For unversioned files.
svn revert "${i}"
fi;
done
echo Remove the Clang tools shim dir
CHROME_TOOLS_SHIM_DIR=${ABS_LLVM_DIR}/tools/chrometools
rm -rfv ${CHROME_TOOLS_SHIM_DIR}
echo Getting LLVM r"${CLANG_REVISION}" in "${LLVM_DIR}"
if ! svn co --force "${LLVM_REPO_URL}/llvm/trunk@${CLANG_REVISION}" \
"${LLVM_DIR}"; then
echo Checkout failed, retrying
rm -rf "${LLVM_DIR}"
svn co --force "${LLVM_REPO_URL}/llvm/trunk@${CLANG_REVISION}" "${LLVM_DIR}"
fi
echo Getting clang r"${CLANG_REVISION}" in "${CLANG_DIR}"
svn co --force "${LLVM_REPO_URL}/cfe/trunk@${CLANG_REVISION}" "${CLANG_DIR}"
# We have moved from building compiler-rt in the LLVM tree, to a separate
# directory. Nuke any previous checkout to avoid building it.
rm -rf "${LLVM_DIR}/projects/compiler-rt"
echo Getting compiler-rt r"${CLANG_REVISION}" in "${COMPILER_RT_DIR}"
svn co --force "${LLVM_REPO_URL}/compiler-rt/trunk@${CLANG_REVISION}" \
"${COMPILER_RT_DIR}"
# clang needs a libc++ checkout, else -stdlib=libc++ won't find includes
# (i.e. this is needed for bootstrap builds).
if [ "${OS}" = "Darwin" ]; then
echo Getting libc++ r"${CLANG_REVISION}" in "${LIBCXX_DIR}"
svn co --force "${LLVM_REPO_URL}/libcxx/trunk@${CLANG_REVISION}" \
"${LIBCXX_DIR}"
fi
# While we're bundling our own libc++ on OS X, we need to compile libc++abi
# into it too (since OS X 10.6 doesn't have libc++abi.dylib either).
if [ "${OS}" = "Darwin" ]; then
echo Getting libc++abi r"${CLANG_REVISION}" in "${LIBCXXABI_DIR}"
svn co --force "${LLVM_REPO_URL}/libcxxabi/trunk@${CLANG_REVISION}" \
"${LIBCXXABI_DIR}"
fi
if [[ -n "$with_patches" ]]; then
# Apply patch for tests failing with --disable-pthreads (llvm.org/PR11974)
pushd "${CLANG_DIR}"
cat << 'EOF' |
--- test/Index/crash-recovery-modules.m (revision 202554)
+++ test/Index/crash-recovery-modules.m (working copy)
@@ -12,6 +12,8 @@
// REQUIRES: crash-recovery
// REQUIRES: shell
+// XFAIL: *
+// (PR11974)
@import Crash;
EOF
patch -p0
popd
pushd "${CLANG_DIR}"
cat << 'EOF' |
--- unittests/libclang/LibclangTest.cpp (revision 215949)
+++ unittests/libclang/LibclangTest.cpp (working copy)
@@ -431,7 +431,7 @@
EXPECT_EQ(0U, clang_getNumDiagnostics(ClangTU));
}
-TEST_F(LibclangReparseTest, ReparseWithModule) {
+TEST_F(LibclangReparseTest, DISABLED_ReparseWithModule) {
const char *HeaderTop = "#ifndef H\n#define H\nstruct Foo { int bar;";
const char *HeaderBottom = "\n};\n#endif\n";
const char *MFile = "#include \"HeaderFile.h\"\nint main() {"
EOF
patch -p0
popd
# This Go bindings test doesn't work after the bootstrap build on Linux. (PR21552)
pushd "${LLVM_DIR}"
cat << 'EOF' |
--- test/Bindings/Go/go.test (revision 223109)
+++ test/Bindings/Go/go.test (working copy)
@@ -1,3 +1,3 @@
-; RUN: llvm-go test llvm.org/llvm/bindings/go/llvm
+; RUN: true
; REQUIRES: shell
EOF
patch -p0
popd
# The UBSan run-time, which is now bundled with the ASan run-time, doesn't work
# on Mac OS X 10.8 (PR23539).
pushd "${COMPILER_RT_DIR}"
cat << 'EOF' |
Index: CMakeLists.txt
===================================================================
--- CMakeLists.txt (revision 241602)
+++ CMakeLists.txt (working copy)
@@ -305,6 +305,7 @@
list(APPEND SANITIZER_COMMON_SUPPORTED_OS iossim)
endif()
endif()
+ set(SANITIZER_MIN_OSX_VERSION "10.7")
if(SANITIZER_MIN_OSX_VERSION VERSION_LESS "10.7")
message(FATAL_ERROR "Too old OS X version: ${SANITIZER_MIN_OSX_VERSION}")
endif()
EOF
patch -p0
popd
fi
# Echo all commands.
set -x
# Set default values for CC and CXX if they're not set in the environment.
CC=${CC:-cc}
CXX=${CXX:-c++}
if [[ -n "${gcc_toolchain}" ]]; then
# Use the specified gcc installation for building.
CC="$gcc_toolchain/bin/gcc"
CXX="$gcc_toolchain/bin/g++"
# Set LD_LIBRARY_PATH to make auxiliary targets (tablegen, bootstrap compiler,
# etc.) find the .so.
export LD_LIBRARY_PATH="$(dirname $(${CXX} -print-file-name=libstdc++.so.6))"
fi
CFLAGS=""
CXXFLAGS=""
LDFLAGS=""
# LLVM uses C++11 starting in llvm 3.5. On Linux, this means libstdc++4.7+ is
# needed, on OS X it requires libc++. clang only automatically links to libc++
# when targeting OS X 10.9+, so add stdlib=libc++ explicitly so clang can run on
# OS X versions as old as 10.7.
# TODO(thakis): Some bots are still on 10.6 (nacl...), so for now bundle
# libc++.dylib. Remove this once all bots are on 10.7+, then use
# -DLLVM_ENABLE_LIBCXX=ON and change deployment_target to 10.7.
deployment_target=""
if [ "${OS}" = "Darwin" ]; then
# When building on 10.9, /usr/include usually doesn't exist, and while
# Xcode's clang automatically sets a sysroot, self-built clangs don't.
CFLAGS="-isysroot $(xcrun --show-sdk-path)"
CXXFLAGS="-stdlib=libc++ -nostdinc++ -I${ABS_LIBCXX_DIR}/include ${CFLAGS}"
if [[ -n "${bootstrap}" ]]; then
deployment_target=10.6
fi
fi
# Build bootstrap clang if requested.
if [[ -n "${bootstrap}" ]]; then
ABS_INSTALL_DIR="${PWD}/${LLVM_BOOTSTRAP_INSTALL_DIR}"
echo "Building bootstrap compiler"
mkdir -p "${LLVM_BOOTSTRAP_DIR}"
pushd "${LLVM_BOOTSTRAP_DIR}"
cmake -GNinja \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DLLVM_TARGETS_TO_BUILD=host \
-DLLVM_ENABLE_THREADS=OFF \
-DCMAKE_INSTALL_PREFIX="${ABS_INSTALL_DIR}" \
-DCMAKE_C_COMPILER="${CC}" \
-DCMAKE_CXX_COMPILER="${CXX}" \
-DCMAKE_C_FLAGS="${CFLAGS}" \
-DCMAKE_CXX_FLAGS="${CXXFLAGS}" \
../llvm
ninja
if [[ -n "${run_tests}" ]]; then
ninja check-all
fi
ninja install
if [[ -n "${gcc_toolchain}" ]]; then
# Copy that gcc's stdlibc++.so.6 to the build dir, so the bootstrap
# compiler can start.
cp -v "$(${CXX} -print-file-name=libstdc++.so.6)" \
"${ABS_INSTALL_DIR}/lib/"
fi
popd
CC="${ABS_INSTALL_DIR}/bin/clang"
CXX="${ABS_INSTALL_DIR}/bin/clang++"
if [[ -n "${gcc_toolchain}" ]]; then
# Tell the bootstrap compiler to use a specific gcc prefix to search
# for standard library headers and shared object file.
CFLAGS="--gcc-toolchain=${gcc_toolchain}"
CXXFLAGS="--gcc-toolchain=${gcc_toolchain}"
fi
echo "Building final compiler"
fi
# Build clang (in a separate directory).
# The clang bots have this path hardcoded in built/scripts/slave/compile.py,
# so if you change it you also need to change these links.
mkdir -p "${LLVM_BUILD_DIR}"
pushd "${LLVM_BUILD_DIR}"
# Build libc++.dylib while some bots are still on OS X 10.6.
if [ "${OS}" = "Darwin" ]; then
rm -rf libcxxbuild
LIBCXXFLAGS="-O3 -std=c++11 -fstrict-aliasing"
# libcxx and libcxxabi both have a file stdexcept.cpp, so put their .o files
# into different subdirectories.
mkdir -p libcxxbuild/libcxx
pushd libcxxbuild/libcxx
${CXX:-c++} -c ${CXXFLAGS} ${LIBCXXFLAGS} "${ABS_LIBCXX_DIR}"/src/*.cpp
popd
mkdir -p libcxxbuild/libcxxabi
pushd libcxxbuild/libcxxabi
${CXX:-c++} -c ${CXXFLAGS} ${LIBCXXFLAGS} "${ABS_LIBCXXABI_DIR}"/src/*.cpp -I"${ABS_LIBCXXABI_DIR}/include"
popd
pushd libcxxbuild
${CC:-cc} libcxx/*.o libcxxabi/*.o -o libc++.1.dylib -dynamiclib \
-nodefaultlibs -current_version 1 -compatibility_version 1 \
-lSystem -install_name @executable_path/libc++.dylib \
-Wl,-unexported_symbols_list,${ABS_LIBCXX_DIR}/lib/libc++unexp.exp \
-Wl,-force_symbols_not_weak_list,${ABS_LIBCXX_DIR}/lib/notweak.exp \
-Wl,-force_symbols_weak_list,${ABS_LIBCXX_DIR}/lib/weak.exp
ln -sf libc++.1.dylib libc++.dylib
popd
LDFLAGS+="-stdlib=libc++ -L${PWD}/libcxxbuild"
if [[ -n "${bootstrap}" ]]; then
# Now that the libc++ headers have been installed and libc++.dylib is built,
# delete the libc++ checkout again so that it's not part of the main
# build below -- the libc++(abi) tests don't pass on OS X in bootstrap
# builds (http://llvm.org/PR24068)
rm -rf "${ABS_LIBCXX_DIR}"
rm -rf "${ABS_LIBCXXABI_DIR}"
CXXFLAGS="-stdlib=libc++ -nostdinc++ -I${ABS_INSTALL_DIR}/include/c++/v1 ${CFLAGS}"
fi
fi
# Find the binutils include dir for the gold plugin.
BINUTILS_INCDIR=""
if [ "${OS}" = "Linux" ]; then
BINUTILS_INCDIR="${ABS_BINUTILS_DIR}/Linux_x64/Release/include"
fi
# If building at head, define a macro that plugins can use for #ifdefing
# out code that builds at head, but not at CLANG_REVISION or vice versa.
if [[ -n ${LLVM_FORCE_HEAD_REVISION:-''} ]]; then
CFLAGS="${CFLAGS} -DLLVM_FORCE_HEAD_REVISION"
CXXFLAGS="${CXXFLAGS} -DLLVM_FORCE_HEAD_REVISION"
fi
# Hook the Chromium tools into the LLVM build. Several Chromium tools have
# dependencies on LLVM/Clang libraries. The LLVM build detects implicit tools
# in the tools subdirectory, so install a shim CMakeLists.txt that forwards to
# the real directory for the Chromium tools.
# Note that the shim directory name intentionally has no _ or _. The implicit
# tool detection logic munges them in a weird way.
mkdir -v ${CHROME_TOOLS_SHIM_DIR}
cat > ${CHROME_TOOLS_SHIM_DIR}/CMakeLists.txt << EOF
# Since tools/clang isn't actually a subdirectory, use the two argument version
# to specify where build artifacts go. CMake doesn't allow reusing the same
# binary dir for multiple source dirs, so the build artifacts have to go into a
# subdirectory...
add_subdirectory(\${CHROMIUM_TOOLS_SRC} \${CMAKE_CURRENT_BINARY_DIR}/a)
EOF
rm -fv CMakeCache.txt
MACOSX_DEPLOYMENT_TARGET=${deployment_target} cmake -GNinja \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DLLVM_ENABLE_THREADS=OFF \
-DLLVM_BINUTILS_INCDIR="${BINUTILS_INCDIR}" \
-DCMAKE_C_COMPILER="${CC}" \
-DCMAKE_CXX_COMPILER="${CXX}" \
-DCMAKE_C_FLAGS="${CFLAGS}" \
-DCMAKE_CXX_FLAGS="${CXXFLAGS}" \
-DCMAKE_EXE_LINKER_FLAGS="${LDFLAGS}" \
-DCMAKE_SHARED_LINKER_FLAGS="${LDFLAGS}" \
-DCMAKE_MODULE_LINKER_FLAGS="${LDFLAGS}" \
-DCMAKE_INSTALL_PREFIX="${ABS_LLVM_BUILD_DIR}" \
-DCHROMIUM_TOOLS_SRC="${ABS_CHROMIUM_TOOLS_DIR}" \
-DCHROMIUM_TOOLS="${chrome_tools}" \
"${ABS_LLVM_DIR}"
env
if [[ -n "${gcc_toolchain}" ]]; then
# Copy in the right stdlibc++.so.6 so clang can start.
mkdir -p lib
cp -v "$(${CXX} ${CXXFLAGS} -print-file-name=libstdc++.so.6)" lib/
fi
ninja
# If any Chromium tools were built, install those now.
if [[ -n "${chrome_tools}" ]]; then
ninja cr-install
fi
STRIP_FLAGS=
if [ "${OS}" = "Darwin" ]; then
# See http://crbug.com/256342
STRIP_FLAGS=-x
cp libcxxbuild/libc++.1.dylib bin/
fi
strip ${STRIP_FLAGS} bin/clang
popd
# Build compiler-rt out-of-tree.
mkdir -p "${COMPILER_RT_BUILD_DIR}"
pushd "${COMPILER_RT_BUILD_DIR}"
rm -fv CMakeCache.txt
MACOSX_DEPLOYMENT_TARGET=${deployment_target} cmake -GNinja \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DLLVM_ENABLE_THREADS=OFF \
-DCMAKE_C_COMPILER="${CC}" \
-DCMAKE_CXX_COMPILER="${CXX}" \
-DLLVM_CONFIG_PATH="${ABS_LLVM_BUILD_DIR}/bin/llvm-config" \
"${ABS_COMPILER_RT_DIR}"
ninja
# Copy selected output to the main tree.
# Darwin doesn't support cp --parents, so pipe through tar instead.
CLANG_VERSION=$("${ABS_LLVM_BUILD_DIR}/bin/clang" --version | \
sed -ne 's/clang version \([0-9]\.[0-9]\.[0-9]\).*/\1/p')
ABS_LLVM_CLANG_LIB_DIR="${ABS_LLVM_BUILD_DIR}/lib/clang/${CLANG_VERSION}"
tar -c *blacklist.txt | tar -C ${ABS_LLVM_CLANG_LIB_DIR} -xv
tar -c include/sanitizer | tar -C ${ABS_LLVM_CLANG_LIB_DIR} -xv
if [[ "${OS}" = "Darwin" ]]; then
tar -c lib/darwin | tar -C ${ABS_LLVM_CLANG_LIB_DIR} -xv
else
tar -c lib/linux | tar -C ${ABS_LLVM_CLANG_LIB_DIR} -xv
fi
popd
if [[ -n "${with_android}" ]]; then
# Make a standalone Android toolchain.
${ANDROID_NDK_DIR}/build/tools/make-standalone-toolchain.sh \
--platform=android-19 \
--install-dir="${LLVM_BUILD_DIR}/android-toolchain" \
--system=linux-x86_64 \
--stl=stlport \
--toolchain=arm-linux-androideabi-4.9
# Android NDK r9d copies a broken unwind.h into the toolchain, see
# http://crbug.com/357890
rm -v "${LLVM_BUILD_DIR}"/android-toolchain/include/c++/*/unwind.h
# Build ASan runtime for Android in a separate build tree.
mkdir -p ${LLVM_BUILD_DIR}/android
pushd ${LLVM_BUILD_DIR}/android
rm -fv CMakeCache.txt
MACOSX_DEPLOYMENT_TARGET=${deployment_target} cmake -GNinja \
-DCMAKE_BUILD_TYPE=Release \
-DLLVM_ENABLE_ASSERTIONS=ON \
-DLLVM_ENABLE_THREADS=OFF \
-DCMAKE_C_COMPILER=${PWD}/../bin/clang \
-DCMAKE_CXX_COMPILER=${PWD}/../bin/clang++ \
-DLLVM_CONFIG_PATH=${PWD}/../bin/llvm-config \
-DCMAKE_C_FLAGS="--target=arm-linux-androideabi --sysroot=${PWD}/../android-toolchain/sysroot -B${PWD}/../android-toolchain" \
-DCMAKE_CXX_FLAGS="--target=arm-linux-androideabi --sysroot=${PWD}/../android-toolchain/sysroot -B${PWD}/../android-toolchain" \
-DANDROID=1 \
"${ABS_COMPILER_RT_DIR}"
ninja libclang_rt.asan-arm-android.so
# And copy it into the main build tree.
cp "$(find -name libclang_rt.asan-arm-android.so)" "${ABS_LLVM_CLANG_LIB_DIR}/lib/linux/"
popd
fi
if [[ -n "$run_tests" || -n "${LLVM_FORCE_HEAD_REVISION:-''}" ]]; then
# Run Chrome tool tests.
ninja -C "${LLVM_BUILD_DIR}" cr-check-all
fi
if [[ -n "$run_tests" ]]; then
# Run the LLVM and Clang tests.
ninja -C "${LLVM_BUILD_DIR}" check-all
fi
# After everything is done, log success for this revision.
echo "${PACKAGE_VERSION}" > "${STAMP_FILE}"

View file

@ -165,8 +165,10 @@ def to_gn_args(args, mode, arch, target_os):
# TODO(zra): Investigate using clang with these configurations.
# Clang compiles tcmalloc's inline assembly for ia32 on Linux wrong, so we
# don't use clang in that configuration. Thus, we use gcc for ia32 *unless*
# a clang-based sanitizer is specified.
# asan or tsan is specified.
has_clang = (host_os != 'win'
and args.os not in ['android']
and not gn_args['target_cpu'].startswith('arm')
and not gn_args['target_cpu'].startswith('mips')
and not ((gn_args['target_os'] == 'linux')
and (gn_args['host_cpu'] == 'x86')
@ -246,7 +248,7 @@ def process_options(args):
if os_name != 'android':
print "Unsupported target os %s" % os_name
return False
if not HOST_OS in ['linux', 'macos']:
if not HOST_OS in ['linux']:
print ("Cross-compilation to %s is not supported on host os %s."
% (os_name, HOST_OS))
return False
@ -365,8 +367,7 @@ def parse_args(args):
type=int,
help='Number of simultaneous GN invocations',
dest='workers',
# Set to multiprocessing.cpu_count() when GN can be run in parallel.
default=1)
default=multiprocessing.cpu_count())
options = parser.parse_args(args)
if not process_options(options):

View file

@ -90,7 +90,7 @@ def ProcessOptions(options, args):
if os_name != 'android':
print "Unsupported target os %s" % os_name
return False
if not HOST_OS in ['linux', 'macos']:
if not HOST_OS in ['linux']:
print ("Cross-compilation to %s is not supported on host os %s."
% (os_name, HOST_OS))
return False