2021-04-15 10:10:20 +00:00
|
|
|
#!/usr/bin/env python3
|
2011-10-05 05:34:19 +00:00
|
|
|
#
|
2018-04-17 17:05:27 +00:00
|
|
|
# Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
|
2011-10-05 05:34:19 +00:00
|
|
|
# for details. All rights reserved. Use of this source code is governed by a
|
|
|
|
# BSD-style license that can be found in the LICENSE file.
|
2012-08-29 20:42:28 +00:00
|
|
|
|
2020-07-14 14:10:11 +00:00
|
|
|
import argparse
|
2020-02-25 10:32:01 +00:00
|
|
|
import io
|
|
|
|
import json
|
2018-04-17 17:05:27 +00:00
|
|
|
import os
|
|
|
|
import subprocess
|
2017-08-15 04:54:34 +00:00
|
|
|
import sys
|
2018-04-17 17:05:27 +00:00
|
|
|
import time
|
|
|
|
import utils
|
|
|
|
|
2020-07-14 14:10:11 +00:00
|
|
|
import gn as gn_py
|
|
|
|
|
2018-04-17 17:05:27 +00:00
|
|
|
HOST_OS = utils.GuessOS()
|
|
|
|
HOST_CPUS = utils.GuessCpus()
|
|
|
|
SCRIPT_DIR = os.path.dirname(sys.argv[0])
|
|
|
|
DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..'))
|
2020-02-07 08:26:35 +00:00
|
|
|
AVAILABLE_ARCHS = utils.ARCH_FAMILY.keys()
|
2018-04-17 17:05:27 +00:00
|
|
|
|
|
|
|
usage = """\
|
|
|
|
usage: %%prog [options] [targets]
|
|
|
|
|
|
|
|
This script invokes ninja to build Dart.
|
|
|
|
"""
|
|
|
|
|
|
|
|
def BuildOptions():
|
2020-07-14 14:10:11 +00:00
|
|
|
parser = argparse.ArgumentParser(
|
2021-01-11 09:58:36 +00:00
|
|
|
description='Runs GN (if necessary) followed by ninja',
|
2020-07-14 14:10:11 +00:00
|
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
|
|
|
|
|
|
|
|
config_group = parser.add_argument_group('Configuration Related Arguments')
|
|
|
|
gn_py.AddCommonConfigurationArgs(config_group)
|
|
|
|
|
|
|
|
gn_group = parser.add_argument_group('GN Related Arguments')
|
|
|
|
gn_py.AddCommonGnOptionArgs(gn_group)
|
|
|
|
|
|
|
|
other_group = parser.add_argument_group('Other Arguments')
|
|
|
|
gn_py.AddOtherArgs(other_group)
|
|
|
|
|
|
|
|
other_group.add_argument("-j",
|
|
|
|
type=int,
|
2024-04-02 22:54:19 +00:00
|
|
|
help='Ninja -j option for RBE builds.',
|
2024-02-29 10:33:09 +00:00
|
|
|
default=200 if sys.platform == 'win32' else 1000)
|
2020-07-14 14:10:11 +00:00
|
|
|
other_group.add_argument("-l",
|
|
|
|
type=int,
|
2024-04-02 22:54:19 +00:00
|
|
|
help='Ninja -l option for RBE builds.',
|
2020-07-14 14:10:11 +00:00
|
|
|
default=64)
|
2023-10-24 14:36:19 +00:00
|
|
|
other_group.add_argument("--no-start-rbe",
|
|
|
|
help="Don't try to start rbe",
|
|
|
|
default=False,
|
2020-07-14 14:10:11 +00:00
|
|
|
action='store_true')
|
2021-05-26 17:48:30 +00:00
|
|
|
other_group.add_argument(
|
|
|
|
"--check-clean",
|
|
|
|
help="Check that a second invocation of Ninja has nothing to do",
|
|
|
|
default=False,
|
|
|
|
action='store_true')
|
2020-07-14 14:10:11 +00:00
|
|
|
|
|
|
|
parser.add_argument('build_targets', nargs='*')
|
|
|
|
|
|
|
|
return parser
|
2018-04-17 17:05:27 +00:00
|
|
|
|
|
|
|
|
|
|
|
def NotifyBuildDone(build_config, success, start):
|
2019-08-05 20:34:31 +00:00
|
|
|
if not success:
|
|
|
|
print("BUILD FAILED")
|
|
|
|
|
|
|
|
sys.stdout.flush()
|
|
|
|
|
|
|
|
# Display a notification if build time exceeded DART_BUILD_NOTIFICATION_DELAY.
|
|
|
|
notification_delay = float(
|
|
|
|
os.getenv('DART_BUILD_NOTIFICATION_DELAY', sys.float_info.max))
|
|
|
|
if (time.time() - start) < notification_delay:
|
|
|
|
return
|
|
|
|
|
2018-04-17 17:05:27 +00:00
|
|
|
if success:
|
2019-08-05 20:34:31 +00:00
|
|
|
message = 'Build succeeded.'
|
2018-04-17 17:05:27 +00:00
|
|
|
else:
|
2019-08-05 20:34:31 +00:00
|
|
|
message = 'Build failed.'
|
|
|
|
title = build_config
|
|
|
|
|
|
|
|
command = None
|
|
|
|
if HOST_OS == 'macos':
|
|
|
|
# Use AppleScript to display a UI non-modal notification.
|
|
|
|
script = 'display notification "%s" with title "%s" sound name "Glass"' % (
|
|
|
|
message, title)
|
|
|
|
command = "osascript -e '%s' &" % script
|
|
|
|
elif HOST_OS == 'linux':
|
|
|
|
if success:
|
|
|
|
icon = 'dialog-information'
|
|
|
|
else:
|
|
|
|
icon = 'dialog-error'
|
|
|
|
command = "notify-send -i '%s' '%s' '%s' &" % (icon, message, title)
|
|
|
|
elif HOST_OS == 'win32':
|
|
|
|
if success:
|
|
|
|
icon = 'info'
|
|
|
|
else:
|
|
|
|
icon = 'error'
|
|
|
|
command = (
|
|
|
|
"powershell -command \""
|
|
|
|
"[reflection.assembly]::loadwithpartialname('System.Windows.Forms')"
|
|
|
|
"| Out-Null;"
|
|
|
|
"[reflection.assembly]::loadwithpartialname('System.Drawing')"
|
|
|
|
"| Out-Null;"
|
|
|
|
"$n = new-object system.windows.forms.notifyicon;"
|
|
|
|
"$n.icon = [system.drawing.systemicons]::information;"
|
|
|
|
"$n.visible = $true;"
|
|
|
|
"$n.showballoontip(%d, '%s', '%s', "
|
|
|
|
"[system.windows.forms.tooltipicon]::%s);\"") % (
|
|
|
|
5000, # Notification stays on for this many milliseconds
|
|
|
|
message,
|
|
|
|
title,
|
|
|
|
icon)
|
|
|
|
|
|
|
|
if command:
|
|
|
|
# Ignore return code, if this command fails, it doesn't matter.
|
|
|
|
os.system(command)
|
2018-04-17 17:05:27 +00:00
|
|
|
|
|
|
|
|
2023-10-24 14:36:19 +00:00
|
|
|
def UseRBE(out_dir):
|
|
|
|
args_gn = os.path.join(out_dir, 'args.gn')
|
|
|
|
return 'use_rbe = true' in open(args_gn, 'r').read()
|
|
|
|
|
|
|
|
|
|
|
|
# Try to start RBE, but don't bail out if we can't. Instead print an error
|
2018-04-17 17:05:27 +00:00
|
|
|
# message, and let the build fail with its own error messages as well.
|
2024-04-02 22:54:19 +00:00
|
|
|
rbe_started = False
|
2023-10-24 14:36:19 +00:00
|
|
|
bootstrap_path = None
|
2019-08-05 20:34:31 +00:00
|
|
|
|
|
|
|
|
2024-04-02 22:54:19 +00:00
|
|
|
def StartRBE(out_dir, env):
|
2023-10-24 14:36:19 +00:00
|
|
|
global rbe_started, bootstrap_path
|
2024-04-03 09:28:30 +00:00
|
|
|
if not rbe_started:
|
|
|
|
rbe_dir = 'buildtools/reclient'
|
|
|
|
with open(os.path.join(out_dir, 'args.gn'), 'r') as fp:
|
|
|
|
for line in fp:
|
|
|
|
if 'rbe_dir' in line:
|
|
|
|
words = line.split()
|
|
|
|
rbe_dir = words[2][1:-1] # rbe_dir = "/path/to/rbe"
|
|
|
|
bootstrap_path = os.path.join(rbe_dir, 'bootstrap')
|
|
|
|
bootstrap_command = [bootstrap_path]
|
|
|
|
process = subprocess.Popen(bootstrap_command, env=env)
|
|
|
|
process.wait()
|
|
|
|
if process.returncode != 0:
|
|
|
|
print('Failed to start RBE')
|
|
|
|
return False
|
|
|
|
rbe_started = True
|
2018-04-17 17:05:27 +00:00
|
|
|
return True
|
|
|
|
|
2023-10-24 14:36:19 +00:00
|
|
|
|
2024-03-05 13:47:17 +00:00
|
|
|
def StopRBE(env):
|
2023-10-24 14:36:19 +00:00
|
|
|
global rbe_started, bootstrap_path
|
2024-04-03 09:28:30 +00:00
|
|
|
if rbe_started:
|
|
|
|
bootstrap_command = [bootstrap_path, '--shutdown']
|
|
|
|
process = subprocess.Popen(bootstrap_command, env=env)
|
|
|
|
process.wait()
|
|
|
|
rbe_started = False
|
2023-10-24 14:36:19 +00:00
|
|
|
|
|
|
|
|
|
|
|
# Returns a tuple (build_config, command to run, whether rbe is used)
|
2024-03-04 14:54:26 +00:00
|
|
|
def BuildOneConfig(options, targets, target_os, mode, arch, sanitizer, env):
|
2020-06-16 23:37:36 +00:00
|
|
|
build_config = utils.GetBuildConf(mode, arch, target_os, sanitizer)
|
|
|
|
out_dir = utils.GetBuildRoot(HOST_OS, mode, arch, target_os, sanitizer)
|
2023-10-24 14:36:19 +00:00
|
|
|
using_rbe = False
|
2022-09-27 11:24:03 +00:00
|
|
|
command = ['buildtools/ninja/ninja', '-C', out_dir]
|
2019-08-05 20:34:31 +00:00
|
|
|
if options.verbose:
|
|
|
|
command += ['-v']
|
2024-04-02 22:54:19 +00:00
|
|
|
if UseRBE(out_dir):
|
|
|
|
if options.no_start_rbe or StartRBE(out_dir, env):
|
2023-10-24 14:36:19 +00:00
|
|
|
using_rbe = True
|
2019-08-05 20:34:31 +00:00
|
|
|
command += [('-j%s' % str(options.j))]
|
|
|
|
command += [('-l%s' % str(options.l))]
|
|
|
|
else:
|
2024-03-26 11:10:48 +00:00
|
|
|
exit(1)
|
2019-08-05 20:34:31 +00:00
|
|
|
command += targets
|
2023-10-24 14:36:19 +00:00
|
|
|
return (build_config, command, using_rbe)
|
2018-04-17 17:05:27 +00:00
|
|
|
|
|
|
|
|
2020-02-25 10:32:01 +00:00
|
|
|
def RunOneBuildCommand(build_config, args, env):
|
2019-08-05 20:34:31 +00:00
|
|
|
start_time = time.time()
|
|
|
|
print(' '.join(args))
|
2020-02-25 10:32:01 +00:00
|
|
|
process = subprocess.Popen(args, env=env, stdin=None)
|
2019-08-05 20:34:31 +00:00
|
|
|
process.wait()
|
|
|
|
if process.returncode != 0:
|
|
|
|
NotifyBuildDone(build_config, success=False, start=start_time)
|
|
|
|
return 1
|
|
|
|
else:
|
|
|
|
NotifyBuildDone(build_config, success=True, start=start_time)
|
2018-04-17 17:05:27 +00:00
|
|
|
|
2019-08-05 20:34:31 +00:00
|
|
|
return 0
|
2018-04-17 17:05:27 +00:00
|
|
|
|
|
|
|
|
2021-05-26 17:48:30 +00:00
|
|
|
def CheckCleanBuild(build_config, args, env):
|
|
|
|
args = args + ['-n', '-d', 'explain']
|
|
|
|
print(' '.join(args))
|
|
|
|
process = subprocess.Popen(args,
|
|
|
|
env=env,
|
|
|
|
stdout=subprocess.PIPE,
|
|
|
|
stderr=subprocess.PIPE,
|
|
|
|
stdin=None)
|
|
|
|
out, err = process.communicate()
|
|
|
|
process.wait()
|
|
|
|
if process.returncode != 0:
|
|
|
|
return 1
|
|
|
|
if 'ninja: no work to do' not in out.decode('utf-8'):
|
|
|
|
print(err.decode('utf-8'))
|
|
|
|
return 1
|
|
|
|
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
2020-02-25 10:32:01 +00:00
|
|
|
def SanitizerEnvironmentVariables():
|
|
|
|
with io.open('tools/bots/test_matrix.json', encoding='utf-8') as fd:
|
|
|
|
config = json.loads(fd.read())
|
|
|
|
env = dict()
|
|
|
|
for k, v in config['sanitizer_options'].items():
|
|
|
|
env[str(k)] = str(v)
|
|
|
|
symbolizer_path = config['sanitizer_symbolizer'].get(HOST_OS, None)
|
|
|
|
if symbolizer_path:
|
|
|
|
symbolizer_path = str(os.path.join(DART_ROOT, symbolizer_path))
|
|
|
|
env['ASAN_SYMBOLIZER_PATH'] = symbolizer_path
|
2020-05-12 09:03:59 +00:00
|
|
|
env['LSAN_SYMBOLIZER_PATH'] = symbolizer_path
|
2020-02-25 10:32:01 +00:00
|
|
|
env['MSAN_SYMBOLIZER_PATH'] = symbolizer_path
|
|
|
|
env['TSAN_SYMBOLIZER_PATH'] = symbolizer_path
|
2020-05-12 09:03:59 +00:00
|
|
|
env['UBSAN_SYMBOLIZER_PATH'] = symbolizer_path
|
2020-02-25 10:32:01 +00:00
|
|
|
return env
|
|
|
|
|
|
|
|
|
2023-10-24 14:36:19 +00:00
|
|
|
def Build(configs, env, options):
|
|
|
|
# Build regular configs.
|
|
|
|
rbe_builds = []
|
|
|
|
for (build_config, args, rbe) in configs:
|
|
|
|
if args is None:
|
|
|
|
return 1
|
|
|
|
if rbe:
|
|
|
|
rbe_builds.append([env, args])
|
|
|
|
elif RunOneBuildCommand(build_config, args, env=env) != 0:
|
|
|
|
return 1
|
|
|
|
|
|
|
|
# Run RBE builds in parallel.
|
|
|
|
active_rbe_builds = []
|
|
|
|
for (env, args) in rbe_builds:
|
|
|
|
print(' '.join(args))
|
|
|
|
process = subprocess.Popen(args, env=env)
|
|
|
|
active_rbe_builds.append([args, process])
|
|
|
|
while active_rbe_builds:
|
|
|
|
time.sleep(0.1)
|
|
|
|
for rbe_build in active_rbe_builds:
|
|
|
|
(args, process) = rbe_build
|
|
|
|
if process.poll() is not None:
|
|
|
|
print(' '.join(args) + " done.")
|
|
|
|
active_rbe_builds.remove(rbe_build)
|
|
|
|
if process.returncode != 0:
|
|
|
|
for (_, to_kill) in active_rbe_builds:
|
|
|
|
to_kill.terminate()
|
|
|
|
return 1
|
|
|
|
|
|
|
|
if options.check_clean:
|
|
|
|
for (build_config, args, rbe) in configs:
|
|
|
|
if CheckCleanBuild(build_config, args, env=env) != 0:
|
|
|
|
return 1
|
|
|
|
|
|
|
|
return 0
|
|
|
|
|
|
|
|
|
2018-04-17 17:05:27 +00:00
|
|
|
def Main():
|
2019-08-05 20:34:31 +00:00
|
|
|
starttime = time.time()
|
|
|
|
# Parse the options.
|
|
|
|
parser = BuildOptions()
|
2020-07-14 14:10:11 +00:00
|
|
|
options = parser.parse_args()
|
|
|
|
|
|
|
|
targets = options.build_targets
|
|
|
|
|
|
|
|
if not gn_py.ProcessOptions(options):
|
2019-08-05 20:34:31 +00:00
|
|
|
parser.print_help()
|
|
|
|
return 1
|
|
|
|
|
2020-02-25 10:32:01 +00:00
|
|
|
# If binaries are built with sanitizers we should use those flags.
|
|
|
|
# If the binaries are not built with sanitizers the flag should have no
|
|
|
|
# effect.
|
|
|
|
env = dict(os.environ)
|
|
|
|
env.update(SanitizerEnvironmentVariables())
|
|
|
|
|
2023-05-31 08:51:22 +00:00
|
|
|
# macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly.
|
|
|
|
#
|
|
|
|
# See:
|
|
|
|
#
|
|
|
|
# * https://openradar.appspot.com/radar?id=5608755232243712
|
|
|
|
# * https://github.com/dart-lang/sdk/issues/52411
|
|
|
|
#
|
|
|
|
# Remove these environment variables to avoid affecting clang's behaviors.
|
|
|
|
if sys.platform == 'darwin':
|
|
|
|
env.pop('CPATH', None)
|
|
|
|
env.pop('LIBRARY_PATH', None)
|
|
|
|
env.pop('SDKROOT', None)
|
|
|
|
|
2020-07-14 14:10:11 +00:00
|
|
|
# Always run GN before building.
|
2024-03-05 13:22:39 +00:00
|
|
|
gn_py.RunGnOnConfiguredConfigurations(options, env)
|
2020-07-14 14:10:11 +00:00
|
|
|
|
2019-08-05 20:34:31 +00:00
|
|
|
# Build all targets for each requested configuration.
|
|
|
|
configs = []
|
|
|
|
for target_os in options.os:
|
|
|
|
for mode in options.mode:
|
|
|
|
for arch in options.arch:
|
2019-12-11 18:12:47 +00:00
|
|
|
for sanitizer in options.sanitizer:
|
|
|
|
configs.append(
|
|
|
|
BuildOneConfig(options, targets, target_os, mode, arch,
|
2024-03-04 14:54:26 +00:00
|
|
|
sanitizer, env))
|
2019-08-05 20:34:31 +00:00
|
|
|
|
2023-10-24 14:36:19 +00:00
|
|
|
exit_code = Build(configs, env, options)
|
2019-08-05 20:34:31 +00:00
|
|
|
|
2024-04-12 22:06:32 +00:00
|
|
|
endtime = time.time()
|
|
|
|
|
2024-03-05 13:47:17 +00:00
|
|
|
StopRBE(env)
|
2021-05-26 17:48:30 +00:00
|
|
|
|
2023-10-24 14:36:19 +00:00
|
|
|
if exit_code == 0:
|
|
|
|
print("The build took %.3f seconds" % (endtime - starttime))
|
|
|
|
return exit_code
|
2018-04-17 17:05:27 +00:00
|
|
|
|
2011-10-05 05:34:19 +00:00
|
|
|
|
|
|
|
if __name__ == '__main__':
|
2019-08-05 20:34:31 +00:00
|
|
|
sys.exit(Main())
|