mirror of
https://github.com/dart-lang/sdk
synced 2024-11-05 18:22:09 +00:00
767943850f
Running tools/test.py locally behaves differently than on the bots because the bots will have special environment variables set. => We want also local runs of tools/test.py to report ASAN errors, so this CL moves the setting of environment variables to tools/test.py Furthermore we enable "additional check_initialization_order=true" asan option to detect ordering issues when initializing global state. Make the IsolateGroup::isolate_group_random_ heap allocated on VM startup to avoid initialization ordering issue (turns out our `Random` can depend on embedder entropy source callback being available) Also add locking around assigning of isolate group ids -- without it we could have two threads racing and possibly getting the same id. See b/149978682 Change-Id: I41b9a34c66934dcc0d3804233f5acf09004ba3bd Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/136630 Commit-Queue: Martin Kustermann <kustermann@google.com> Reviewed-by: Ryan Macnak <rmacnak@google.com> Reviewed-by: Alexander Thomas <athom@google.com> Reviewed-by: Alexander Aprelev <aam@google.com>
414 lines
13 KiB
Python
Executable file
414 lines
13 KiB
Python
Executable file
#!/usr/bin/env python
|
|
#
|
|
# Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
|
|
# for details. All rights reserved. Use of this source code is governed by a
|
|
# BSD-style license that can be found in the LICENSE file.
|
|
|
|
import io
|
|
import json
|
|
import multiprocessing
|
|
import optparse
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import utils
|
|
|
|
HOST_OS = utils.GuessOS()
|
|
HOST_CPUS = utils.GuessCpus()
|
|
SCRIPT_DIR = os.path.dirname(sys.argv[0])
|
|
DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..'))
|
|
AVAILABLE_ARCHS = utils.ARCH_FAMILY.keys()
|
|
|
|
usage = """\
|
|
usage: %%prog [options] [targets]
|
|
|
|
This script invokes ninja to build Dart.
|
|
"""
|
|
|
|
|
|
def BuildOptions():
|
|
result = optparse.OptionParser(usage=usage)
|
|
result.add_option(
|
|
"-a",
|
|
"--arch",
|
|
help='Target architectures (comma-separated).',
|
|
metavar='[all,' + ','.join(AVAILABLE_ARCHS) + ']',
|
|
default=utils.GuessArchitecture())
|
|
result.add_option(
|
|
"-b",
|
|
"--bytecode",
|
|
help='Build with the kernel bytecode interpreter. DEPRECATED.',
|
|
default=False,
|
|
action='store_true')
|
|
result.add_option(
|
|
"-j", type=int, help='Ninja -j option for Goma builds.', default=1000)
|
|
result.add_option(
|
|
"-l", type=int, help='Ninja -l option for Goma builds.', default=64)
|
|
result.add_option(
|
|
"-m",
|
|
"--mode",
|
|
help='Build variants (comma-separated).',
|
|
metavar='[all,debug,release,product]',
|
|
default='debug')
|
|
result.add_option(
|
|
"--no-start-goma",
|
|
help="Don't try to start goma",
|
|
default=False,
|
|
action='store_true')
|
|
result.add_option(
|
|
"--os",
|
|
help='Target OSs (comma-separated).',
|
|
metavar='[all,host,android]',
|
|
default='host')
|
|
result.add_option(
|
|
"--sanitizer",
|
|
type=str,
|
|
help='Build variants (comma-separated).',
|
|
metavar='[all,none,asan,lsan,msan,tsan,ubsan]',
|
|
default='none')
|
|
# TODO(38701): Remove this and everything that references it once the
|
|
# forked NNBD SDK is merged back in.
|
|
result.add_option(
|
|
"--nnbd",
|
|
help='Use the NNBD fork of the SDK.',
|
|
default=False,
|
|
action='store_true')
|
|
result.add_option(
|
|
"-v",
|
|
"--verbose",
|
|
help='Verbose output.',
|
|
default=False,
|
|
action="store_true")
|
|
return result
|
|
|
|
|
|
def ProcessOsOption(os_name):
|
|
if os_name == 'host':
|
|
return HOST_OS
|
|
return os_name
|
|
|
|
|
|
def ProcessOptions(options, args):
|
|
if options.arch == 'all':
|
|
options.arch = 'ia32,x64,simarm,simarm64'
|
|
if options.mode == 'all':
|
|
options.mode = 'debug,release,product'
|
|
if options.os == 'all':
|
|
options.os = 'host,android'
|
|
if options.sanitizer == 'all':
|
|
options.sanitizer = 'none,asan,lsan,msan,tsan,ubsan'
|
|
options.mode = options.mode.split(',')
|
|
options.arch = options.arch.split(',')
|
|
options.os = options.os.split(',')
|
|
options.sanitizer = options.sanitizer.split(',')
|
|
for mode in options.mode:
|
|
if not mode in ['debug', 'release', 'product']:
|
|
print("Unknown mode %s" % mode)
|
|
return False
|
|
for i, arch in enumerate(options.arch):
|
|
if not arch in AVAILABLE_ARCHS:
|
|
# Normalise to lower case form to make it less case-picky.
|
|
arch_lower = arch.lower()
|
|
if arch_lower in AVAILABLE_ARCHS:
|
|
options.arch[i] = arch_lower
|
|
continue
|
|
print("Unknown arch %s" % arch)
|
|
return False
|
|
options.os = [ProcessOsOption(os_name) for os_name in options.os]
|
|
for os_name in options.os:
|
|
if not os_name in ['android', 'freebsd', 'linux', 'macos', 'win32']:
|
|
print("Unknown os %s" % os_name)
|
|
return False
|
|
if os_name != HOST_OS:
|
|
if os_name != 'android':
|
|
print("Unsupported target os %s" % os_name)
|
|
return False
|
|
if not HOST_OS in ['linux', 'macos']:
|
|
print("Cross-compilation to %s is not supported on host os %s."
|
|
% (os_name, HOST_OS))
|
|
return False
|
|
if not arch in [
|
|
'ia32', 'x64', 'arm', 'arm_x64', 'armv6', 'arm64'
|
|
]:
|
|
print(
|
|
"Cross-compilation to %s is not supported for architecture %s."
|
|
% (os_name, arch))
|
|
return False
|
|
# We have not yet tweaked the v8 dart build to work with the Android
|
|
# NDK/SDK, so don't try to build it.
|
|
if not args:
|
|
print(
|
|
"For android builds you must specify a target, such as 'runtime'."
|
|
)
|
|
return False
|
|
return True
|
|
|
|
|
|
def NotifyBuildDone(build_config, success, start):
|
|
if not success:
|
|
print("BUILD FAILED")
|
|
|
|
sys.stdout.flush()
|
|
|
|
# Display a notification if build time exceeded DART_BUILD_NOTIFICATION_DELAY.
|
|
notification_delay = float(
|
|
os.getenv('DART_BUILD_NOTIFICATION_DELAY', sys.float_info.max))
|
|
if (time.time() - start) < notification_delay:
|
|
return
|
|
|
|
if success:
|
|
message = 'Build succeeded.'
|
|
else:
|
|
message = 'Build failed.'
|
|
title = build_config
|
|
|
|
command = None
|
|
if HOST_OS == 'macos':
|
|
# Use AppleScript to display a UI non-modal notification.
|
|
script = 'display notification "%s" with title "%s" sound name "Glass"' % (
|
|
message, title)
|
|
command = "osascript -e '%s' &" % script
|
|
elif HOST_OS == 'linux':
|
|
if success:
|
|
icon = 'dialog-information'
|
|
else:
|
|
icon = 'dialog-error'
|
|
command = "notify-send -i '%s' '%s' '%s' &" % (icon, message, title)
|
|
elif HOST_OS == 'win32':
|
|
if success:
|
|
icon = 'info'
|
|
else:
|
|
icon = 'error'
|
|
command = (
|
|
"powershell -command \""
|
|
"[reflection.assembly]::loadwithpartialname('System.Windows.Forms')"
|
|
"| Out-Null;"
|
|
"[reflection.assembly]::loadwithpartialname('System.Drawing')"
|
|
"| Out-Null;"
|
|
"$n = new-object system.windows.forms.notifyicon;"
|
|
"$n.icon = [system.drawing.systemicons]::information;"
|
|
"$n.visible = $true;"
|
|
"$n.showballoontip(%d, '%s', '%s', "
|
|
"[system.windows.forms.tooltipicon]::%s);\"") % (
|
|
5000, # Notification stays on for this many milliseconds
|
|
message,
|
|
title,
|
|
icon)
|
|
|
|
if command:
|
|
# Ignore return code, if this command fails, it doesn't matter.
|
|
os.system(command)
|
|
|
|
|
|
def GenerateBuildfilesIfNeeded():
|
|
if os.path.exists(utils.GetBuildDir(HOST_OS)):
|
|
return True
|
|
command = [
|
|
'python',
|
|
os.path.join(DART_ROOT, 'tools', 'generate_buildfiles.py')
|
|
]
|
|
print("Running " + ' '.join(command))
|
|
process = subprocess.Popen(command)
|
|
process.wait()
|
|
if process.returncode != 0:
|
|
print("Tried to generate missing buildfiles, but failed. "
|
|
"Try running manually:\n\t$ " + ' '.join(command))
|
|
return False
|
|
return True
|
|
|
|
|
|
def RunGNIfNeeded(out_dir, target_os, mode, arch, use_nnbd, sanitizer):
|
|
if os.path.isfile(os.path.join(out_dir, 'args.gn')):
|
|
return
|
|
gn_os = 'host' if target_os == HOST_OS else target_os
|
|
gn_command = [
|
|
'python',
|
|
os.path.join(DART_ROOT, 'tools', 'gn.py'),
|
|
'--sanitizer',
|
|
sanitizer,
|
|
'-m',
|
|
mode,
|
|
'-a',
|
|
arch,
|
|
'--os',
|
|
gn_os,
|
|
'-v',
|
|
]
|
|
if use_nnbd:
|
|
gn_command.append('--nnbd')
|
|
|
|
process = subprocess.Popen(gn_command)
|
|
process.wait()
|
|
if process.returncode != 0:
|
|
print("Tried to run GN, but it failed. Try running it manually: \n\t$ "
|
|
+ ' '.join(gn_command))
|
|
|
|
|
|
def UseGoma(out_dir):
|
|
args_gn = os.path.join(out_dir, 'args.gn')
|
|
return 'use_goma = true' in open(args_gn, 'r').read()
|
|
|
|
|
|
# Try to start goma, but don't bail out if we can't. Instead print an error
|
|
# message, and let the build fail with its own error messages as well.
|
|
goma_started = False
|
|
|
|
|
|
def EnsureGomaStarted(out_dir):
|
|
global goma_started
|
|
if goma_started:
|
|
return True
|
|
args_gn_path = os.path.join(out_dir, 'args.gn')
|
|
goma_dir = None
|
|
with open(args_gn_path, 'r') as fp:
|
|
for line in fp:
|
|
if 'goma_dir' in line:
|
|
words = line.split()
|
|
goma_dir = words[2][1:-1] # goma_dir = "/path/to/goma"
|
|
if not goma_dir:
|
|
print('Could not find goma for ' + out_dir)
|
|
return False
|
|
if not os.path.exists(goma_dir) or not os.path.isdir(goma_dir):
|
|
print('Could not find goma at ' + goma_dir)
|
|
return False
|
|
goma_ctl = os.path.join(goma_dir, 'goma_ctl.py')
|
|
goma_ctl_command = [
|
|
'python',
|
|
goma_ctl,
|
|
'ensure_start',
|
|
]
|
|
process = subprocess.Popen(goma_ctl_command)
|
|
process.wait()
|
|
if process.returncode != 0:
|
|
print(
|
|
"Tried to run goma_ctl.py, but it failed. Try running it manually: "
|
|
+ "\n\t" + ' '.join(goma_ctl_command))
|
|
return False
|
|
goma_started = True
|
|
return True
|
|
|
|
# Returns a tuple (build_config, command to run, whether goma is used)
|
|
def BuildOneConfig(options, targets, target_os, mode, arch, sanitizer):
|
|
build_config = utils.GetBuildConf(mode, arch, target_os, sanitizer,
|
|
options.nnbd)
|
|
out_dir = utils.GetBuildRoot(HOST_OS, mode, arch, target_os, sanitizer,
|
|
options.nnbd)
|
|
using_goma = False
|
|
# TODO(zra): Remove auto-run of gn, replace with prompt for user to run
|
|
# gn.py manually.
|
|
RunGNIfNeeded(out_dir, target_os, mode, arch, options.nnbd, sanitizer)
|
|
command = ['ninja', '-C', out_dir]
|
|
if options.verbose:
|
|
command += ['-v']
|
|
if UseGoma(out_dir):
|
|
if options.no_start_goma or EnsureGomaStarted(out_dir):
|
|
using_goma = True
|
|
command += [('-j%s' % str(options.j))]
|
|
command += [('-l%s' % str(options.l))]
|
|
else:
|
|
# If we couldn't ensure that goma is started, let the build start, but
|
|
# slowly so we can see any helpful error messages that pop out.
|
|
command += ['-j1']
|
|
command += targets
|
|
return (build_config, command, using_goma)
|
|
|
|
|
|
def RunOneBuildCommand(build_config, args, env):
|
|
start_time = time.time()
|
|
print(' '.join(args))
|
|
process = subprocess.Popen(args, env=env, stdin=None)
|
|
process.wait()
|
|
if process.returncode != 0:
|
|
NotifyBuildDone(build_config, success=False, start=start_time)
|
|
return 1
|
|
else:
|
|
NotifyBuildDone(build_config, success=True, start=start_time)
|
|
|
|
return 0
|
|
|
|
|
|
def RunOneGomaBuildCommand(options):
|
|
(env, args) = options
|
|
try:
|
|
print(' '.join(args))
|
|
process = subprocess.Popen(args, env=env, stdin=None)
|
|
process.wait()
|
|
print(' '.join(args) + " done.")
|
|
return process.returncode
|
|
except KeyboardInterrupt:
|
|
return 1
|
|
|
|
|
|
def SanitizerEnvironmentVariables():
|
|
with io.open('tools/bots/test_matrix.json', encoding='utf-8') as fd:
|
|
config = json.loads(fd.read())
|
|
env = dict()
|
|
for k, v in config['sanitizer_options'].items():
|
|
env[str(k)] = str(v)
|
|
symbolizer_path = config['sanitizer_symbolizer'].get(HOST_OS, None)
|
|
if symbolizer_path:
|
|
symbolizer_path = str(os.path.join(DART_ROOT, symbolizer_path))
|
|
env['ASAN_SYMBOLIZER_PATH'] = symbolizer_path
|
|
env['MSAN_SYMBOLIZER_PATH'] = symbolizer_path
|
|
env['TSAN_SYMBOLIZER_PATH'] = symbolizer_path
|
|
return env
|
|
|
|
|
|
def Main():
|
|
starttime = time.time()
|
|
# Parse the options.
|
|
parser = BuildOptions()
|
|
(options, args) = parser.parse_args()
|
|
if not ProcessOptions(options, args):
|
|
parser.print_help()
|
|
return 1
|
|
# Determine which targets to build. By default we build the "all" target.
|
|
if len(args) == 0:
|
|
targets = ['all']
|
|
else:
|
|
targets = args
|
|
|
|
if not GenerateBuildfilesIfNeeded():
|
|
return 1
|
|
|
|
# If binaries are built with sanitizers we should use those flags.
|
|
# If the binaries are not built with sanitizers the flag should have no
|
|
# effect.
|
|
env = dict(os.environ)
|
|
env.update(SanitizerEnvironmentVariables())
|
|
|
|
# Build all targets for each requested configuration.
|
|
configs = []
|
|
for target_os in options.os:
|
|
for mode in options.mode:
|
|
for arch in options.arch:
|
|
for sanitizer in options.sanitizer:
|
|
configs.append(
|
|
BuildOneConfig(options, targets, target_os, mode, arch,
|
|
sanitizer))
|
|
|
|
# Build regular configs.
|
|
goma_builds = []
|
|
for (build_config, args, goma) in configs:
|
|
if args is None:
|
|
return 1
|
|
if goma:
|
|
goma_builds.append([env, args])
|
|
elif RunOneBuildCommand(build_config, args, env=env) != 0:
|
|
return 1
|
|
|
|
# Run goma builds in parallel.
|
|
pool = multiprocessing.Pool(multiprocessing.cpu_count())
|
|
results = pool.map(RunOneGomaBuildCommand, goma_builds, chunksize=1)
|
|
for r in results:
|
|
if r != 0:
|
|
return 1
|
|
|
|
endtime = time.time()
|
|
print("The build took %.3f seconds" % (endtime - starttime))
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(Main())
|