dart-sdk/tools/build.py

#!/usr/bin/env python3
#
# Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

import argparse
import io
import json
import os
import subprocess
import sys
import time
import utils

import gn as gn_py

HOST_OS = utils.GuessOS()
HOST_CPUS = utils.GuessCpus()
SCRIPT_DIR = os.path.dirname(sys.argv[0])
DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..'))
AVAILABLE_ARCHS = utils.ARCH_FAMILY.keys()

usage = """\
usage: %%prog [options] [targets]

This script invokes ninja to build Dart.
"""

def BuildOptions():
    parser = argparse.ArgumentParser(
        description='Runs GN (if necessary) followed by ninja',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    config_group = parser.add_argument_group('Configuration Related Arguments')
    gn_py.AddCommonConfigurationArgs(config_group)

    gn_group = parser.add_argument_group('GN Related Arguments')
    gn_py.AddCommonGnOptionArgs(gn_group)

    other_group = parser.add_argument_group('Other Arguments')
    gn_py.AddOtherArgs(other_group)

    other_group.add_argument("-j",
                             type=int,
                             help='Ninja -j option for RBE builds.',
                             default=200 if sys.platform == 'win32' else 1000)
    other_group.add_argument("-l",
                             type=int,
                             help='Ninja -l option for RBE builds.',
                             default=64)
    other_group.add_argument("--no-start-rbe",
                             help="Don't try to start rbe",
                             default=False,
                             action='store_true')
    other_group.add_argument(
        "--check-clean",
        help="Check that a second invocation of Ninja has nothing to do",
        default=False,
        action='store_true')

    parser.add_argument('build_targets', nargs='*')

    return parser


def NotifyBuildDone(build_config, success, start):
    if not success:
        print("BUILD FAILED")

    sys.stdout.flush()

    # Display a notification if build time exceeded DART_BUILD_NOTIFICATION_DELAY.
    notification_delay = float(
        os.getenv('DART_BUILD_NOTIFICATION_DELAY', sys.float_info.max))
    if (time.time() - start) < notification_delay:
        return

    if success:
        message = 'Build succeeded.'
    else:
        message = 'Build failed.'
    title = build_config

    command = None
    if HOST_OS == 'macos':
        # Use AppleScript to display a UI non-modal notification.
        script = 'display notification  "%s" with title "%s" sound name "Glass"' % (
            message, title)
        command = "osascript -e '%s' &" % script
    elif HOST_OS == 'linux':
        if success:
            icon = 'dialog-information'
        else:
            icon = 'dialog-error'
        command = "notify-send -i '%s' '%s' '%s' &" % (icon, message, title)
    elif HOST_OS == 'win32':
        if success:
            icon = 'info'
        else:
            icon = 'error'
        command = (
            "powershell -command \""
            "[reflection.assembly]::loadwithpartialname('System.Windows.Forms')"
            "| Out-Null;"
            "[reflection.assembly]::loadwithpartialname('System.Drawing')"
            "| Out-Null;"
            "$n = new-object system.windows.forms.notifyicon;"
            "$n.icon = [system.drawing.systemicons]::information;"
            "$n.visible = $true;"
            "$n.showballoontip(%d, '%s', '%s', "
            "[system.windows.forms.tooltipicon]::%s);\"") % (
                5000,  # Notification stays on for this many milliseconds
                message,
                title,
                icon)

    if command:
        # Ignore return code, if this command fails, it doesn't matter.
        os.system(command)


def UseRBE(out_dir):
    args_gn = os.path.join(out_dir, 'args.gn')
    return 'use_rbe = true' in open(args_gn, 'r').read()


# Try to start RBE, but don't bail out if we can't. Instead print an error
# message, and let the build fail with its own error messages as well.
rbe_started = False
bootstrap_path = None


def StartRBE(out_dir, env):
    global rbe_started, bootstrap_path
    if not rbe_started:
        rbe_dir = 'buildtools/reclient'
        with open(os.path.join(out_dir, 'args.gn'), 'r') as fp:
            for line in fp:
                if 'rbe_dir' in line:
                    words = line.split()
                    rbe_dir = words[2][1:-1]  # rbe_dir = "/path/to/rbe"
        bootstrap_path = os.path.join(rbe_dir, 'bootstrap')
        bootstrap_command = [bootstrap_path]
        process = subprocess.Popen(bootstrap_command, env=env)
        process.wait()
        if process.returncode != 0:
            print('Failed to start RBE')
            return False
        rbe_started = True
    return True


def StopRBE(env):
    global rbe_started, bootstrap_path
    if rbe_started:
        bootstrap_command = [bootstrap_path, '--shutdown']
        process = subprocess.Popen(bootstrap_command, env=env)
        process.wait()
        rbe_started = False


# Returns a tuple (build_config, command to run, whether rbe is used)
def BuildOneConfig(options, targets, target_os, mode, arch, sanitizer, env):
    build_config = utils.GetBuildConf(mode, arch, target_os, sanitizer)
    out_dir = utils.GetBuildRoot(HOST_OS, mode, arch, target_os, sanitizer)
    using_rbe = False
    command = ['buildtools/ninja/ninja', '-C', out_dir]
    if options.verbose:
        command += ['-v']
    if UseRBE(out_dir):
        if options.no_start_rbe or StartRBE(out_dir, env):
            using_rbe = True
            command += [('-j%s' % str(options.j))]
            command += [('-l%s' % str(options.l))]
        else:
            exit(1)
    command += targets
    return (build_config, command, using_rbe)


def RunOneBuildCommand(build_config, args, env):
    start_time = time.time()
    print(' '.join(args))
    process = subprocess.Popen(args, env=env, stdin=None)
    process.wait()
    if process.returncode != 0:
        NotifyBuildDone(build_config, success=False, start=start_time)
        return 1
    else:
        NotifyBuildDone(build_config, success=True, start=start_time)

    return 0


def CheckCleanBuild(build_config, args, env):
    args = args + ['-n', '-d', 'explain']
    print(' '.join(args))
    process = subprocess.Popen(args,
                               env=env,
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE,
                               stdin=None)
    out, err = process.communicate()
    process.wait()
    if process.returncode != 0:
        return 1
    if 'ninja: no work to do' not in out.decode('utf-8'):
        print(err.decode('utf-8'))
        return 1

    return 0


def SanitizerEnvironmentVariables():
    with io.open('tools/bots/test_matrix.json', encoding='utf-8') as fd:
        config = json.loads(fd.read())
        env = dict()
        for k, v in config['sanitizer_options'].items():
            env[str(k)] = str(v)
        symbolizer_path = config['sanitizer_symbolizer'].get(HOST_OS, None)
        if symbolizer_path:
            symbolizer_path = str(os.path.join(DART_ROOT, symbolizer_path))
            env['ASAN_SYMBOLIZER_PATH'] = symbolizer_path
            env['LSAN_SYMBOLIZER_PATH'] = symbolizer_path
            env['MSAN_SYMBOLIZER_PATH'] = symbolizer_path
            env['TSAN_SYMBOLIZER_PATH'] = symbolizer_path
            env['UBSAN_SYMBOLIZER_PATH'] = symbolizer_path
        return env


def Build(configs, env, options):
    # Build regular configs.
    rbe_builds = []
    for (build_config, args, rbe) in configs:
        if args is None:
            return 1
        if rbe:
            rbe_builds.append([env, args])
        elif RunOneBuildCommand(build_config, args, env=env) != 0:
            return 1

    # Run RBE builds in parallel.
    active_rbe_builds = []
    for (env, args) in rbe_builds:
        print(' '.join(args))
        process = subprocess.Popen(args, env=env)
        active_rbe_builds.append([args, process])
    while active_rbe_builds:
        time.sleep(0.1)
        for rbe_build in active_rbe_builds:
            (args, process) = rbe_build
            if process.poll() is not None:
                print(' '.join(args) + " done.")
                active_rbe_builds.remove(rbe_build)
                if process.returncode != 0:
                    for (_, to_kill) in active_rbe_builds:
                        to_kill.terminate()
                    return 1

    if options.check_clean:
        for (build_config, args, rbe) in configs:
            if CheckCleanBuild(build_config, args, env=env) != 0:
                return 1

    return 0


def Main():
    starttime = time.time()
    # Parse the options.
    parser = BuildOptions()
    options = parser.parse_args()

    targets = options.build_targets

    if not gn_py.ProcessOptions(options):
        parser.print_help()
        return 1

    # If binaries are built with sanitizers we should use those flags.
    # If the binaries are not built with sanitizers the flag should have no
    # effect.
    env = dict(os.environ)
    env.update(SanitizerEnvironmentVariables())

    # macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly.
    #
    # See:
    #
    #   * https://openradar.appspot.com/radar?id=5608755232243712
    #   * https://github.com/dart-lang/sdk/issues/52411
    #
    # Remove these environment variables to avoid affecting clang's behaviors.
    if sys.platform == 'darwin':
        env.pop('CPATH', None)
        env.pop('LIBRARY_PATH', None)
        env.pop('SDKROOT', None)

    # Always run GN before building.
    gn_py.RunGnOnConfiguredConfigurations(options, env)

    # Build all targets for each requested configuration.
    configs = []
    for target_os in options.os:
        for mode in options.mode:
            for arch in options.arch:
                for sanitizer in options.sanitizer:
                    configs.append(
                        BuildOneConfig(options, targets, target_os, mode, arch,
                                       sanitizer, env))

    exit_code = Build(configs, env, options)

    endtime = time.time()

    StopRBE(env)

    if exit_code == 0:
        print("The build took %.3f seconds" % (endtime - starttime))
    return exit_code


if __name__ == '__main__':
    sys.exit(Main())