From 7d7e39a952f8ad2e04f676c01066852c6c7c873f Mon Sep 17 00:00:00 2001 From: Zach Anderson Date: Tue, 3 Jul 2018 22:50:16 +0000 Subject: [PATCH] [build] Move dart2js run for Observatory from python to GN Change-Id: Ief90d6125e30c6bed808bacce521fc84957fddb9 Reviewed-on: https://dart-review.googlesource.com/63662 Commit-Queue: Zach Anderson Reviewed-by: Ryan Macnak --- build/dart/dart_action.gni | 198 ++++++++++++---------- runtime/BUILD.gn | 1 - runtime/observatory/BUILD.gn | 96 +++++------ tools/make_version.py | 17 +- tools/observatory_tool.py | 172 +------------------ tools/print_version.py | 24 --- utils/compiler/BUILD.gn | 1 + utils/compiler/create_snapshot_entry.dart | 4 +- 8 files changed, 169 insertions(+), 344 deletions(-) delete mode 100755 tools/print_version.py diff --git a/build/dart/dart_action.gni b/build/dart/dart_action.gni index ee5e6e6c525..d1825f9af6a 100644 --- a/build/dart/dart_action.gni +++ b/build/dart/dart_action.gni @@ -73,9 +73,8 @@ template("_compiled_action") { # If that's not the case, we'll need another argument to the script to # specify this, since we can't know what the output name is (it might be in # another file not processed yet). - host_executable = - get_label_info(host_tool, "root_out_dir") + "/" + - get_label_info(host_tool, "name") + executable_suffix + host_executable = get_label_info(host_tool, "root_out_dir") + "/" + + get_label_info(host_tool, "name") + executable_suffix # Add the executable itself as an input. inputs += [ host_executable ] @@ -102,9 +101,9 @@ template("_compiled_action") { template("_prebuilt_tool_action") { assert(defined(invoker.binary), - "The path to where the prebuilt binary lives must be defined") + "The path to where the prebuilt binary lives must be defined") assert(defined(invoker.target), - "The target to use if the prebuilt doesn't exist must be defined") + "The target to use if the prebuilt doesn't exist must be defined") vm_args = [] if (defined(invoker.vm_args)) { @@ -112,15 +111,16 @@ template("_prebuilt_tool_action") { } if (_is_fuchsia || prebuilt_dart_exe_works) { - not_needed(invoker, ["target"]) + not_needed(invoker, [ "target" ]) action(target_name) { - forward_variables_from(invoker, [ - "depfile", - "deps", - "outputs", - "testonly", - "visibility", - ]) + forward_variables_from(invoker, + [ + "depfile", + "deps", + "outputs", + "testonly", + "visibility", + ]) script = "$_dart_root/build/gn_run_binary.py" inputs = [] @@ -136,18 +136,14 @@ template("_prebuilt_tool_action") { } args = [ - "compiled_action", - rebase_path(invoker.binary), - ] + vm_args + "compiled_action", + rebase_path(invoker.binary), + ] + vm_args if (defined(invoker.packages)) { - args += [ - "--packages=" + rebase_path(invoker.packages), - ] + args += [ "--packages=" + rebase_path(invoker.packages) ] } if (defined(invoker.dfe)) { - args += [ - "--dfe=" + rebase_path(invoker.dfe), - ] + args += [ "--dfe=" + rebase_path(invoker.dfe) ] } if (defined(invoker.script)) { args += [ rebase_path(invoker.script) ] @@ -155,16 +151,17 @@ template("_prebuilt_tool_action") { args += invoker.args } } else { - not_needed(invoker, ["binary"]) + not_needed(invoker, [ "binary" ]) _compiled_action(target_name) { - forward_variables_from(invoker, [ - "depfile", - "deps", - "inputs", - "outputs", - "testonly", - "visibility", - ]) + forward_variables_from(invoker, + [ + "depfile", + "deps", + "inputs", + "outputs", + "testonly", + "visibility", + ]) if (defined(invoker.script)) { inputs += [ invoker.script ] @@ -176,9 +173,7 @@ template("_prebuilt_tool_action") { tool = invoker.target args = vm_args if (defined(invoker.packages)) { - args += [ - "--packages=" + rebase_path(invoker.packages), - ] + args += [ "--packages=" + rebase_path(invoker.packages) ] } if (defined(invoker.script)) { args += [ rebase_path(invoker.script) ] @@ -226,23 +221,39 @@ template("prebuilt_dart_action") { binary = prebuilt_dart dfe = "$prebuilt_dart_sdk/bin/snapshots/kernel-service.dart.snapshot" } else { - binary = - "$_dart_root/tools/sdks/dart-sdk/bin/dart$executable_suffix" - dfe = - "$_dart_root/tools/sdks/dart-sdk/bin/snapshots/kernel-service.dart.snapshot" + binary = "$_dart_root/tools/sdks/dart-sdk/bin/dart$executable_suffix" + dfe = "$_dart_root/tools/sdks/dart-sdk/bin/snapshots/kernel-service.dart.snapshot" } target = "$_dart_root/runtime/bin:dart_bootstrap" } } if (_is_fuchsia) { -template("_prebuilt_gen_snapshot_action") { - _prebuilt_tool_action(target_name) { - forward_variables_from(invoker, "*") - binary = prebuilt_gen_snapshot - target = "error" + template("_prebuilt_gen_snapshot_action") { + _prebuilt_tool_action(target_name) { + forward_variables_from(invoker, "*") + binary = prebuilt_gen_snapshot + target = "error" + } } } + +template("prebuilt_dart2js_action") { + assert(defined(invoker.script), "script must be defined for $target_name") + assert(defined(invoker.outputs), "outputs must be defined for $target_name") + assert(defined(invoker.args), "args must be defined for $target_name") + + _prebuilt_tool_action(target_name) { + forward_variables_from(invoker, "*") + + # TODO(zra): On Fuchsia use a prebuilt dart2js, or a prebuilt Observatory. + extension = "" + if (is_win) { + extension = ".bat" + } + binary = "$_dart_root/tools/sdks/dart-sdk/bin/dart2js${extension}" + target = "error" + } } # This template runs the specified tool produced by the in-progress build. @@ -283,15 +294,16 @@ template("_built_tool_action") { } _compiled_action(target_name) { - forward_variables_from(invoker, [ - "depfile", - "deps", - "inputs", - "outputs", - "tool", - "testonly", - "visibility", - ]) + forward_variables_from(invoker, + [ + "depfile", + "deps", + "inputs", + "outputs", + "tool", + "testonly", + "visibility", + ]) if (!defined(invoker.inputs)) { inputs = [] @@ -341,19 +353,20 @@ template("dart_action") { if (!_is_fuchsia || !use_prebuilt_dart_sdk) { _built_tool_action(target_name) { tool = "$_dart_root/runtime/bin:dart" - forward_variables_from(invoker, [ - "args", - "depfile", - "deps", - "inputs", - "outputs", - "packages", - "script", - "testonly", - "tool", - "visibility", - "vm_args", - ]) + forward_variables_from(invoker, + [ + "args", + "depfile", + "deps", + "inputs", + "outputs", + "packages", + "script", + "testonly", + "tool", + "visibility", + "vm_args", + ]) } } else { prebuilt_dart_action(target_name) { @@ -389,19 +402,20 @@ template("dart_bootstrap_action") { if (!_is_fuchsia || !use_prebuilt_dart_sdk) { _built_tool_action(target_name) { tool = "$_dart_root/runtime/bin:dart_bootstrap" - forward_variables_from(invoker, [ - "args", - "depfile", - "deps", - "inputs", - "outputs", - "packages", - "script", - "testonly", - "tool", - "visibility", - "vm_args", - ]) + forward_variables_from(invoker, + [ + "args", + "depfile", + "deps", + "inputs", + "outputs", + "packages", + "script", + "testonly", + "tool", + "visibility", + "vm_args", + ]) } } else { # We already have a prebuilt dart at the right version, so there is no @@ -432,23 +446,25 @@ template("dart_bootstrap_action") { # testonly # visibility template("gen_snapshot_action") { - assert(!defined(invoker.script), + assert( + !defined(invoker.script), "script must not be defined for $target_name. If there is a script use args instead.") if (!_is_fuchsia || !use_prebuilt_dart_sdk) { _built_tool_action(target_name) { tool = "$_dart_root/runtime/bin:gen_snapshot" - forward_variables_from(invoker, [ - "args", - "depfile", - "deps", - "inputs", - "outputs", - "packages", - "testonly", - "tool", - "visibility", - "vm_args", - ]) + forward_variables_from(invoker, + [ + "args", + "depfile", + "deps", + "inputs", + "outputs", + "packages", + "testonly", + "tool", + "visibility", + "vm_args", + ]) } } else { _prebuilt_gen_snapshot_action(target_name) { diff --git a/runtime/BUILD.gn b/runtime/BUILD.gn index 29f65450104..ed7631b2756 100644 --- a/runtime/BUILD.gn +++ b/runtime/BUILD.gn @@ -231,7 +231,6 @@ library_for_all_configs("libdart") { action("generate_version_cc_file") { inputs = [ "../tools/utils.py", - "../tools/print_version.py", "../tools/VERSION", "vm/version_in.cc", ] diff --git a/runtime/observatory/BUILD.gn b/runtime/observatory/BUILD.gn index 181a9cd6e37..f9d1fee5a6e 100644 --- a/runtime/observatory/BUILD.gn +++ b/runtime/observatory/BUILD.gn @@ -2,78 +2,62 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import("../../build/dart/dart_host_sdk_toolchain.gni") -import("../../build/dart/prebuilt_dart_sdk.gni") -import("../../build/executable_suffix.gni") +import("../../build/dart/dart_action.gni") import("observatory_sources.gni") -# Construct arguments to the observatory tool for finding pub. -build_deps = [] -build_args = [] -if (!prebuilt_dart_exe_works) { - build_deps += [ "../bin:dart_bootstrap($dart_host_toolchain)" ] +prebuilt_dart2js_action("build_observatory") { + script = "web/main.dart" - dart_out_dir = get_label_info("../bin:dart_bootstrap($dart_host_toolchain)", - "root_out_dir") - dart_bootstrap = rebase_path("$dart_out_dir/dart_bootstrap$executable_suffix") - - build_args = [ - "--sdk=True", - "--dart-executable", - dart_bootstrap, - ] -} else { - build_args = [ "--sdk=True" ] -} - -current_dir = rebase_path(".", "//") - -action("build_observatory") { - deps = build_deps - - sources = rebase_path(observatory_sources, "", ".") - - inputs = sources + [ rebase_path(".packages") ] - - script = "../../tools/observatory_tool.py" - args = build_args + [ - "--silent=True", - "--directory", - rebase_path("."), - "--command", - "build", - rebase_path("web/main.dart"), - rebase_path("$target_gen_dir/observatory/web/main.dart.js"), - rebase_path(".packages"), - ] + # dart2js produces a .deps file, but it is not in a format that is understood + # by ninja, so we explicitly list all the sources here. + inputs = [ ".packages" ] + observatory_sources + output = "$target_gen_dir/observatory/web/main.dart.js" outputs = [ - "$target_gen_dir/observatory/web/main.dart.js", + output, + ] + + version_string = exec_script("../../tools/make_version.py", + [ + "--quiet", + "--no_git_hash", + ], # Arguments to the script + "trim string", # Input conversions + [ + "../../tools/VERSION", + "../tools/utils.py", + ]) # Dependencies + + args = [ + "--no-preview-dart-2", + "-DOBS_VER=${version_string}", + "-o", + rebase_path(output), + "--packages=" + rebase_path(".packages"), + "--minify", ] } +# TODO(zra): Investigate using GN's copy() with set_sources_assignment_filter +# instead of using a python script. action("deploy_observatory") { - deps = [ ":build_observatory" ] + build_deps + deps = [ + ":build_observatory", + ] script = "../../tools/observatory_tool.py" inputs = [ - script, "$target_gen_dir/observatory/web/main.dart.js", ] - args = build_args + [ - "--silent=True", - "--directory", - rebase_path("$target_out_dir/observatory"), - "--command", - "deploy", - rebase_path("$target_out_dir/observatory/deployed"), - rebase_path("web"), - rebase_path("lib"), - rebase_path("$target_gen_dir/observatory/web/main.dart.js"), - rebase_path("../../third_party/observatory_pub_packages/packages"), - ] + args = [ + rebase_path("$target_out_dir/observatory/deployed"), + rebase_path("web"), + rebase_path("lib"), + rebase_path("$target_gen_dir/observatory/web/main.dart.js"), + rebase_path("../../third_party/observatory_pub_packages/packages"), + ] outputs = [ "$target_out_dir/observatory/deployed/web/main.dart.js", diff --git a/tools/make_version.py b/tools/make_version.py index f1a15e0a57f..634586aeb35 100755 --- a/tools/make_version.py +++ b/tools/make_version.py @@ -96,7 +96,7 @@ def main(args): parser.add_option("--input", action="store", type="string", - help="input template file") + help="input template file.") parser.add_option("--no_git_hash", action="store_true", default=False, @@ -111,10 +111,23 @@ def main(args): help="disable console output") (options, args) = parser.parse_args() + + # If there is no input template, then write the bare version string to + # options.output. If there is no options.output, then write the version + # string to stdout. + if not options.input: + version_string = MakeVersionString( + options.quiet, options.no_git_hash, options.custom_for_pub) + if options.output: + open(options.output, 'w').write(version_string) + else: + print version_string + return 0 + if not options.output: sys.stderr.write('--output not specified\n') return -1 - if not len(options.input): + if not options.input: sys.stderr.write('--input not specified\n') return -1 diff --git a/tools/observatory_tool.py b/tools/observatory_tool.py index 42028b75bfa..6d9c9e54e5c 100755 --- a/tools/observatory_tool.py +++ b/tools/observatory_tool.py @@ -4,17 +4,10 @@ # BSD-style license that can be found in the LICENSE file. """Helper for building and deploying Observatory""" -import argparse import os -import platform import shutil -import subprocess import sys -import utils -SCRIPT_DIR = os.path.dirname(sys.argv[0]) -DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..')) -DART2JS_PATH = os.path.join(DART_ROOT, 'pkg', 'compiler', 'bin', 'dart2js.dart') IGNORE_PATTERNS = shutil.ignore_patterns( '$sdk', '*.concat.js', @@ -37,112 +30,6 @@ IGNORE_PATTERNS = shutil.ignore_patterns( 'webcomponents-lite.js', 'webcomponents.*') -usage = """observatory_tool.py [options]""" - -def DisplayBootstrapWarning(): - print """\ - -WARNING: Your system cannot run the checked-in Dart SDK. Using the -bootstrap Dart executable will make debug builds slow. -Please see the Wiki for instructions on replacing the checked-in Dart SDK. - -https://github.com/dart-lang/sdk/wiki/The-checked-in-SDK-in-tools - -To use the dart_bootstrap binary please update the Build function -in the tools/observatory_tool.py script. - -""" - -def DisplayFailureMessage(): - print """\ - -ERROR: Observatory failed to build. What should you do? - -1. Revert to a working revision of the Dart VM -2. Contact zra@, rmacnak@, dart-vm-team@ -3. File a bug: https://github.com/dart-lang/sdk/issues/new - -""" - -# Run |command|. If its return code is 0, return 0 and swallow its output. -# If its return code is non-zero, emit its output unless |always_silent| is -# True, and return the return code. -def RunCommand(command, always_silent=False): - try: - subprocess.check_output(command, - stderr=subprocess.STDOUT) - return 0 - except subprocess.CalledProcessError as e: - if not always_silent: - print ("Command failed: " + ' '.join(command) + "\n" + - "output: " + e.output) - DisplayFailureMessage() - return e.returncode - -def BuildArguments(): - result = argparse.ArgumentParser(usage=usage) - result.add_argument("--dart-executable", help="dart executable", default=None) - result.add_argument("--dart2js-executable", help="dart2js executable", - default=None) - result.add_argument("--directory", help="observatory root", default=None) - result.add_argument("--command", help="[build, deploy]", default=None) - result.add_argument("--silent", help="silence all output", default=None) - result.add_argument("--sdk", help="Use prebuilt sdk", default=None) - return result - -def ProcessOptions(options, args): - # Fix broken boolean parsing in argparse, where False ends up being True. - if (options.silent is not None) and (options.silent == "True"): - options.silent = True - elif (options.silent is None) or (options.silent == "False"): - options.silent = False - else: - print "--silent expects 'True' or 'False' argument." - return False - - if (options.sdk is not None) and (options.sdk == "True"): - options.sdk = True - elif (options.sdk is None) or (options.sdk == "False"): - options.sdk = False - else: - print "--sdk expects 'True' or 'False' argument." - return False - - # Required options. - if options.command is None or options.directory is None: - return False - - # If a dart2js execuble was provided, try and use that. - # TODO(whesse): Drop the dart2js-executable option if it isn't used. - if options.dart2js_executable is not None: - try: - if 0 == RunCommand([options.dart2js_executable, '--version'], - always_silent=True): - return True - except OSError as e: - pass - options.dart2js_executable = None - - # Use the checked in dart2js executable. - if options.sdk and utils.CheckedInSdkCheckExecutable(): - dart2js_binary = 'dart2js.bat' if utils.IsWindows() else 'dart2js' - options.dart2js_executable = os.path.join(utils.CheckedInSdkPath(), - 'bin', - dart2js_binary) - try: - if 0 == RunCommand([options.dart2js_executable, '--version'], - always_silent=True): - return True - except OSError as e: - pass - options.dart2js_executable = None - - # We need a dart executable and will run from source - return (options.dart_executable is not None) - -def ChangeDirectory(directory): - os.chdir(directory); - # - Copy over the filtered web directory # - Merge in the .js file # - Copy over the filtered dependency lib directories @@ -167,62 +54,11 @@ def Deploy(output_dir, web_dir, observatory_lib, js_file, pub_packages_dir): shutil.copytree(observatory_lib, os.path.join(packages_dir, 'observatory'), ignore=IGNORE_PATTERNS) -def Build(dart_executable, - dart2js_executable, - script_path, - output_path, - packages_path, - silent): - if dart2js_executable is not None: - command = [dart2js_executable] - else: - if not silent: - DisplayBootstrapWarning() - command = [dart_executable, DART2JS_PATH] - command += ['--no-preview-dart-2'] - command += ['-DOBS_VER=' + utils.GetVersion(no_git_hash=True)] - command += [script_path, '-o', output_path, '--packages=%s' % packages_path] - # Add the defaults pub used - command += ['--minify'] - if not silent: - print >> sys.stderr, 'Running command "%s"' % command - return RunCommand(command) -def ExecuteCommand(options, args): - cmd = options.command - if (cmd == 'build'): - return Build(options.dart_executable, - options.dart2js_executable, - args[0], - args[1], - args[2], - options.silent) - elif (cmd == 'deploy'): - Deploy(args[0], args[1], args[2], args[3], args[4]) - else: - print >> sys.stderr, ('ERROR: command "%s" not supported') % (cmd) - return -1; +def Main(): + args = sys.argv[1:] + return Deploy(args[0], args[1], args[2], args[3], args[4]) -def main(): - # Parse the options. - parser = BuildArguments() - (options, args) = parser.parse_known_args() - if not ProcessOptions(options, args): - parser.print_help() - return 1 - # Calculate absolute paths before changing directory. - if (options.dart_executable != None): - options.dart_executable = os.path.abspath(options.dart_executable) - if (options.dart2js_executable != None): - options.dart2js_executable = os.path.abspath(options.dart2js_executable) - if len(args) == 1: - args[0] = os.path.abspath(args[0]) - try: - # Pub must be run from the project's root directory. - ChangeDirectory(options.directory) - return ExecuteCommand(options, args) - except: - DisplayFailureMessage() if __name__ == '__main__': - sys.exit(main()); + sys.exit(Main()); diff --git a/tools/print_version.py b/tools/print_version.py deleted file mode 100755 index 13cbf750793..00000000000 --- a/tools/print_version.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file -# for details. All rights reserved. Use of this source code is governed by a -# BSD-style license that can be found in the LICENSE file. -# -# A script which will be invoked from gyp to print the current version of the -# SDK. -# -# Usage: print_version -# - -import sys -import utils - -def Main(): - version = utils.GetVersion() - if not version: - print 'Error: Couldn\'t determine version string.' - return 1 - print version - -if __name__ == '__main__': - sys.exit(Main()) diff --git a/utils/compiler/BUILD.gn b/utils/compiler/BUILD.gn index 8eab0818d1d..d9f1cd04707 100644 --- a/utils/compiler/BUILD.gn +++ b/utils/compiler/BUILD.gn @@ -41,6 +41,7 @@ dart_action("dart2js_create_snapshot_entry") { inputs = [ "../../sdk/lib/_internal/sdk_library_metadata/lib/libraries.dart", "$target_gen_dir/dart2js_files.stamp", + "../../tools/make_version.py", "../../tools/VERSION", ] diff --git a/utils/compiler/create_snapshot_entry.dart b/utils/compiler/create_snapshot_entry.dart index 2c6497b0d34..dca5fd54de4 100644 --- a/utils/compiler/create_snapshot_entry.dart +++ b/utils/compiler/create_snapshot_entry.dart @@ -11,9 +11,9 @@ import 'dart:async'; Future getVersion(var rootPath) { var suffix = Platform.operatingSystem == 'windows' ? '.exe' : ''; - var printVersionScript = rootPath.resolve("tools/print_version.py"); + var printVersionScript = rootPath.resolve("tools/make_version.py"); return Process - .run("python$suffix", [printVersionScript.toFilePath()]).then((result) { + .run("python$suffix", [printVersionScript.toFilePath(), "--quiet"]).then((result) { if (result.exitCode != 0) { throw "Could not generate version"; }