[SDK] Adds build targets, wrappers and SDK builds for Dart AOT.

Adds:
- dart2aot, a script similar to dart2js which compiles my.dart to my.dart.aot.
- dartaotruntime, a minimal Dart runtime that only runs AOT blobs.
- some extra tooling like gen_kernel and gen_snapshot used by the above.
- build rules for all of the above, including adding it to the full SDK builds.

Bug:https://github.com/dart-lang/sdk/issues/27596
Change-Id: Ic35f832b2b86be959212b8d21cfc5a082da5ced4
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/97627
Reviewed-by: Alexander Thomas <athom@google.com>
Reviewed-by: Vyacheslav Egorov <vegorov@google.com>
Commit-Queue: Clement Skau <cskau@google.com>

runtime/bin/BUILD.gn

@@ -848,6 +848,20 @@ dart_executable("dart_precompiled_runtime_for_linking") {
   target_type = "static_library"
 }
 
+dart_executable("dartaotruntime") {
+  extra_configs = [ "..:dart_precompiled_runtime_config" ]
+  extra_deps = [ "..:libdart_precompiled_runtime" ]
+  extra_sources = [
+    "builtin.cc",
+    "snapshot_empty.cc",
+    "loader.cc",
+    "loader.h",
+    "gzip.cc",
+    "gzip.h",
+    "observatory_assets_empty.cc",
+  ]
+}
+
 executable("process_test") {
   sources = [
     "process_test.cc",

sdk/BUILD.gn

@@ -20,8 +20,10 @@ declare_args() {
   # Build a SDK with less stuff. It excludes dart2js, ddc, and web libraries.
   dart_platform_sdk = true
 
-  # Path to stripped dart binary relative to build output directory.
+  # Path to stripped dart binaries relative to build output directory.
   dart_stripped_binary = "dart"
+  dartaotruntime_stripped_binary = "dartaotruntime"
+  gen_snapshot_stripped_binary = "gen_snapshot"
 }
 
 # The directory layout of the SDK is as follows:
@@ -30,11 +32,14 @@ declare_args() {
 # ....bin/
 # ......dart or dart.exe (executable)
 # ......dart.lib (import library for VM native extensions on Windows)
+# ......dartaotruntime or dartaotruntime.exe (executable)
 # ......dartdoc
 # ......dartfmt
+# ......dart2aot
 # ......dart2js
 # ......dartanalyzer
 # ......dartdevc
+# ......utils/gen_snapshot or utils/gen_snapshot.exe (executable)
 # ......pub
 # ......snapshots/
 # ........analysis_server.dart.snapshot
@@ -43,6 +48,7 @@ declare_args() {
 # ........dartdoc.dart.snapshot
 # ........dartfmt.dart.snapshot
 # ........dartdevc.dart.snapshot
+# ........gen_kernel.dart.snapshot
 # ........kernel_worker.dart.snapshot
 # ........pub.dart.snapshot
 #.........resources/
@@ -341,6 +347,75 @@ if (target_os != current_os && target_os == "fuchsia") {
   }
 }
 
+copy("copy_dartaotruntime") {
+  deps = [
+    "../runtime/bin:dartaotruntime",
+  ]
+  dartaotruntime_out = get_label_info("../runtime/bin:dartaotruntime", "root_out_dir")
+  if (is_win) {
+    sources = [
+      "$dartaotruntime_out/dartaotruntime.exe",
+    ]
+  } else {
+    sources = [
+      "$dartaotruntime_out/$dartaotruntime_stripped_binary",
+    ]
+  }
+  if (is_win) {
+    sources += [ "$dartaotruntime_out/dartaotruntime.lib" ]
+  }
+  outputs = [
+    "$root_out_dir/dart-sdk/bin/{{source_file_part}}",
+  ]
+}
+
+copy("copy_gen_snapshot") {
+  deps = [
+    "../runtime/bin:gen_snapshot",
+  ]
+  gen_snapshot_out = get_label_info("../runtime/bin:gen_snapshot", "root_out_dir")
+  if (is_win) {
+    sources = [
+      "$gen_snapshot_out/gen_snapshot.exe",
+    ]
+  } else {
+    sources = [
+      "$gen_snapshot_out/$gen_snapshot_stripped_binary",
+    ]
+  }
+  if (is_win) {
+    sources += [ "$gen_snapshot_out/gen_snapshot.lib" ]
+  }
+  outputs = [
+    "$root_out_dir/dart-sdk/bin/utils/{{source_file_part}}",
+  ]
+}
+
+copy("copy_dart2aot") {
+  ext = ""
+  if (is_win) {
+    ext = ".bat"
+  }
+  sources = [
+    "bin/dart2aot$ext",
+  ]
+  outputs = [
+    "$root_out_dir/dart-sdk/bin/{{source_file_part}}",
+  ]
+}
+
+copy("copy_gen_kernel_snapshot") {
+  deps = [
+    "../utils/gen_kernel",
+  ]
+  sources = [
+    "$root_gen_dir/gen_kernel.dart.snapshot",
+  ]
+  outputs = [
+    "$root_out_dir/dart-sdk/bin/snapshots/{{source_file_part}}",
+  ]
+}
+
 # A template for copying the things in _platform_sdk_scripts and
 # _full_sdk_scripts into bin/
 template("copy_sdk_script") {

sdk/bin/dart2aot

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+# Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+# Script for generating AOT snapshot in two steps:
+# - Compilation to kernel with additional AOT specific transformations.
+# - Compilation of kernel into snapshot using gen_snapshot.
+
+# Parse incoming arguments and extract the value of --packages option if any
+# was passed. Split options (--xyz) and non-options into two separate arrays.
+# All options will be passed to gen_snapshot, while --packages will be
+# passed to Fasta.
+
+set -e
+
+OPTIONS=()
+GEN_KERNEL_OPTIONS=()
+PACKAGES=
+BUILD_ELF=0
+
+ARGV=()
+for arg in "$@"; do
+  case $arg in
+    --packages=*)
+    PACKAGES="$arg"
+    ;;
+    --enable-asserts)
+    GEN_KERNEL_OPTIONS+=("$arg")
+    OPTIONS+=("$arg")
+    ;;
+    --tfa | \
+    --no-tfa | \
+    -D* )
+    GEN_KERNEL_OPTIONS+=("$arg")
+    ;;
+    --build-elf)
+    BUILD_ELF=1
+    ;;
+    --*)
+    OPTIONS+=("$arg")
+    ;;
+    *)
+    ARGV+=("$arg")
+    ;;
+  esac
+done
+
+if [ "${#ARGV[@]}" -ne 2 ]; then
+    echo "Usage: $0 [options] <source> <snapshot>"
+    exit 1
+fi
+
+SOURCE_FILE="${ARGV[0]}"
+SNAPSHOT_FILE="${ARGV[1]}"
+
+if [ $BUILD_ELF -eq 1 ]; then
+  GEN_SNAPSHOT_OPTION="--snapshot-kind=app-aot-assembly"
+  GEN_SNAPSHOT_FILENAME="--assembly=${SNAPSHOT_FILE}.S"
+else
+  GEN_SNAPSHOT_OPTION="--snapshot-kind=app-aot-blobs"
+  GEN_SNAPSHOT_FILENAME="--blobs_container_filename=${SNAPSHOT_FILE}"
+fi
+
+function follow_links() {
+  file="$1"
+  while [ -h "$file" ]; do
+    # On Mac OS, readlink -f doesn't work.
+    file="$(readlink "$file")"
+  done
+  echo "$file"
+}
+
+# Unlike $0, $BASH_SOURCE points to the absolute path of this file.
+PROG_NAME="$(follow_links "$BASH_SOURCE")"
+
+# Handle the case where dart-sdk/bin has been symlinked to.
+BIN_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
+
+SDK_DIR="$(cd "${BIN_DIR}/.." ; pwd -P)"
+
+DART="$BIN_DIR/dart"
+GEN_SNAPSHOT="$BIN_DIR/utils/gen_snapshot"
+
+SNAPSHOT_DIR="$BIN_DIR/snapshots"
+SNAPSHOT="$SNAPSHOT_DIR/gen_kernel.dart.snapshot"
+
+# Step 1: Generate Kernel binary from the input Dart source.
+"$DART"                                                                        \
+     "${SNAPSHOT}"                                                             \
+     --platform "${SDK_DIR}/lib/_internal/vm_platform_strong.dill"             \
+     --aot                                                                     \
+     -Ddart.vm.product=true                                                    \
+     "${GEN_KERNEL_OPTIONS[@]}"                                                \
+     $PACKAGES                                                                 \
+     -o "$SNAPSHOT_FILE.dill"                                                  \
+     "$SOURCE_FILE"
+
+# Step 2: Generate snapshot from the Kernel binary.
+"$GEN_SNAPSHOT"                                                                \
+     "$GEN_SNAPSHOT_OPTION"                                                    \
+     "$GEN_SNAPSHOT_FILENAME"                                                  \
+     "${OPTIONS[@]}"                                                           \
+     "$SNAPSHOT_FILE.dill"
+
+# Step 3: Assemble the assembly file into an ELF object.
+if [ $BUILD_ELF -eq 1 ]; then
+    gcc -shared -o "$SNAPSHOT_FILE" "${SNAPSHOT_FILE}.S"
+fi

sdk/bin/dart2aot.bat

@@ -0,0 +1,58 @@
+@echo off
+REM Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+REM for details. All rights reserved. Use of this source code is governed by a
+REM BSD-style license that can be found in the LICENSE file.
+
+setlocal
+rem Handle the case where dart-sdk/bin has been symlinked to.
+set DIR_NAME_WITH_SLASH=%~dp0
+set DIR_NAME=%DIR_NAME_WITH_SLASH:~0,-1%%
+call :follow_links "%DIR_NAME%", RETURNED_BIN_DIR
+rem Get rid of surrounding quotes.
+for %%i in ("%RETURNED_BIN_DIR%") do set BIN_DIR=%%~fi
+
+rem Get absolute full name for SDK_DIR.
+for %%i in ("%BIN_DIR%\..\") do set SDK_DIR=%%~fi
+
+rem Remove trailing backslash if there is one
+IF %SDK_DIR:~-1%==\ set SDK_DIR=%SDK_DIR:~0,-1%
+
+rem Get absolute full name for DART_ROOT.
+for %%i in ("%SDK_DIR%\..\") do set DART_ROOT=%%~fi
+
+rem Remove trailing backslash if there is one
+if %DART_ROOT:~-1%==\ set DART_ROOT=%DART_ROOT:~0,-1%
+
+set DART=%BIN_DIR%\dart
+set GEN_KERNEL=%BIN_DIR%\snapshots\gen_kernel.dart.snapshot
+set VM_PLATFORM_STRONG=%SDK_DIR%\lib\_internal\vm_platform_strong.dill
+set GEN_SNAPSHOT=%BIN_DIR%\utils\gen_snapshot
+
+set SOURCE_FILE=%1
+set SNAPSHOT_FILE=%2
+set GEN_SNAPSHOT_OPTION=--snapshot-kind=app-aot-blobs
+set GEN_SNAPSHOT_FILENAME=--blobs_container_filename=%SNAPSHOT_FILE%
+
+REM Step 1: Generate Kernel binary from the input Dart source.
+%DART% %GEN_KERNEL% --platform %VM_PLATFORM_STRONG% --aot -Ddart.vm.product=true -o %SNAPSHOT_FILE%.dill %SOURCE_FILE%
+
+REM Step 2: Generate snapshot from the Kernel binary.
+%GEN_SNAPSHOT% %GEN_SNAPSHOT_OPTION% %GEN_SNAPSHOT_FILENAME% %SNAPSHOT_FILE%.dill
+
+endlocal
+
+exit /b %errorlevel%
+
+:follow_links
+setlocal
+for %%i in (%1) do set result=%%~fi
+set current=
+for /f "usebackq tokens=2 delims=[]" %%i in (`dir /a:l "%~dp1" 2^>nul ^
+                                             ^| %SystemRoot%\System32\find.exe ">     %~n1 [" 2^>nul`) do (
+  set current=%%i
+)
+if not "%current%"=="" call :follow_links "%current%", result
+endlocal & set %~2=%result%
+goto :eof
+
+:end

tools/bots/dart_sdk.py

@@ -66,10 +66,19 @@ def CreateUploadSDKZips():
       sdk_path = os.path.join(bot_utils.DART_DIR,
                               utils.GetBuildRoot(BUILD_OS, 'release', arch),
                               'dart-sdk')
+      product_sdk_path = os.path.join(bot_utils.DART_DIR,
+                              utils.GetBuildRoot(BUILD_OS, 'product', arch),
+                              'dart-sdk')
       sdk_zip = os.path.join(bot_utils.DART_DIR,
                              utils.GetBuildRoot(BUILD_OS, 'release', arch),
                              'dartsdk-%s-%s.zip' % (BUILD_OS, arch))
       FileDelete(sdk_zip)
+      # We don't support precompilation on ia32.
+      if arch != 'ia32':
+        # Patch in all the PRODUCT built AOT binaries.
+        CopyBetween(product_sdk_path, sdk_path, 'bin', 'utils', 'gen_snapshot')
+        CopyBetween(product_sdk_path, sdk_path, 'bin', 'dartaotruntime')
+      # Zip it up.
       CreateZip(sdk_path, sdk_zip)
       DartArchiveUploadSDKs(BUILD_OS, arch, sdk_zip)
 
@@ -186,6 +195,16 @@ def FileDelete(f):
   if os.path.exists(f):
     os.remove(f)
 
+def CopyBetween(src_path, dst_path, *relatives):
+  try:
+    os.makedirs(os.path.join(dst_path, *relatives[:-1]))
+  except OSError:
+    # This is fine.
+    pass
+  shutil.copy2(
+      os.path.join(src_path, *relatives),
+      os.path.join(dst_path, *relatives[:-1]))
+
 def DartArchiveFile(local_path, remote_path, checksum_files=False):
   gsutil = bot_utils.GSUtil()
   gsutil.upload(local_path, remote_path, public=True)

tools/bots/test_matrix.json

@@ -1328,6 +1328,18 @@
           "arguments": ["--arch=ia32,x64,arm,arm64",
                         "--mode=release", "create_sdk"]
         },
+        {
+          "name": "build gen_kernel.dart.snapshot and dart2aot",
+          "script": "tools/build.py",
+          "arguments": ["--arch=x64,arm,arm64", "--mode=release",
+                        "copy_gen_kernel_snapshot", "copy_dart2aot"]
+        },
+        {
+          "name": "build gen_snapshot and dartaotruntime",
+          "script": "tools/build.py",
+          "arguments": ["--arch=x64,arm,arm64", "--mode=product",
+                        "copy_gen_snapshot", "copy_dartaotruntime"]
+        },
         {
           "name": "upload sdk",
           "script": "tools/bots/dart_sdk.py"
@@ -1360,6 +1372,18 @@
           "arguments": ["--arch=ia32,x64",
                         "--mode=release", "create_sdk"]
         },
+        {
+          "name": "build gen_kernel.dart.snapshot and dart2aot",
+          "script": "tools/build.py",
+          "arguments": ["--arch=x64", "--mode=release",
+                        "copy_gen_kernel_snapshot", "copy_dart2aot"]
+        },
+        {
+          "name": "build gen_snapshot and dartaotruntime",
+          "script": "tools/build.py",
+          "arguments": ["--arch=x64", "--mode=product",
+                        "copy_gen_snapshot", "copy_dartaotruntime"]
+        },
         {
           "name": "upload sdk",
           "script": "tools/bots/dart_sdk.py"

tools/gn.py

@@ -245,6 +245,8 @@ def ToGnArgs(args, mode, arch, target_os):
   if not args.platform_sdk and not gn_args['target_cpu'].startswith('arm'):
     gn_args['dart_platform_sdk'] = args.platform_sdk
   gn_args['dart_stripped_binary'] = 'exe.stripped/dart'
+  gn_args['dartaotruntime_stripped_binary'] = 'exe.stripped/dartaotruntime'
+  gn_args['gen_snapshot_stripped_binary'] = 'exe.stripped/gen_snapshot'
 
   # Setup the user-defined sysroot.
   if UseSysroot(args, gn_args):

utils/gen_kernel/BUILD.gn

@@ -0,0 +1,22 @@
+# Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+import("../application_snapshot.gni")
+
+application_snapshot("gen_kernel") {
+  main_dart = "../../pkg/vm/bin/gen_kernel.dart"
+  deps = [
+    "../../runtime/vm:vm_platform",
+  ]
+  # NOTE: The output filename must be kept in sync with the output of the
+  # vm_platform rule.
+  vm_platform_out = get_label_info("../../runtime/vm:vm_platform", "root_out_dir")
+  vm_platform = "$vm_platform_out/vm_platform_strong.dill"
+  training_args = [
+      "--platform",
+      rebase_path(vm_platform),
+      rebase_path("../../pkg/vm/bin/gen_kernel.dart"),
+      "-o -",
+  ]
+}