Link kernel service dill with the command line dart executable.

This linked in kernel service dill file will be used to load the kernel
isolate if the attempt to lookup the kernel service snapshot fails. The
kernel service snapshot is looked up in the following order.

1. If the "--dfe" option is specified, the file specified is used.
2. If the kernel service snapshot is found next to the executable,
then it is used.
3. If the kernel service snapshot is found in the "snapshots" directory
next to the executable, then it is used.

Change-Id: I5a0e757eb27b26a274b22b4bc36350fee59a100f
Reviewed-on: https://dart-review.googlesource.com/32446
Reviewed-by: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Siva Annamalai <asiva@google.com>
Commit-Queue: Siva Chandra <sivachandra@google.com>

BUILD.gn

@@ -56,6 +56,7 @@ group("runtime_kernel") {
     "runtime/vm:kernel_platform_files($host_toolchain)",
 
     # TODO(rmacnak): Link this into 'dart'.
+    "utils/kernel-service:copy_kernel_service_snapshot",
     "utils/kernel-service:kernel-service",
   ]
 }

pkg/vm/bin/gen_kernel.dart

@@ -24,7 +24,10 @@ final ArgParser _argParser = new ArgParser(allowTrailingOptions: true)
       help:
           'Produce kernel file for AOT compilation (enables global transformations).',
       defaultsTo: false)
-  ..addFlag('strong-mode', help: 'Enable strong mode', defaultsTo: true);
+  ..addFlag('strong-mode', help: 'Enable strong mode', defaultsTo: true)
+  ..addFlag('embed-sources',
+      help: 'Embed source files in the generated kernel program',
+      defaultsTo: true);
 
 final String _usage = '''
 Usage: dart pkg/vm/bin/gen_kernel.dart --platform vm_platform_strong.dill [options] input.dart
@@ -68,7 +71,8 @@ Future<int> compile(List<String> arguments) async {
     ..linkedDependencies = <Uri>[Uri.base.resolve(platformKernel)]
     ..packagesFileUri = packages != null ? Uri.base.resolve(packages) : null
     ..reportMessages = true
-    ..onError = errorDetector;
+    ..onError = errorDetector
+    ..embedSourceText = options['embed-sources'];
 
   Program program = await compileToKernel(
       Uri.base.resolve(filename), compilerOptions,

runtime/bin/BUILD.gn

@@ -648,14 +648,36 @@ bin_to_assembly("isolate_snapshot_instructions_assembly") {
   executable = true
 }
 
+action("kernel_service_dill_cc") {
+  deps = [
+    "../../utils/kernel-service:kernel_service_dill",
+  ]
+  script = "../tools/dill_to_data_cc.py"
+  inputs = [
+    "../tools/dill_to_data_cc.py",
+    "$root_gen_dir/kernel_service.dill",
+  ]
+  outputs = [
+    "$root_gen_dir/kernel_service.dill.cc",
+  ]
+  args = [
+    "--dill_file=" + rebase_path("$root_gen_dir/kernel_service.dill"),
+    "--data_symbol=kKernelServiceDill",
+    "--size_symbol=kKernelServiceDillSize",
+    "--output=" + rebase_path("$root_gen_dir/kernel_service.dill.cc"),
+  ]
+}
+
 source_set("dart_snapshot_cc") {
   deps = [
     ":isolate_snapshot_data_assembly",
     ":isolate_snapshot_instructions_assembly",
+    ":kernel_service_dill_cc",
     ":vm_snapshot_data_assembly",
     ":vm_snapshot_instructions_assembly",
   ]
   sources = [
+    "$root_gen_dir/kernel_service.dill.cc",
     "$target_gen_dir/isolate_snapshot_data.S",
     "$target_gen_dir/isolate_snapshot_instructions.S",
     "$target_gen_dir/vm_snapshot_data.S",

runtime/bin/dfe.cc

@@ -4,19 +4,59 @@
 
 #include "bin/dfe.h"
 #include "bin/dartutils.h"
+#include "bin/directory.h"
 #include "bin/error_exit.h"
 #include "bin/file.h"
+#include "bin/platform.h"
+#include "bin/utils.h"
 
 #include "vm/kernel.h"
 
+extern "C" {
+extern const uint8_t kKernelServiceDill[];
+extern intptr_t kKernelServiceDillSize;
+}
+
 namespace dart {
 namespace bin {
 
+#if defined(DART_NO_SNAPSHOT) || defined(DART_PRECOMPILER)
+const uint8_t* kernel_service_dill = NULL;
+const intptr_t kernel_service_dill_size = 0;
+#else
+const uint8_t* kernel_service_dill = kKernelServiceDill;
+const intptr_t kernel_service_dill_size = kKernelServiceDillSize;
+#endif
+
+const char kKernelServiceSnapshot[] = "kernel-service.dart.snapshot";
+const char kSnapshotsDirectory[] = "snapshots";
 const char kPlatformBinaryName[] = "vm_platform.dill";
 const char kPlatformStrongBinaryName[] = "vm_platform_strong.dill";
 
+void* DFE::kKernelServiceProgram = NULL;
+
+static char* GetDirectoryPrefixFromExeName() {
+  const char* name = Platform::GetExecutableName();
+  const char* sep = File::PathSeparator();
+  // Locate the last occurance of |sep| in |name|.
+  intptr_t i;
+  for (i = strlen(name) - 1; i >= 0; --i) {
+    const char* str = name + i;
+    if (strstr(str, sep) == str) {
+      break;
+    }
+  }
+
+  if (i < 0) {
+    return strdup("");
+  }
+
+  return StringUtils::StrNDup(name, i + 1);
+}
+
 DFE::DFE()
-    : frontend_filename_(NULL),
+    : use_dfe_(false),
+      frontend_filename_(NULL),
       kernel_binaries_path_(NULL),
       platform_binary_filename_(NULL),
       kernel_platform_(NULL),
@@ -24,6 +64,9 @@ DFE::DFE()
       kernel_file_specified_(false) {}
 
 DFE::~DFE() {
+  if (frontend_filename_ != NULL) {
+    free(frontend_filename_);
+  }
   frontend_filename_ = NULL;
 
   free(kernel_binaries_path_);
@@ -39,6 +82,46 @@ DFE::~DFE() {
   application_kernel_binary_ = NULL;
 }
 
+char* DFE::FrontendFilename() {
+  if (frontend_filename_ == NULL) {
+    // Look for the frontend snapshot next to the executable.
+    char* dir_prefix = GetDirectoryPrefixFromExeName();
+    // |dir_prefix| includes the last path seperator.
+    frontend_filename_ =
+        OS::SCreate(NULL, "%s%s", dir_prefix, kKernelServiceSnapshot);
+
+    if (!File::Exists(NULL, frontend_filename_)) {
+      // If the frontend snapshot is not found next to the executable,
+      // then look for it in the "snapshots" directory.
+      free(frontend_filename_);
+      // |dir_prefix| includes the last path seperator.
+      frontend_filename_ =
+          OS::SCreate(NULL, "%s%s%s%s", dir_prefix, kSnapshotsDirectory,
+                      File::PathSeparator(), kKernelServiceSnapshot);
+    }
+
+    free(dir_prefix);
+    if (!File::Exists(NULL, frontend_filename_)) {
+      free(frontend_filename_);
+      frontend_filename_ = NULL;
+    }
+  }
+  return frontend_filename_;
+}
+
+static void NoopRelease(uint8_t* buffer) {}
+
+void* DFE::KernelServiceProgram() {
+  if (kernel_service_dill == NULL) {
+    return NULL;
+  }
+  if (kKernelServiceProgram == NULL) {
+    kKernelServiceProgram = Dart_ReadKernelBinary(
+        kernel_service_dill, kernel_service_dill_size, NoopRelease);
+  }
+  return kKernelServiceProgram;
+}
+
 void DFE::SetKernelBinaries(const char* name) {
   kernel_binaries_path_ = strdup(name);
 }

runtime/bin/dfe.h

@@ -18,9 +18,16 @@ class DFE {
   DFE();
   ~DFE();
 
-  const char* frontend_filename() const { return frontend_filename_; }
-  void set_frontend_filename(const char* name) { frontend_filename_ = name; }
-  bool UseDartFrontend() const { return frontend_filename_ != NULL; }
+  char* FrontendFilename();
+  void set_frontend_filename(const char* name) {
+    if (frontend_filename_ != NULL) {
+      free(frontend_filename_);
+    }
+    frontend_filename_ = strdup(name);
+    set_use_dfe();
+  }
+  void set_use_dfe() { use_dfe_ = true; }
+  bool UseDartFrontend() const { return use_dfe_; }
 
   const char* GetPlatformBinaryFilename();
 
@@ -64,6 +71,8 @@ class DFE {
   // valid kernel file, false otherwise.
   void* ReadScript(const char* script_uri) const;
 
+  static void* KernelServiceProgram();
+
  private:
   // Tries to read [script_uri] as a Kernel IR file.
   // Returns `true` if successful and sets [kernel_file] and [kernel_length]
@@ -74,7 +83,8 @@ class DFE {
                          const uint8_t** kernel_ir,
                          intptr_t* kernel_ir_size) const;
 
-  const char* frontend_filename_;
+  bool use_dfe_;
+  char* frontend_filename_;
   char* kernel_binaries_path_;
   char* platform_binary_filename_;
   void* kernel_platform_;
@@ -85,6 +95,8 @@ class DFE {
 
   bool kernel_file_specified_;  // Kernel file was specified on the cmd line.
 
+  static void* kKernelServiceProgram;
+
   DISALLOW_COPY_AND_ASSIGN(DFE);
 };
 

runtime/bin/main.cc

@@ -299,51 +299,60 @@ static Dart_Isolate IsolateSetupHelper(Dart_Isolate isolate,
 #if !defined(DART_PRECOMPILED_RUNTIME)
 // Returns newly created Kernel Isolate on success, NULL on failure.
 // For now we only support the kernel isolate coming up from an
-// application snapshot or from sources which are compiled by the
-// VM parser.
-static Dart_Isolate CreateAndSetupKernelIsolate(const char* main,
+// application snapshot or from a .dill file.
+static Dart_Isolate CreateAndSetupKernelIsolate(const char* script_uri,
+                                                const char* main,
                                                 const char* package_root,
                                                 const char* packages_config,
                                                 Dart_IsolateFlags* flags,
                                                 char** error,
                                                 int* exit_code) {
-  if (!dfe.UseDartFrontend()) {
-    *error = strdup("Kernel isolate not supported.");
-    return NULL;
-  }
-  const char* script_uri = dfe.frontend_filename();
+  const char* kernel_snapshot_uri = dfe.FrontendFilename();
+  const char* uri =
+      kernel_snapshot_uri != NULL ? kernel_snapshot_uri : script_uri;
+
   if (packages_config == NULL) {
     packages_config = Options::packages_file();
   }
 
-  // Kernel isolate uses an app snapshot or the core libraries snapshot.
+  Dart_Isolate isolate;
+  IsolateData* isolate_data = NULL;
   bool isolate_run_app_snapshot = false;
-  const uint8_t* isolate_snapshot_data = core_isolate_snapshot_data;
-  const uint8_t* isolate_snapshot_instructions =
-      core_isolate_snapshot_instructions;
-  AppSnapshot* app_snapshot = Snapshot::TryReadAppSnapshot(script_uri);
-  if (app_snapshot != NULL) {
-    isolate_run_app_snapshot = true;
-    const uint8_t* ignore_vm_snapshot_data;
-    const uint8_t* ignore_vm_snapshot_instructions;
-    app_snapshot->SetBuffers(
-        &ignore_vm_snapshot_data, &ignore_vm_snapshot_instructions,
-        &isolate_snapshot_data, &isolate_snapshot_instructions);
+  if (kernel_snapshot_uri != NULL) {
+    // Kernel isolate uses an app snapshot or the core libraries snapshot.
+    const uint8_t* isolate_snapshot_data = core_isolate_snapshot_data;
+    const uint8_t* isolate_snapshot_instructions =
+        core_isolate_snapshot_instructions;
+    AppSnapshot* app_snapshot = Snapshot::TryReadAppSnapshot(uri);
+    if (app_snapshot != NULL) {
+      isolate_run_app_snapshot = true;
+      const uint8_t* ignore_vm_snapshot_data;
+      const uint8_t* ignore_vm_snapshot_instructions;
+      app_snapshot->SetBuffers(
+          &ignore_vm_snapshot_data, &ignore_vm_snapshot_instructions,
+          &isolate_snapshot_data, &isolate_snapshot_instructions);
+    }
+    IsolateData* isolate_data =
+        new IsolateData(uri, package_root, packages_config, app_snapshot);
+    isolate = Dart_CreateIsolate(
+        DART_KERNEL_ISOLATE_NAME, main, isolate_snapshot_data,
+        isolate_snapshot_instructions, flags, isolate_data, error);
+  } else {
+    void* kernel_service_program = DFE::KernelServiceProgram();
+    IsolateData* isolate_data =
+        new IsolateData(uri, package_root, packages_config, NULL);
+    isolate_data->kernel_program = kernel_service_program;
+    isolate = Dart_CreateIsolateFromKernel(uri, main, kernel_service_program,
+                                           flags, isolate_data, error);
   }
 
-  IsolateData* isolate_data =
-      new IsolateData(script_uri, package_root, packages_config, app_snapshot);
-  Dart_Isolate isolate = Dart_CreateIsolate(
-      DART_KERNEL_ISOLATE_NAME, main, isolate_snapshot_data,
-      isolate_snapshot_instructions, flags, isolate_data, error);
   if (isolate == NULL) {
     delete isolate_data;
     return NULL;
   }
 
-  return IsolateSetupHelper(isolate, false, script_uri, package_root,
-                            packages_config, isolate_snapshot_data,
-                            isolate_run_app_snapshot, error, exit_code);
+  return IsolateSetupHelper(isolate, false, uri, package_root, packages_config,
+                            true, isolate_run_app_snapshot, error, exit_code);
 }
 #endif  // !defined(DART_PRECOMPILED_RUNTIME)
 
@@ -542,8 +551,9 @@ static Dart_Isolate CreateIsolateAndSetup(const char* script_uri,
   int exit_code = 0;
 #if !defined(DART_PRECOMPILED_RUNTIME)
   if (strcmp(script_uri, DART_KERNEL_ISOLATE_NAME) == 0) {
-    return CreateAndSetupKernelIsolate(main, package_root, package_config,
-                                       flags, error, &exit_code);
+    return CreateAndSetupKernelIsolate(script_uri, main, package_root,
+                                       package_config, flags, error,
+                                       &exit_code);
   }
 #endif  // !defined(DART_PRECOMPILED_RUNTIME)
   if (strcmp(script_uri, DART_VM_SERVICE_ISOLATE_NAME) == 0) {

runtime/bin/main_options.cc

@@ -66,6 +66,8 @@ CB_OPTIONS_LIST(CB_OPTION_DEFINITION)
 #if !defined(DART_PRECOMPILED_RUNTIME)
 DFE* Options::dfe_ = NULL;
 
+DEFINE_BOOL_OPTION_CB(preview_dart_2, { Options::dfe()->set_use_dfe(); });
+
 DEFINE_STRING_OPTION_CB(dfe, { Options::dfe()->set_frontend_filename(value); });
 
 DEFINE_STRING_OPTION_CB(kernel_binaries,

runtime/tools/dill_to_data_cc.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+# Script to create a cc file with uint8 array of bytes corresponding to the
+# content of given a dill file.
+
+import optparse
+import sys
+
+# Option => Help mapping.
+OPTION_MAP = {
+  'dill_file': 'Path to the input dill file.',
+  'data_symbol': 'The C name for the data array.',
+  'size_symbol': 'The C name for the data size variable.',
+  'output': 'Path to the generated cc file.',
+}
+
+
+def BuildOptionParser():
+  parser = optparse.OptionParser()
+  for opt, help_text in OPTION_MAP.iteritems():
+    parser.add_option('--%s' % opt, type='string', help=help_text)
+  return parser
+
+
+def ValidateOptions(options):
+  for opt in OPTION_MAP.keys():
+    if getattr(options, opt) is None:
+      sys.stderr.write('--%s option not specified.\n' % opt)
+      return False
+  return True
+
+
+def WriteData(input_filename, data_symbol, output_file):
+  output_file.write('uint8_t %s[] = {\n' % data_symbol)
+  with open(input_filename, 'rb') as f:
+    first = True
+    size = 0
+    for byte in f.read():
+      if first:
+        output_file.write('  %d' % ord(byte))
+        first = False
+      else:
+        output_file.write(',\n  %d' % ord(byte))
+      size += 1
+  output_file.write('\n};\n')
+  return size
+
+
+def WriteSize(size_symbol, size, output_file):
+  output_file.write('intptr_t %s = %d;\n' % (size_symbol, size))
+
+
+def Main():
+  opt_parser = BuildOptionParser()
+  (options, args) = opt_parser.parse_args()
+  if not ValidateOptions(options):
+    opt_parser.print_help()
+    return 1
+  if args:
+    sys.stderr.write('Unknown args: "%s"\n' % str(args))
+    parser.print_help()
+    return 1
+
+  with open(options.output, 'w') as output_file:
+    output_file.write('#include <stdint.h>\n')
+    output_file.write('extern "C" {\n')
+    size = WriteData(options.dill_file, options.data_symbol, output_file)
+    WriteSize(options.size_symbol, size, output_file)
+    output_file.write("}")
+
+
+if __name__ == '__main__':
+  sys.exit(Main())

sdk/BUILD.gn

@@ -174,6 +174,10 @@ _full_sdk_snapshots = [
     "utils_wrapper",
     "../utils/compiler:utils_wrapper",
   ],
+  [
+    "kernel-service",
+    "../utils/kernel-service",
+  ],
 ]
 
 # Libraries that go under lib/
@@ -565,9 +569,9 @@ copy("copy_dev_compiler_summary") {
   ]
   gen_dir = get_label_info("../utils/dartdevc:dartdevc_sdk", "target_gen_dir")
   sources = [
-    "$gen_dir/kernel/ddc_sdk.dill",
     # TODO(vsm): Remove post CFE.
     "$gen_dir/ddc_sdk.sum",
+    "$gen_dir/kernel/ddc_sdk.dill",
   ]
   outputs = [
     "$root_out_dir/dart-sdk/lib/_internal/{{source_file_part}}",
@@ -650,7 +654,8 @@ copy("copy_dev_compiler_js_amd_kernel") {
   deps = [
     "../utils/dartdevc:dartdevc_sdk_kernel_summary",
   ]
-  gen_dir = get_label_info("../utils/dartdevc:dartdevc_sdk_kernel_summary", "target_gen_dir")
+  gen_dir = get_label_info("../utils/dartdevc:dartdevc_sdk_kernel_summary",
+                           "target_gen_dir")
   sources = [
     "$gen_dir/kernel/amd/dart_sdk.js",
     "$gen_dir/kernel/amd/dart_sdk.js.map",
@@ -667,7 +672,8 @@ copy("copy_dev_compiler_js_common_kernel") {
   deps = [
     "../utils/dartdevc:dartdevc_sdk_kernel_summary",
   ]
-  gen_dir = get_label_info("../utils/dartdevc:dartdevc_sdk_kernel_summary", "target_gen_dir")
+  gen_dir = get_label_info("../utils/dartdevc:dartdevc_sdk_kernel_summary",
+                           "target_gen_dir")
   sources = [
     "$gen_dir/kernel/common/dart_sdk.js",
     "$gen_dir/kernel/common/dart_sdk.js.map",
@@ -684,7 +690,8 @@ copy("copy_dev_compiler_js_es6_kernel") {
   deps = [
     "../utils/dartdevc:dartdevc_sdk_kernel_summary",
   ]
-  gen_dir = get_label_info("../utils/dartdevc:dartdevc_sdk_kernel_summary", "target_gen_dir")
+  gen_dir = get_label_info("../utils/dartdevc:dartdevc_sdk_kernel_summary",
+                           "target_gen_dir")
   sources = [
     "$gen_dir/kernel/es6/dart_sdk.js",
     "$gen_dir/kernel/es6/dart_sdk.js.map",
@@ -700,7 +707,8 @@ copy("copy_dev_compiler_js_legacy_kernel") {
   deps = [
     "../utils/dartdevc:dartdevc_sdk_kernel_summary",
   ]
-  gen_dir = get_label_info("../utils/dartdevc:dartdevc_sdk_kernel_summary", "target_gen_dir")
+  gen_dir = get_label_info("../utils/dartdevc:dartdevc_sdk_kernel_summary",
+                           "target_gen_dir")
   sources = [
     "$gen_dir/kernel/legacy/dart_sdk.js",
     "$gen_dir/kernel/legacy/dart_sdk.js.map",
@@ -719,12 +727,12 @@ group("copy_dev_compiler_js") {
   ]
   deps = [
     ":copy_dev_compiler_js_amd",
-    ":copy_dev_compiler_js_common",
-    ":copy_dev_compiler_js_es6",
-    ":copy_dev_compiler_js_legacy",
     ":copy_dev_compiler_js_amd_kernel",
+    ":copy_dev_compiler_js_common",
     ":copy_dev_compiler_js_common_kernel",
+    ":copy_dev_compiler_js_es6",
     ":copy_dev_compiler_js_es6_kernel",
+    ":copy_dev_compiler_js_legacy",
     ":copy_dev_compiler_js_legacy_kernel",
   ]
 }

utils/kernel-service/BUILD.gn

@@ -5,6 +5,8 @@
 import("../../build/dart_host_sdk_toolchain.gni")
 import("../application_snapshot.gni")
 
+# TODO: Switch this to use kernel based app-jit snapshot
+# when we are ready to switch to the kernel based core snapshots.
 application_snapshot("kernel-service") {
   main_dart = "../../pkg/vm/bin/kernel_service.dart"
   deps = [
@@ -15,3 +17,47 @@ application_snapshot("kernel-service") {
     "file://" + rebase_path("../../pkg/compiler/lib/src/dart2js.dart"),
   ]
 }
+
+copy("copy_kernel_service_snapshot") {
+  deps = [
+    ":kernel-service",
+  ]
+  sources = [
+    rebase_path("$root_gen_dir/kernel-service.dart.snapshot"),
+  ]
+  outputs = [
+    "$root_out_dir/kernel-service.dart.snapshot",
+  ]
+}
+
+compiled_action("kernel_service_dill") {
+  deps = [
+    "../../runtime/vm:vm_legacy_platform",
+  ]
+  tool = "../../runtime/bin:dart_bootstrap"
+  kernel_service_script = "../../pkg/vm/bin/kernel_service.dart"
+  gen_kernel_script = "../../pkg/vm/bin/gen_kernel.dart"
+
+  inputs = [
+    gen_kernel_script,
+    kernel_service_script,
+    "$root_out_dir/vm_platform.dill",
+  ]
+  outputs = [
+    "$root_gen_dir/kernel_service.dill",
+  ]
+
+  # TODO: Switch over to vm_platform_strong.dill and remove the
+  # flags --no-strong-mode once https://github.com/dart-lang/sdk/issues/31623
+  # is fixed.
+  args = [
+    rebase_path(gen_kernel_script),
+    "--packages=file:///" + rebase_path("../../.packages"),
+    "--platform=file:///" + rebase_path("$root_out_dir/vm_platform.dill"),
+    "--no-aot",
+    "--no-strong-mode",
+    "--no-embed-sources",
+    "--output=" + rebase_path("$root_gen_dir/kernel_service.dill"),
+    "file:///" + rebase_path(kernel_service_script),
+  ]
+}