[ VM / Service ] Dynamically load source from linked-in platform kernel

- Added `Dart_SetDartLibrarySourcesKernel` to the Dart embedding API.
- vm_platform_strong.dill now contains the Dart SDK sources
- If vm_platform_strong.dill is linked into the Dart binary, the first
  request for a Dart SDK Script object from the VM service will load the
  sources for the script from the linked in kernel buffer.

Change-Id: I664abe31f9378d25ec79c21edce0b237a278495d
Reviewed-on: https://dart-review.googlesource.com/c/93375
Commit-Queue: Ben Konyi <bkonyi@google.com>
Reviewed-by: Siva Annamalai <asiva@google.com>

runtime/bin/BUILD.gn

@@ -434,7 +434,7 @@ dart_io("standalone_dart_io") {
 
 gen_snapshot_action("generate_snapshot_bin") {
   deps = [
-    "../vm:vm_platform",
+    "../vm:vm_platform_stripped",
   ]
   vm_snapshot_data = "$target_gen_dir/vm_snapshot_data.bin"
   vm_snapshot_instructions = "$target_gen_dir/vm_snapshot_instructions.bin"
@@ -442,7 +442,7 @@ gen_snapshot_action("generate_snapshot_bin") {
   isolate_snapshot_instructions =
       "$target_gen_dir/isolate_snapshot_instructions.bin"
 
-  platform_dill = "$root_out_dir/vm_platform_strong.dill"
+  platform_dill = "$root_out_dir/vm_platform_strong_stripped.dill"
   inputs = [
     platform_dill,
   ]

runtime/bin/dfe.cc

@@ -31,10 +31,6 @@ intptr_t kPlatformStrongDillSize = 0;
 namespace dart {
 namespace bin {
 
-#if !defined(DART_PRECOMPILED_RUNTIME)
-DFE dfe;
-#endif
-
 #if defined(DART_NO_SNAPSHOT) || defined(DART_PRECOMPILER)
 const uint8_t* kernel_service_dill = NULL;
 const intptr_t kernel_service_dill_size = 0;
@@ -47,6 +43,10 @@ const uint8_t* platform_strong_dill = kPlatformStrongDill;
 const intptr_t platform_strong_dill_size = kPlatformStrongDillSize;
 #endif
 
+#if !defined(DART_PRECOMPILED_RUNTIME)
+DFE dfe;
+#endif
+
 const char kKernelServiceSnapshot[] = "kernel-service.dart.snapshot";
 const char kSnapshotsDirectory[] = "snapshots";
 
@@ -94,6 +94,9 @@ void DFE::Init() {
     return;
   }
 
+  Dart_SetDartLibrarySourcesKernel(platform_strong_dill,
+                                   platform_strong_dill_size);
+
   if (frontend_filename_ == NULL) {
     // Look for the frontend snapshot next to the executable.
     char* dir_prefix = GetDirectoryPrefixFromExeName();

runtime/include/dart_api.h

@@ -3124,6 +3124,19 @@ Dart_CompileSourcesToKernel(const char* script_uri,
 
 DART_EXPORT Dart_KernelCompilationResult Dart_KernelListDependencies();
 
+/**
+ * Sets the kernel buffer which will be used to load Dart SDK sources
+ * dynamically at runtime.
+ *
+ * \param platform_kernel A buffer containing kernel which has sources for the
+ * Dart SDK populated. Note: The VM does not take ownership of this memory.
+ *
+ * \param platform_kernel_size The length of the platform_kernel buffer.
+ */
+DART_EXPORT void Dart_SetDartLibrarySourcesKernel(
+    const uint8_t* platform_kernel,
+    const intptr_t platform_kernel_size);
+
 #define DART_KERNEL_ISOLATE_NAME "kernel-service"
 
 /*

runtime/vm/BUILD.gn

@@ -98,47 +98,60 @@ library_for_all_configs("libdart_lib") {
   nosnapshot_sources = []
 }
 
-compile_platform("vm_legacy_platform") {
+template("gen_vm_platform") {
-  single_root_scheme = "org-dartlang-sdk"
+  assert(defined(invoker.output_postfix),
-  single_root_base = rebase_path("../../")
+         "Must define output postfix (e.g., '_strong'")
-  libraries_specification_uri = "org-dartlang-sdk:///sdk/lib/libraries.json"
+  compile_platform(target_name) {
-
+    output_postfix = invoker.output_postfix
-  outputs = [
+    if (defined(invoker.add_implicit_vm_platform_dependency)) {
-    "$root_out_dir/vm_platform.dill",
+      add_implicit_vm_platform_dependency =
-    "$root_out_dir/vm_outline.dill",
+          invoker.add_implicit_vm_platform_dependency
-  ]
+    }
-
+    single_root_scheme = "org-dartlang-sdk"
-  args = [
+    single_root_base = rebase_path("../../")
-    "--legacy-mode",
+    libraries_specification_uri = "org-dartlang-sdk:///sdk/lib/libraries.json"
-    "dart:core",
+    outputs = [
-  ]
+      "$root_out_dir/vm_platform" + output_postfix + ".dill",
+      "$root_out_dir/vm_outline" + output_postfix + ".dill",
+    ]
+    args = [ "dart:core" ]
+    if (defined(invoker.exclude_source) && invoker.exclude_source) {
+      args += [ "--exclude-source" ]
+    }
+    if (defined(invoker.legacy) && invoker.legacy) {
+      args += [ "--legacy-mode" ]
+      outline = "vm_outline_strong.dill"
+    } else {
+      outline = "vm_outline" + output_postfix + ".dill"
+    }
+    if (dart_platform_bytecode) {
+      args += [ "--bytecode" ]
+    }
+  }
 }
 
-compile_platform("vm_platform") {
+gen_vm_platform("vm_legacy_platform") {
+  exclude_source = false
+  legacy = true
+  output_postfix = ""
+}
+
+gen_vm_platform("vm_platform") {
   add_implicit_vm_platform_dependency = false
+  exclude_source = false
+  output_postfix = "_strong"
+}
 
-  single_root_scheme = "org-dartlang-sdk"
+gen_vm_platform("vm_platform_stripped") {
-  single_root_base = rebase_path("../../")
+  add_implicit_vm_platform_dependency = false
-  libraries_specification_uri = "org-dartlang-sdk:///sdk/lib/libraries.json"
+  exclude_source = true
-
+  output_postfix = "_strong_stripped"
-  outputs = [
-    "$root_out_dir/vm_platform_strong.dill",
-    "$root_out_dir/vm_outline_strong.dill",
-  ]
-
-  args = [
-    "--exclude-source",
-    "dart:core",
-  ]
-
-  if (dart_platform_bytecode) {
-    args += [ "--bytecode" ]
-  }
 }
 
 group("kernel_platform_files") {
   public_deps = [
     ":vm_legacy_platform",
     ":vm_platform",
+    ":vm_platform_stripped",
   ]
 }

runtime/vm/dart_api_impl.cc

@@ -5498,6 +5498,15 @@ DART_EXPORT Dart_KernelCompilationResult Dart_KernelListDependencies() {
   return result;
 }
 
+DART_EXPORT void Dart_SetDartLibrarySourcesKernel(
+    const uint8_t* platform_kernel,
+    const intptr_t platform_kernel_size) {
+#if !defined(PRODUCT)
+  Service::SetDartLibraryKernelForSources(platform_kernel,
+                                          platform_kernel_size);
+#endif
+}
+
 // --- Service support ---
 
 DART_EXPORT bool Dart_IsServiceIsolate(Dart_Isolate isolate) {

runtime/vm/kernel_loader.cc

@@ -283,6 +283,24 @@ void KernelLoader::index_programs(
   subprogram_file_starts->Reverse();
 }
 
+RawString* KernelLoader::FindSourceForScript(const uint8_t* kernel_buffer,
+                                             intptr_t kernel_buffer_length,
+                                             const String& uri) {
+  Thread* thread = Thread::Current();
+  Zone* zone = thread->zone();
+  TranslationHelper translation_helper(thread);
+  KernelReaderHelper reader(zone, &translation_helper, kernel_buffer,
+                            kernel_buffer_length, 0);
+  intptr_t source_table_size = reader.SourceTableSize();
+  for (intptr_t i = 0; i < source_table_size; ++i) {
+    const String& source_uri = reader.SourceTableUriFor(i);
+    if (source_uri.EndsWith(uri)) {
+      return reader.GetSourceFor(i).raw();
+    }
+  }
+  return String::null();
+}
+
 void KernelLoader::InitializeFields() {
   const intptr_t source_table_size = helper_.SourceTableSize();
   const Array& scripts =

runtime/vm/kernel_loader.h

@@ -176,6 +176,10 @@ class KernelLoader : public ValueObject {
                                     intptr_t* p_num_classes,
                                     intptr_t* p_num_procedures);
 
+  static RawString* FindSourceForScript(const uint8_t* kernel_buffer,
+                                        intptr_t kernel_buffer_length,
+                                        const String& url);
+
   RawLibrary* LoadLibrary(intptr_t index);
 
   void FinishTopLevelClassLoading(const Class& toplevel_class,

runtime/vm/object.cc

@@ -9086,6 +9086,25 @@ RawString* Script::Source() const {
   return raw_ptr()->source_;
 }
 
+bool Script::IsPartOfDartColonLibrary() const {
+  const String& script_url = String::Handle(url());
+  return (script_url.StartsWith(Symbols::DartScheme()) ||
+          script_url.StartsWith(Symbols::DartSchemePrivate()));
+}
+
+#if !defined(DART_PRECOMPILED_RUNTIME)
+void Script::LoadSourceFromKernel(const uint8_t* kernel_buffer,
+                                  intptr_t kernel_buffer_len) const {
+  const char* dart_prefix = "dart:";
+  const size_t dart_prefix_len = strlen(dart_prefix);
+  String& uri = String::Handle(url());
+  uri ^= String::SubString(uri, dart_prefix_len);
+  String& source = String::Handle(kernel::KernelLoader::FindSourceForScript(
+      kernel_buffer, kernel_buffer_len, uri));
+  set_source(source);
+}
+#endif  // !defined(DART_PRECOMPILED_RUNTIME)
+
 void Script::set_compile_time_constants(const Array& value) const {
   StorePointer(&raw_ptr()->compile_time_constants_, value.raw());
 }
@@ -18922,6 +18941,25 @@ bool String::StartsWith(const String& other) const {
   return true;
 }
 
+bool String::EndsWith(const String& other) const {
+  if (other.IsNull()) {
+    return false;
+  }
+  const intptr_t len = this->Length();
+  const intptr_t other_len = other.Length();
+  const intptr_t offset = len - other_len;
+
+  if ((other_len == 0) || (other_len > len)) {
+    return false;
+  }
+  for (int i = offset; i < len; i++) {
+    if (this->CharAt(i) != other.CharAt(i - offset)) {
+      return false;
+    }
+  }
+  return true;
+}
+
 RawInstance* String::CheckAndCanonicalize(Thread* thread,
                                           const char** error_str) const {
   if (IsCanonical()) {

runtime/vm/object.h

@@ -3385,6 +3385,8 @@ class Script : public Object {
   RawString* resolved_url() const { return raw_ptr()->resolved_url_; }
   bool HasSource() const;
   RawString* Source() const;
+  bool IsPartOfDartColonLibrary() const;
+
   RawGrowableObjectArray* GenerateLineNumberArray() const;
   RawScript::Kind kind() const {
     return static_cast<RawScript::Kind>(raw_ptr()->kind_);
@@ -3461,6 +3463,11 @@ class Script : public Object {
                         const String& source,
                         RawScript::Kind kind);
 
+#if !defined(DART_PRECOMPILED_RUNTIME)
+  void LoadSourceFromKernel(const uint8_t* kernel_buffer,
+                            intptr_t kernel_buffer_len) const;
+#endif  // !defined(DART_PRECOMPILED_RUNTIME)
+
  private:
   void set_resolved_url(const String& value) const;
   void set_source(const String& value) const;
@@ -7006,6 +7013,7 @@ class String : public Instance {
   intptr_t CompareTo(const String& other) const;
 
   bool StartsWith(const String& other) const;
+  bool EndsWith(const String& other) const;
 
   // Strings are canonicalized using the symbol table.
   virtual RawInstance* CheckAndCanonicalize(Thread* thread,

runtime/vm/service.cc

@@ -121,6 +121,9 @@ StreamInfo Service::logging_stream("_Logging");
 StreamInfo Service::extension_stream("Extension");
 StreamInfo Service::timeline_stream("Timeline");
 
+const uint8_t* Service::dart_library_kernel_ = NULL;
+intptr_t Service::dart_library_kernel_len_ = 0;
+
 static StreamInfo* streams_[] = {
     &Service::vm_stream,      &Service::isolate_stream,
     &Service::debug_stream,   &Service::gc_stream,
@@ -1328,6 +1331,12 @@ int64_t Service::MaxRSS() {
   return info.max_rss;
 }
 
+void Service::SetDartLibraryKernelForSources(const uint8_t* kernel_bytes,
+                                             intptr_t kernel_length) {
+  dart_library_kernel_ = kernel_bytes;
+  dart_library_kernel_len_ = kernel_length;
+}
+
 EmbedderServiceHandler* Service::FindRootEmbedderHandler(const char* name) {
   EmbedderServiceHandler* current = root_service_handler_head_;
   while (current != NULL) {
@@ -4322,9 +4331,22 @@ static bool GetObject(Thread* thread, JSONStream* js) {
 
   // Handle heap objects.
   ObjectIdRing::LookupResult lookup_result;
-  const Object& obj =
+  Object& obj = Object::Handle(LookupHeapObject(thread, id, &lookup_result));
-      Object::Handle(LookupHeapObject(thread, id, &lookup_result));
   if (obj.raw() != Object::sentinel().raw()) {
+#if !defined(DART_PRECOMPILED_RUNTIME)
+    // If obj is a script from dart:* and doesn't have source loaded, try and
+    // load the source before sending the response.
+    if (obj.IsScript()) {
+      const Script& script = Script::Cast(obj);
+      if (!script.HasSource() && script.IsPartOfDartColonLibrary() &&
+          Service::HasDartLibraryKernelForSources()) {
+        const uint8_t* kernel_buffer = Service::dart_library_kernel();
+        const intptr_t kernel_buffer_len =
+            Service::dart_library_kernel_length();
+        script.LoadSourceFromKernel(kernel_buffer, kernel_buffer_len);
+      }
+    }
+#endif  // !defined(DART_PRECOMPILED_RUNTIME)
     // We found a heap object for this id.  Return it.
     obj.PrintJSON(js, false);
     return true;

runtime/vm/service.h

@@ -184,6 +184,18 @@ class Service : public AllStatic {
   static int64_t CurrentRSS();
   static int64_t MaxRSS();
 
+  static void SetDartLibraryKernelForSources(const uint8_t* kernel_bytes,
+                                             intptr_t kernel_length);
+  static bool HasDartLibraryKernelForSources() {
+    return (dart_library_kernel_ != NULL);
+  }
+
+  static const uint8_t* dart_library_kernel() { return dart_library_kernel_; }
+
+  static intptr_t dart_library_kernel_length() {
+    return dart_library_kernel_len_;
+  }
+
  private:
   static RawError* InvokeMethod(Isolate* isolate,
                                 const Array& message,
@@ -228,11 +240,8 @@ class Service : public AllStatic {
   static Dart_GetVMServiceAssetsArchive get_service_assets_callback_;
   static Dart_EmbedderInformationCallback embedder_information_callback_;
 
-  static bool needs_isolate_events_;
+  static const uint8_t* dart_library_kernel_;
-  static bool needs_debug_events_;
+  static intptr_t dart_library_kernel_len_;
-  static bool needs_gc_events_;
-  static bool needs_echo_events_;
-  static bool needs_graph_events_;
 };
 
 }  // namespace dart

utils/compile_platform.gni

@@ -34,6 +34,11 @@ template("compile_platform") {
         invoker.add_implicit_vm_platform_dependency
   }
 
+  outline = "vm_outline_strong.dill"
+  if (defined(invoker.outline)) {
+    outline = invoker.outline
+  }
+
   prebuilt_dart_action(target_name) {
     script = "$_dart_root/pkg/front_end/tool/_fasta/compile_platform.dart"
 
@@ -44,7 +49,6 @@ template("compile_platform") {
     inputs = []
     deps = []
     args = []
-
     if (defined(invoker.deps)) {
       deps += invoker.deps
     }
@@ -54,7 +58,7 @@ template("compile_platform") {
     }
 
     if (add_implicit_vm_platform_dependency) {
-      inputs += [ "$root_out_dir/vm_outline_strong.dill" ]
+      inputs += [ "$root_out_dir/$outline" ]
       deps += [ "$_dart_root/runtime/vm:vm_platform" ]
     }
     depfile = outputs[0] + ".d"
@@ -73,7 +77,7 @@ template("compile_platform") {
           [ rebase_path(invoker.libraries_specification_uri, root_build_dir) ]
     }
     args +=
-        [ rebase_path("$root_out_dir/vm_outline_strong.dill", root_build_dir) ]
+        [ rebase_path("$root_out_dir/$outline", root_build_dir) ]
     args += rebase_path(outputs, root_build_dir)
   }
 }