[kernel] Support native extensions in the VM through Kernel.

Change-Id: I860e66b3e66a882ff771e477c318362cefbd4eaa
Reviewed-on: https://dart-review.googlesource.com/35925
Commit-Queue: Samir Jindel <sjindel@google.com>
Reviewed-by: Siva Annamalai <asiva@google.com>

pkg/front_end/testcases/compile.status

@@ -122,3 +122,4 @@ rasta/foo: RuntimeError # Expected, this file has no main method.
 incomplete_field_formal_parameter: Fail # Fasta doesn't recover well
 
 co19_language_metadata_syntax_t04: RuntimeError # Fasta doesn't recover well
+external_import: RuntimeError # Expected -- test uses import which doesn't exist.

pkg/front_end/testcases/strong.status

@@ -218,3 +218,5 @@ rasta/foo: RuntimeError # Expected, this file has no main method.
 incomplete_field_formal_parameter: Fail # Fasta doesn't recover well
 
 co19_language_metadata_syntax_t04: RuntimeError # Fasta doesn't recover well
+
+external_import: RuntimeError # The native extension to import doesn't exist. This is ok.

runtime/bin/loader.cc

@@ -700,6 +700,19 @@ Dart_Handle Loader::LibraryTagHandler(Dart_LibraryTag tag,
     ASSERT(!Dart_IsServiceIsolate(current) && !Dart_IsKernelIsolate(current));
     return dfe.ReadKernelBinary(current, url_string);
   }
+  if (tag == Dart_kImportResolvedExtensionTag) {
+    if (strncmp(url_string, "file://", 7)) {
+      return DartUtils::NewError(
+          "Resolved native extensions must use the file:// scheme.");
+    }
+    const char* absolute_path = DartUtils::RemoveScheme(url_string);
+
+    if (!File::IsAbsolutePath(absolute_path)) {
+      return DartUtils::NewError("Native extension path must be absolute.");
+    }
+
+    return Extensions::LoadExtension("/", absolute_path, library);
+  }
   ASSERT(Dart_IsKernelIsolate(Dart_CurrentIsolate()) || !dfe.UseDartFrontend());
   if (tag != Dart_kScriptTag) {
     // Special case for handling dart: imports and parts.

runtime/include/dart_api.h

@@ -2703,6 +2703,7 @@ typedef enum {
   Dart_kSourceTag,
   Dart_kImportTag,
   Dart_kKernelTag,
+  Dart_kImportResolvedExtensionTag,
 } Dart_LibraryTag;
 
 /**
@@ -2751,6 +2752,13 @@ typedef enum {
  * The dart front end typically compiles all the scripts, imports and part
  * files into one intermediate file hence we don't use the source/import or
  * script tags.
+ *
+ * Dart_kImportResolvedExtensionTag
+ *
+ * This tag is used to load an external import (shared object file) without
+ * performing path resolution first. The 'url' provided should be an absolute
+ * path with the 'file://' schema. It doesn't require the service isolate to be
+ * available and will not initialize a Loader for the isolate.
  */
 typedef Dart_Handle (*Dart_LibraryTagHandler)(
     Dart_LibraryTag tag,

runtime/vm/kernel_loader.cc

@@ -177,7 +177,8 @@ KernelLoader::KernelLoader(Program* program)
                0),
       external_name_class_(Class::Handle(Z)),
       external_name_field_(Field::Handle(Z)),
-      potential_natives_(GrowableObjectArray::Handle(Z)) {
+      potential_natives_(GrowableObjectArray::Handle(Z)),
+      potential_extension_libraries_(GrowableObjectArray::Handle(Z)) {
   if (!program->is_single_program()) {
     FATAL(
         "Trying to load a concatenated dill file at a time where that is "
@@ -349,7 +350,8 @@ KernelLoader::KernelLoader(const Script& script,
       builder_(&translation_helper_, script, zone_, kernel_data, 0),
       external_name_class_(Class::Handle(Z)),
       external_name_field_(Field::Handle(Z)),
-      potential_natives_(GrowableObjectArray::Handle(Z)) {
+      potential_natives_(GrowableObjectArray::Handle(Z)),
+      potential_extension_libraries_(GrowableObjectArray::Handle(Z)) {
   T.active_class_ = &active_class_;
   T.finalize_ = false;
 
@@ -435,6 +437,112 @@ void KernelLoader::AnnotateNativeProcedures(const Array& constant_table) {
   }
 }
 
+RawString* KernelLoader::DetectExternalName() {
+  builder_.ReadTag();
+  builder_.ReadPosition();
+  NameIndex annotation_class = H.EnclosingName(
+      builder_.ReadCanonicalNameReference());  // read target reference,
+  ASSERT(H.IsClass(annotation_class));
+  StringIndex class_name_index = H.CanonicalNameString(annotation_class);
+
+  // Just compare by name, do not generate the annotation class.
+  if (!H.StringEquals(class_name_index, "ExternalName")) {
+    builder_.SkipArguments();
+    return String::null();
+  }
+  ASSERT(H.IsLibrary(H.CanonicalNameParent(annotation_class)));
+  StringIndex library_name_index =
+      H.CanonicalNameString(H.CanonicalNameParent(annotation_class));
+  if (!H.StringEquals(library_name_index, "dart:_internal")) {
+    builder_.SkipArguments();
+    return String::null();
+  }
+
+  // Read arguments:
+  intptr_t total_arguments = builder_.ReadUInt();  // read argument count.
+  builder_.SkipListOfDartTypes();                  // read list of types.
+  intptr_t positional_arguments = builder_.ReadListLength();
+  ASSERT(total_arguments == 1 && positional_arguments == 1);
+
+  Tag tag = builder_.ReadTag();
+  ASSERT(tag == kStringLiteral);
+  String& result = H.DartSymbol(
+      builder_.ReadStringReference());  // read index into string table.
+
+  // List of named.
+  intptr_t list_length = builder_.ReadListLength();  // read list length.
+  ASSERT(list_length == 0);
+
+  return result.raw();
+}
+
+void KernelLoader::LoadNativeExtensionLibraries(const Array& constant_table) {
+  const intptr_t length = !potential_extension_libraries_.IsNull()
+                              ? potential_extension_libraries_.Length()
+                              : 0;
+  if (length == 0) return;
+
+  // Obtain `dart:_internal::ExternalName.name`.
+  EnsureExternalClassIsLookedUp();
+
+  Instance& constant = Instance::Handle(Z);
+  String& uri_path = String::Handle(Z);
+  Library& library = Library::Handle(Z);
+  Object& result = Object::Handle(Z);
+
+  for (intptr_t i = 0; i < length; ++i) {
+    library ^= potential_extension_libraries_.At(i);
+    builder_.SetOffset(library.kernel_offset());
+
+    LibraryHelper library_helper(&builder_);
+    library_helper.ReadUntilExcluding(LibraryHelper::kAnnotations);
+
+    const intptr_t annotation_count = builder_.ReadListLength();
+    for (intptr_t j = 0; j < annotation_count; ++j) {
+      uri_path = String::null();
+
+      const intptr_t tag = builder_.PeekTag();
+      if (tag == kConstantExpression) {
+        builder_.ReadByte();  // Skip the tag.
+
+        const intptr_t constant_table_index = builder_.ReadUInt();
+        constant ^= constant_table.At(constant_table_index);
+        if (constant.clazz() == external_name_class_.raw()) {
+          uri_path ^= constant.GetField(external_name_field_);
+        }
+      } else if (tag == kConstructorInvocation ||
+                 tag == kConstConstructorInvocation) {
+        uri_path = DetectExternalName();
+      } else {
+        builder_.SkipExpression();
+      }
+
+      if (uri_path.IsNull()) continue;
+
+      Dart_LibraryTagHandler handler = I->library_tag_handler();
+      if (handler == NULL) {
+        H.ReportError("no library handler registered.");
+      }
+
+      I->BlockClassFinalization();
+      {
+        TransitionVMToNative transition(thread_);
+        Api::Scope api_scope(thread_);
+        Dart_Handle retval = handler(Dart_kImportResolvedExtensionTag,
+                                     Api::NewHandle(thread_, library.raw()),
+                                     Api::NewHandle(thread_, uri_path.raw()));
+        result = Api::UnwrapHandle(retval);
+      }
+      I->UnblockClassFinalization();
+
+      if (result.IsError()) {
+        H.ReportError(Error::Cast(result), "library handler failed");
+      }
+    }
+  }
+  potential_extension_libraries_ = GrowableObjectArray::null();
+}
+
 Object& KernelLoader::LoadProgram(bool process_pending_classes) {
   ASSERT(kernel_program_info_.constants() == Array::null());
 
@@ -468,9 +576,10 @@ Object& KernelLoader::LoadProgram(bool process_pending_classes) {
     //     b) set the native names for native functions which have been created
     //        so far (the rest will be directly set during LoadProcedure)
     AnnotateNativeProcedures(constants);
-    ASSERT(kernel_program_info_.constants() == Array::null());
+    LoadNativeExtensionLibraries(constants);
 
     //     c) update all scripts with the constants array
+    ASSERT(kernel_program_info_.constants() == Array::null());
     kernel_program_info_.set_constants(constants);
 
     NameIndex main = program_->main_method();
@@ -629,6 +738,17 @@ void KernelLoader::LoadLibrary(intptr_t index) {
   const Script& script = Script::Handle(
       Z, ScriptAt(library_helper.source_uri_index_, import_uri_index));
 
+  library_helper.ReadUntilExcluding(LibraryHelper::kAnnotations);
+  intptr_t annotation_count = builder_.ReadListLength();  // read list length.
+  if (annotation_count > 0) {
+    EnsurePotentialExtensionLibraries();
+    potential_extension_libraries_.Add(library);
+  }
+  for (intptr_t i = 0; i < annotation_count; ++i) {
+    builder_.SkipExpression();  // read ith annotation.
+  }
+  library_helper.SetJustRead(LibraryHelper::kAnnotations);
+
   library_helper.ReadUntilExcluding(LibraryHelper::kDependencies);
   LoadLibraryImportsAndExports(&library);
   library_helper.SetJustRead(LibraryHelper::kDependencies);
@@ -1072,46 +1192,16 @@ void KernelLoader::LoadProcedure(const Library& library,
     for (int i = 0; i < annotation_count; ++i) {
       const intptr_t tag = builder_.PeekTag();
       if (tag == kConstructorInvocation || tag == kConstConstructorInvocation) {
-        builder_.ReadTag();
-        builder_.ReadPosition();
-        NameIndex annotation_class = H.EnclosingName(
-            builder_.ReadCanonicalNameReference());  // read target reference,
-        ASSERT(H.IsClass(annotation_class));
-        StringIndex class_name_index = H.CanonicalNameString(annotation_class);
-        // Just compare by name, do not generate the annotation class.
-        if (!H.StringEquals(class_name_index, "ExternalName")) {
-          builder_.SkipArguments();
-          continue;
-        }
-        ASSERT(H.IsLibrary(H.CanonicalNameParent(annotation_class)));
-        StringIndex library_name_index =
-            H.CanonicalNameString(H.CanonicalNameParent(annotation_class));
-        if (!H.StringEquals(library_name_index, "dart:_internal")) {
-          builder_.SkipArguments();
-          continue;
-        }
+        String& detected_name = String::Handle(DetectExternalName());
+        if (detected_name.IsNull()) continue;
 
         is_external = false;
-        // Read arguments:
-        intptr_t total_arguments = builder_.ReadUInt();  // read argument count.
-        builder_.SkipListOfDartTypes();                  // read list of types.
-        intptr_t positional_arguments = builder_.ReadListLength();
-        ASSERT(total_arguments == 1 && positional_arguments == 1);
-
-        Tag tag = builder_.ReadTag();
-        ASSERT(tag == kStringLiteral);
-        native_name = &H.DartSymbol(
-            builder_.ReadStringReference());  // read index into string table.
-
-        // List of named.
-        intptr_t list_length = builder_.ReadListLength();  // read list length.
-        ASSERT(list_length == 0);
+        native_name = &detected_name;
 
         // Skip remaining annotations
         for (++i; i < annotation_count; ++i) {
           builder_.SkipExpression();  // read ith annotation.
         }
-        break;
       } else if (tag == kConstantExpression) {
         if (kernel_program_info_.constants() == Array::null()) {
           // We can only read in the constant table once all classes have been

runtime/vm/kernel_loader.h

@@ -141,7 +141,9 @@ class KernelLoader {
   static void FinishLoading(const Class& klass);
 
   const Array& ReadConstantTable();
+  RawString* DetectExternalName();
   void AnnotateNativeProcedures(const Array& constant_table);
+  void LoadNativeExtensionLibraries(const Array& constant_table);
 
   const String& DartSymbol(StringIndex index) {
     return translation_helper_.DartSymbol(index);
@@ -252,6 +254,12 @@ class KernelLoader {
     }
   }
 
+  void EnsurePotentialExtensionLibraries() {
+    if (potential_extension_libraries_.IsNull()) {
+      potential_extension_libraries_ = GrowableObjectArray::New();
+    }
+  }
+
   Program* program_;
 
   Thread* thread_;
@@ -277,6 +285,7 @@ class KernelLoader {
   Class& external_name_class_;
   Field& external_name_field_;
   GrowableObjectArray& potential_natives_;
+  GrowableObjectArray& potential_extension_libraries_;
 
   Mapping<Library> libraries_;
   Mapping<Class> classes_;

tests/standalone_2/io/test_extension_fail_test.dart

@@ -24,7 +24,7 @@ Future copyFileToDirectory(String file, String directory) {
 String getExtensionPath(String buildDirectory) {
   switch (Platform.operatingSystem) {
     case 'linux':
-      return join(buildDirectory, 'lib.target', 'libtest_extension.so');
+      return join(buildDirectory, 'libtest_extension.so');
     case 'macos':
       return join(buildDirectory, 'libtest_extension.dylib');
     case 'windows':
@@ -46,7 +46,7 @@ bool checkStdError(String err) {
 }
 
 // name is either "extension" or "relative_extension"
-Future test(String name, bool checkForBall) {
+Future test(String name, bool checkForBall) async {
   String scriptDirectory = dirname(Platform.script.toFilePath());
   String buildDirectory = dirname(Platform.executable);
   Directory tempDirectory =
@@ -55,18 +55,25 @@ Future test(String name, bool checkForBall) {
 
   // Copy test_extension shared library, test_extension.dart and
   // test_extension_fail_tester.dart to the temporary test directory.
-  copyFileToDirectory(getExtensionPath(buildDirectory), testDirectory)
-      .then((_) {
+  try {
+    if (name == "extension") {
+      print(getExtensionPath(buildDirectory));
+      await copyFileToDirectory(
+          getExtensionPath(buildDirectory), testDirectory);
+    } else {
+      var extensionDir = testDirectory + "/extension";
+      Directory dir = await (new Directory(extensionDir).create());
+      await copyFileToDirectory(getExtensionPath(buildDirectory), extensionDir);
+    }
     var extensionDartFile = join(scriptDirectory, 'test_${name}.dart');
-    return copyFileToDirectory(extensionDartFile, testDirectory);
-  }).then((_) {
+    await copyFileToDirectory(extensionDartFile, testDirectory);
     var testExtensionTesterFile =
         join(scriptDirectory, 'test_${name}_fail_tester.dart');
-    return copyFileToDirectory(testExtensionTesterFile, testDirectory);
-  }).then((_) {
-    var script = join(testDirectory, 'test_${name}_fail_tester.dart');
-    return Process.run(Platform.executable, ['--trace-loading', script]);
-  }).then((ProcessResult result) {
+    await copyFileToDirectory(testExtensionTesterFile, testDirectory);
+    var args = new List<String>.from(Platform.executableArguments)
+      ..add('--trace-loading')
+      ..add(join(testDirectory, 'test_${name}_fail_tester.dart'));
+    var result = await Process.run(Platform.executable, args);
     print("ERR: ${result.stderr}\n\n");
     print("OUT: ${result.stdout}\n\n");
     if (!checkExitCode(result.exitCode)) {
@@ -80,7 +87,9 @@ Future test(String name, bool checkForBall) {
         throw new StateError("stderr doesn't contain 'ball'.");
       }
     }
-  }).whenComplete(() => tempDirectory.deleteSync(recursive: true));
+  } finally {
+    await tempDirectory.deleteSync(recursive: true);
+  }
 }
 
 main() async {

tests/standalone_2/io/test_extension_test.dart

@@ -40,16 +40,7 @@ List<String> getExtensionNames(String arch) {
 }
 
 String getExtensionPath(String buildDirectory, String filename) {
-  switch (Platform.operatingSystem) {
-    case 'android':
-    case 'linux':
-      return join(buildDirectory, 'lib.target', filename);
-    case 'macos':
-    case 'windows':
-      return join(buildDirectory, filename);
-    default:
-      Expect.fail('Unknown operating system ${Platform.operatingSystem}');
-  }
+  return join(buildDirectory, filename);
 }
 
 String getArchFromBuildDir(String buildDirectory) {
@@ -64,7 +55,7 @@ String getArchFromBuildDir(String buildDirectory) {
   return 'unknown';
 }
 
-Future testExtension(bool withArchSuffix) {
+Future testExtension(bool withArchSuffix) async {
   String scriptDirectory = dirname(Platform.script.toFilePath());
   String buildDirectory = dirname(Platform.executable);
   Directory tempDirectory =
@@ -79,34 +70,34 @@ Future testExtension(bool withArchSuffix) {
     fileNames = getExtensionNames('');
   }
 
-  // Copy test_extension shared library, test_extension.dart and
-  // test_extension_tester.dart to the temporary test directory.
-  return copyFileToDirectory(getExtensionPath(buildDirectory, fileNames[0]),
-      join(testDirectory, fileNames[1])).then((_) {
+  try {
+    // Copy test_extension shared library, test_extension.dart and
+    // test_extension_tester.dart to the temporary test directory.
+    await copyFileToDirectory(getExtensionPath(buildDirectory, fileNames[0]),
+        join(testDirectory, fileNames[1]));
+
     var extensionDartFile = join(scriptDirectory, 'test_extension.dart');
-    return copyFileToDirectory(extensionDartFile, testDirectory);
-  }).then((_) {
+    await copyFileToDirectory(extensionDartFile, testDirectory);
+
     var testExtensionTesterFile =
         join(scriptDirectory, 'test_extension_tester.dart');
-    return copyFileToDirectory(testExtensionTesterFile, testDirectory);
-  }).then<ProcessResult>((_) {
-    var script = join(testDirectory, 'test_extension_tester.dart');
-    return Process.run(Platform.executable, [script]);
-  })
-    ..then((ProcessResult result) {
-      if (result.exitCode != 0) {
-        print('Subprocess failed with exit code ${result.exitCode}');
-        print('stdout:');
-        print('${result.stdout}');
-        print('stderr:');
-        print('${result.stderr}');
-      }
-      Expect.equals(0, result.exitCode);
-      tempDirectory.deleteSync(recursive: true);
-    })
-    ..catchError((_) {
-      tempDirectory.deleteSync(recursive: true);
-    });
+    await copyFileToDirectory(testExtensionTesterFile, testDirectory);
+
+    var args = new List<String>.from(Platform.executableArguments)
+      ..add(join(testDirectory, 'test_extension_tester.dart'));
+    ProcessResult result = await Process.run(Platform.executable, args);
+
+    if (result.exitCode != 0) {
+      print('Subprocess failed with exit code ${result.exitCode}');
+      print('stdout:');
+      print('${result.stdout}');
+      print('stderr:');
+      print('${result.stderr}');
+    }
+    Expect.equals(0, result.exitCode);
+  } finally {
+    tempDirectory.deleteSync(recursive: true);
+  }
 }
 
 Future testWithArchSuffix() {