[vm/bytecode] Support native extensions

Change-Id: I224f740db674e20d643191605d0a7463bf85d39f
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/107451
Commit-Queue: Alexander Markov <alexmarkov@google.com>
Reviewed-by: Ryan Macnak <rmacnak@google.com>

pkg/kernel/lib/external_name.dart

@@ -20,16 +20,36 @@ String getExternalName(Member procedure) {
     return null;
   }
   for (final Expression annotation in procedure.annotations) {
-    if (annotation is ConstructorInvocation) {
-      if (_isExternalName(annotation.target.enclosingClass)) {
-        return (annotation.arguments.positional.single as StringLiteral).value;
-      }
-    } else if (annotation is ConstantExpression) {
-      final constant = annotation.constant;
-      if (constant is InstanceConstant) {
-        if (_isExternalName(constant.classNode)) {
-          return (constant.fieldValues.values.single as StringConstant).value;
-        }
+    final value = _getExternalNameValue(annotation);
+    if (value != null) {
+      return value;
+    }
+  }
+  return null;
+}
+
+/// Returns native extension URIs for given [library].
+List<String> getNativeExtensionUris(Library library) {
+  final uris = <String>[];
+  for (var annotation in library.annotations) {
+    final value = _getExternalNameValue(annotation);
+    if (value != null) {
+      uris.add(value);
+    }
+  }
+  return uris;
+}
+
+String _getExternalNameValue(Expression annotation) {
+  if (annotation is ConstructorInvocation) {
+    if (_isExternalName(annotation.target.enclosingClass)) {
+      return (annotation.arguments.positional.single as StringLiteral).value;
+    }
+  } else if (annotation is ConstantExpression) {
+    final constant = annotation.constant;
+    if (constant is InstanceConstant) {
+      if (_isExternalName(constant.classNode)) {
+        return (constant.fieldValues.values.single as StringConstant).value;
       }
     }
   }

pkg/vm/lib/bytecode/dbc.dart

@@ -10,7 +10,7 @@ library vm.bytecode.dbc;
 /// Before bumping current bytecode version format, make sure that
 /// all users have switched to a VM which is able to consume new
 /// version of bytecode.
-const int currentBytecodeFormatVersion = 11;
+const int currentBytecodeFormatVersion = 12;
 
 enum Opcode {
   kUnusedOpcode000,

pkg/vm/lib/bytecode/declarations.dart

@@ -18,21 +18,26 @@ import 'source_positions.dart' show LineStarts, SourcePositions;
 class LibraryDeclaration {
   static const usesDartMirrorsFlag = 1 << 0;
   static const usesDartFfiFlag = 1 << 1;
+  static const hasExtensionsFlag = 1 << 2;
 
   ObjectHandle importUri;
   final int flags;
   final ObjectHandle name;
   final ObjectHandle script;
+  final List<ObjectHandle> extensionUris;
   final List<ClassDeclaration> classes;
 
-  LibraryDeclaration(
-      this.importUri, this.flags, this.name, this.script, this.classes);
+  LibraryDeclaration(this.importUri, this.flags, this.name, this.script,
+      this.extensionUris, this.classes);
 
   void write(BufferedWriter writer) {
     final start = writer.offset;
     writer.writePackedUInt30(flags);
     writer.writePackedObject(name);
     writer.writePackedObject(script);
+    if ((flags & hasExtensionsFlag) != 0) {
+      writer.writePackedList(extensionUris);
+    }
     writer.writePackedUInt30(classes.length);
     for (var cls in classes) {
       writer.writePackedObject(cls.name);
@@ -50,7 +55,11 @@ class LibraryDeclaration {
       final className = reader.readPackedObject();
       return reader.readLinkOffset<ClassDeclaration>()..name = className;
     });
-    return new LibraryDeclaration(null, flags, name, script, classes);
+    final extensionUris = ((flags & hasExtensionsFlag) != 0)
+        ? reader.readPackedList<ObjectHandle>()
+        : const <ObjectHandle>[];
+    return new LibraryDeclaration(
+        null, flags, name, script, extensionUris, classes);
   }
 
   @override
@@ -65,6 +74,9 @@ class LibraryDeclaration {
     if ((flags & usesDartFfiFlag) != 0) {
       sb.writeln('    uses dart:ffi');
     }
+    if ((flags & hasExtensionsFlag) != 0) {
+      sb.writeln('    extensions: $extensionUris');
+    }
     sb.writeln();
     for (var cls in classes) {
       sb.write(cls);

pkg/vm/lib/bytecode/gen_bytecode.dart

@@ -20,7 +20,8 @@ import 'package:kernel/ast.dart' hide MapEntry, Component, FunctionDeclaration;
 import 'package:kernel/ast.dart' as ast show Component, FunctionDeclaration;
 import 'package:kernel/class_hierarchy.dart' show ClassHierarchy;
 import 'package:kernel/core_types.dart' show CoreTypes;
-import 'package:kernel/external_name.dart' show getExternalName;
+import 'package:kernel/external_name.dart'
+    show getExternalName, getNativeExtensionUris;
 import 'package:kernel/library_index.dart' show LibraryIndex;
 import 'package:kernel/target/targets.dart' show ConstantsBackend;
 import 'package:kernel/type_algebra.dart'
@@ -257,7 +258,14 @@ class BytecodeGenerator extends RecursiveVisitor<Null> {
     }
     final name = objectTable.getNameHandle(null, library.name ?? '');
     final script = getScript(library.fileUri, true);
-    return new LibraryDeclaration(importUri, flags, name, script, classes);
+    final extensionUris = getNativeExtensionUris(library)
+        .map((String uri) => objectTable.getNameHandle(null, uri))
+        .toList();
+    if (extensionUris.isNotEmpty) {
+      flags |= LibraryDeclaration.hasExtensionsFlag;
+    }
+    return new LibraryDeclaration(
+        importUri, flags, name, script, extensionUris, classes);
   }
 
   ClassDeclaration getClassDeclaration(Class cls, Members members) {

runtime/bin/entrypoints_verification_test_extension.cc

@@ -20,7 +20,8 @@
 
 #define ASSERT(E)                                                              \
   if (!(E)) {                                                                  \
-    fprintf(stderr, "Assertion \"" #E "\" failed!");                           \
+    fprintf(stderr, "Assertion \"" #E "\" failed at %s:%d!\n", __FILE__,       \
+            __LINE__);                                                         \
     abort();                                                                   \
   }
 

runtime/vm/compiler/frontend/bytecode_reader.cc

@@ -2367,6 +2367,7 @@ void BytecodeReaderHelper::ReadLibraryDeclaration(const Library& library,
   // pkg/vm/lib/bytecode/declarations.dart.
   const int kUsesDartMirrorsFlag = 1 << 0;
   const int kUsesDartFfiFlag = 1 << 1;
+  const int kHasExtensionsFlag = 1 << 2;
 
   ASSERT(library.is_declared_in_bytecode());
   ASSERT(!library.Loaded());
@@ -2393,6 +2394,28 @@ void BytecodeReaderHelper::ReadLibraryDeclaration(const Library& library,
 
   const auto& script = Script::CheckedHandle(Z, ReadObject());
 
+  if ((flags & kHasExtensionsFlag) != 0) {
+    const intptr_t num_extensions = reader_.ReadUInt();
+    auto& import_namespace = Namespace::Handle(Z);
+    auto& native_library = Library::Handle(Z);
+    for (intptr_t i = 0; i < num_extensions; ++i) {
+      name ^= ReadObject();
+      ASSERT(name.StartsWith(Symbols::DartExtensionScheme()));
+
+      // Create a dummy library and add it as an import to the current library.
+      // Actual loading occurs in KernelLoader::LoadNativeExtensionLibraries().
+      // This also allows later to discover and reload this native extension,
+      // e.g. when running from an app-jit snapshot.
+      // See Loader::ReloadNativeExtensions(...) which relies on
+      // Dart_GetImportsOfScheme('dart-ext').
+      native_library = Library::New(name);
+      import_namespace = Namespace::New(native_library, Array::null_array(),
+                                        Array::null_array());
+      library.AddImport(import_namespace);
+    }
+    H.AddPotentialExtensionLibrary(library);
+  }
+
   // The bootstrapper will take care of creating the native wrapper classes,
   // but we will add the synthetic constructors to them here.
   if (name.raw() ==

runtime/vm/compiler/frontend/kernel_translation_helper.cc

@@ -95,6 +95,23 @@ RawGrowableObjectArray* TranslationHelper::EnsurePotentialPragmaFunctions() {
   return funcs.raw();
 }
 
+void TranslationHelper::AddPotentialExtensionLibrary(const Library& library) {
+  if (potential_extension_libraries_ == nullptr) {
+    potential_extension_libraries_ =
+        &GrowableObjectArray::Handle(Z, GrowableObjectArray::New());
+  }
+  potential_extension_libraries_->Add(library);
+}
+
+RawGrowableObjectArray* TranslationHelper::GetPotentialExtensionLibraries() {
+  if (potential_extension_libraries_ != nullptr) {
+    GrowableObjectArray* result = potential_extension_libraries_;
+    potential_extension_libraries_ = nullptr;
+    return result->raw();
+  }
+  return GrowableObjectArray::null();
+}
+
 void TranslationHelper::SetStringOffsets(const TypedData& string_offsets) {
   ASSERT(string_offsets_.IsNull());
   string_offsets_ = string_offsets.raw();

runtime/vm/compiler/frontend/kernel_translation_helper.h

@@ -72,6 +72,9 @@ class TranslationHelper {
 
   RawGrowableObjectArray* EnsurePotentialPragmaFunctions();
 
+  void AddPotentialExtensionLibrary(const Library& library);
+  RawGrowableObjectArray* GetPotentialExtensionLibraries();
+
   void SetKernelProgramInfo(const KernelProgramInfo& info);
   const KernelProgramInfo& GetKernelProgramInfo() const { return info_; }
 
@@ -220,6 +223,7 @@ class TranslationHelper {
   ExternalTypedData& constants_table_;
   KernelProgramInfo& info_;
   Smi& name_index_handle_;
+  GrowableObjectArray* potential_extension_libraries_ = nullptr;
 
   DISALLOW_COPY_AND_ASSIGN(TranslationHelper);
 };

runtime/vm/constants_kbc.h

@@ -749,7 +749,7 @@ class KernelBytecode {
   // Maximum bytecode format version supported by VM.
   // The range of supported versions should include version produced by bytecode
   // generator (currentBytecodeFormatVersion in pkg/vm/lib/bytecode/dbc.dart).
-  static const intptr_t kMaxSupportedBytecodeFormatVersion = 11;
+  static const intptr_t kMaxSupportedBytecodeFormatVersion = 12;
 
   enum Opcode {
 #define DECLARE_BYTECODE(name, encoding, kind, op1, op2, op3) k##name,

runtime/vm/kernel_loader.cc

@@ -212,7 +212,6 @@ KernelLoader::KernelLoader(Program* program,
       evaluating_(GrowableObjectArray::Handle(Z)),
       potential_natives_(GrowableObjectArray::Handle(Z)),
       potential_pragma_functions_(GrowableObjectArray::Handle(Z)),
-      potential_extension_libraries_(GrowableObjectArray::Handle(Z)),
       pragma_class_(Class::Handle(Z)),
       name_index_handle_(Smi::Handle(Z)),
       expression_evaluation_library_(Library::Handle(Z)),
@@ -460,7 +459,6 @@ KernelLoader::KernelLoader(const Script& script,
       evaluating_(GrowableObjectArray::Handle(Z)),
       potential_natives_(GrowableObjectArray::Handle(Z)),
       potential_pragma_functions_(GrowableObjectArray::Handle(Z)),
-      potential_extension_libraries_(GrowableObjectArray::Handle(Z)),
       pragma_class_(Class::Handle(Z)),
       name_index_handle_(Smi::Handle(Z)),
       expression_evaluation_library_(Library::Handle(Z)),
@@ -618,10 +616,11 @@ bool KernelLoader::DetectPragmaCtor() {
 }
 
 void KernelLoader::LoadNativeExtensionLibraries() {
-  const intptr_t length = !potential_extension_libraries_.IsNull()
-                              ? potential_extension_libraries_.Length()
-                              : 0;
-  if (length == 0) return;
+  const auto& potential_extension_libraries =
+      GrowableObjectArray::Handle(Z, H.GetPotentialExtensionLibraries());
+  if (potential_extension_libraries.IsNull()) {
+    return;
+  }
 
   // Prepare lazy constant reading.
   ConstantEvaluator constant_evaluator(&helper_, &type_translator_,
@@ -633,70 +632,89 @@ void KernelLoader::LoadNativeExtensionLibraries() {
   Instance& constant = Instance::Handle(Z);
   String& uri_path = String::Handle(Z);
   Library& library = Library::Handle(Z);
-#if !defined(DART_PRECOMPILER)
-  Object& result = Object::Handle(Z);
-#endif
 
+  const intptr_t length = potential_extension_libraries.Length();
   for (intptr_t i = 0; i < length; ++i) {
-    library ^= potential_extension_libraries_.At(i);
-    ASSERT(!library.is_declared_in_bytecode());
-    helper_.SetOffset(library.kernel_offset());
+    library ^= potential_extension_libraries.At(i);
 
-    LibraryHelper library_helper(&helper_);
-    library_helper.ReadUntilExcluding(LibraryHelper::kAnnotations);
-
-    const intptr_t annotation_count = helper_.ReadListLength();
-    for (intptr_t j = 0; j < annotation_count; ++j) {
-      uri_path = String::null();
-
-      const intptr_t tag = helper_.PeekTag();
-      if (tag == kConstantExpression || tag == kDeprecated_ConstantExpression) {
-        helper_.ReadByte();  // Skip the tag.
-
-        if (tag == kConstantExpression) {
-          helper_.ReadPosition();  // Skip fileOffset.
-          helper_.SkipDartType();  // Skip type.
+    if (library.is_declared_in_bytecode()) {
+      const auto& imports = Array::Handle(Z, library.imports());
+      auto& ns = Namespace::Handle(Z);
+      auto& importee = Library::Handle(Z);
+      for (intptr_t j = 0; j < imports.Length(); ++j) {
+        ns ^= imports.At(j);
+        if (ns.IsNull()) continue;
+        importee = ns.library();
+        uri_path = importee.url();
+        if (uri_path.StartsWith(Symbols::DartExtensionScheme())) {
+          LoadNativeExtension(library, uri_path);
         }
-        const intptr_t constant_table_offset = helper_.ReadUInt();
-        constant = constant_evaluator.EvaluateConstantExpression(
-            constant_table_offset);
-        if (constant.clazz() == external_name_class_.raw()) {
-          uri_path ^= constant.GetField(external_name_field_);
+      }
+    } else {
+      helper_.SetOffset(library.kernel_offset());
+
+      LibraryHelper library_helper(&helper_);
+      library_helper.ReadUntilExcluding(LibraryHelper::kAnnotations);
+
+      const intptr_t annotation_count = helper_.ReadListLength();
+      for (intptr_t j = 0; j < annotation_count; ++j) {
+        uri_path = String::null();
+
+        const intptr_t tag = helper_.PeekTag();
+        if (tag == kConstantExpression ||
+            tag == kDeprecated_ConstantExpression) {
+          helper_.ReadByte();  // Skip the tag.
+
+          if (tag == kConstantExpression) {
+            helper_.ReadPosition();  // Skip fileOffset.
+            helper_.SkipDartType();  // Skip type.
+          }
+          const intptr_t constant_table_offset = helper_.ReadUInt();
+          constant = constant_evaluator.EvaluateConstantExpression(
+              constant_table_offset);
+          if (constant.clazz() == external_name_class_.raw()) {
+            uri_path ^= constant.GetField(external_name_field_);
+          }
+        } else if (tag == kConstructorInvocation ||
+                   tag == kConstConstructorInvocation) {
+          uri_path = DetectExternalNameCtor();
+        } else {
+          helper_.SkipExpression();
         }
-      } else if (tag == kConstructorInvocation ||
-                 tag == kConstConstructorInvocation) {
-        uri_path = DetectExternalNameCtor();
-      } else {
-        helper_.SkipExpression();
+
+        if (uri_path.IsNull()) continue;
+
+        LoadNativeExtension(library, uri_path);
+
+        // Create a dummy library and add it as an import to the current
+        // library. This allows later to discover and reload this native
+        // extension, e.g. when running from an app-jit snapshot.
+        // See Loader::ReloadNativeExtensions(...) which relies on
+        // Dart_GetImportsOfScheme('dart-ext').
+        const auto& native_library = Library::Handle(Library::New(uri_path));
+        library.AddImport(Namespace::Handle(Namespace::New(
+            native_library, Array::null_array(), Array::null_array())));
       }
-
-      if (uri_path.IsNull()) continue;
-
-#if !defined(DART_PRECOMPILER)
-      if (!I->HasTagHandler()) {
-        H.ReportError("no library handler registered.");
-      }
-
-      I->BlockClassFinalization();
-      result = I->CallTagHandler(Dart_kImportExtensionTag, library, uri_path);
-      I->UnblockClassFinalization();
-
-      if (result.IsError()) {
-        H.ReportError(Error::Cast(result), "library handler failed");
-      }
-#endif
-
-      // Create a dummy library and add it as an import to the current library.
-      // This allows later to discover and reload this native extension, e.g.
-      // when running from an app-jit snapshot.
-      // See Loader::ReloadNativeExtensions(...) which relies on
-      // Dart_GetImportsOfScheme('dart-ext').
-      const auto& native_library = Library::Handle(Library::New(uri_path));
-      library.AddImport(Namespace::Handle(Namespace::New(
-          native_library, Array::null_array(), Array::null_array())));
     }
   }
-  potential_extension_libraries_ = GrowableObjectArray::null();
+}
+
+void KernelLoader::LoadNativeExtension(const Library& library,
+                                       const String& uri_path) {
+#if !defined(DART_PRECOMPILER)
+  if (!I->HasTagHandler()) {
+    H.ReportError("no library handler registered.");
+  }
+
+  I->BlockClassFinalization();
+  const auto& result = Object::Handle(
+      Z, I->CallTagHandler(Dart_kImportExtensionTag, library, uri_path));
+  I->UnblockClassFinalization();
+
+  if (result.IsError()) {
+    H.ReportError(Error::Cast(result), "library handler failed");
+  }
+#endif
 }
 
 RawObject* KernelLoader::LoadProgram(bool process_pending_classes) {
@@ -1022,8 +1040,7 @@ RawLibrary* KernelLoader::LoadLibrary(intptr_t index) {
   if (annotation_count > 0) {
     // This must wait until we can evaluate constants.
     // So put on the "pending" list.
-    EnsurePotentialExtensionLibraries();
-    potential_extension_libraries_.Add(library);
+    H.AddPotentialExtensionLibrary(library);
   }
   for (intptr_t i = 0; i < annotation_count; ++i) {
     helper_.SkipExpression();  // read ith annotation.

runtime/vm/kernel_loader.h

@@ -243,6 +243,7 @@ class KernelLoader : public ValueObject {
 
   void AnnotateNativeProcedures();
   void LoadNativeExtensionLibraries();
+  void LoadNativeExtension(const Library& library, const String& uri_path);
   void EvaluateDelayedPragmas();
 
   void ReadVMAnnotations(const Library& library,
@@ -389,12 +390,6 @@ class KernelLoader : public ValueObject {
         translation_helper_.EnsurePotentialPragmaFunctions();
   }
 
-  void EnsurePotentialExtensionLibraries() {
-    if (potential_extension_libraries_.IsNull()) {
-      potential_extension_libraries_ = GrowableObjectArray::New();
-    }
-  }
-
   // Returns `true` if the [library] was newly enqueued or `false`
   // if it was already enqueued. Allocates storage on first enqueue.
   bool EnqueueLibraryForEvaluation(const Library& library) {
@@ -456,7 +451,6 @@ class KernelLoader : public ValueObject {
   GrowableObjectArray& evaluating_;
   GrowableObjectArray& potential_natives_;
   GrowableObjectArray& potential_pragma_functions_;
-  GrowableObjectArray& potential_extension_libraries_;
 
   Class& pragma_class_;
 

samples/samples.status

@@ -23,7 +23,7 @@ sample_extension/test/sample_extension_app_snapshot_test: Pass, RuntimeError # I
 [ $arch != x64 || $compiler != dartk || $system != linux || $hot_reload || $hot_reload_rollback ]
 ffi/sqlite/test/sqlite_test: Skip # FFI not supported or libsqlite3.so not available.
 
-[ $compiler == app_jitk || $compiler == dartkb || $compiler == dartkp ]
+[ $compiler == app_jitk || $compiler == dartkp ]
 sample_extension/test/sample_extension_app_snapshot_test: SkipByDesign
 sample_extension/test/sample_extension_test: SkipByDesign
 

tests/standalone_2/standalone_2_kernel.status

@@ -26,15 +26,7 @@ io/zlib_test: RuntimeError
 
 [ $compiler == dartkb ]
 io/dart_std_io_pipe_test: Pass, Timeout # Please triage
-io/platform_resolved_executable_test/00: RuntimeError # Reruns the same script (dill file without AST) without passing --enable-interpreter or --use-bytecode-compiler.
-io/platform_resolved_executable_test/01: RuntimeError # Reruns the same script (dill file without AST) without passing --enable-interpreter or --use-bytecode-compiler.
-io/platform_resolved_executable_test/02: RuntimeError # Reruns the same script (dill file without AST) without passing --enable-interpreter or --use-bytecode-compiler.
-io/platform_resolved_executable_test/03: RuntimeError
-io/platform_resolved_executable_test/04: RuntimeError
-io/platform_resolved_executable_test/05: RuntimeError # Reruns the same script (dill file without AST) without passing --enable-interpreter or --use-bytecode-compiler.
 io/platform_test: RuntimeError # Platform.script points to dill file.
-io/test_extension_fail_test: RuntimeError # Platform.script points to dill file.
-io/test_extension_test: RuntimeError # Platform.script points to dill file.
 no_lazy_dispatchers_test: SkipByDesign # KBC interpreter doesn't support --no_lazy_dispatchers
 
 [ $compiler == dartkp ]

tools/VERSION

@@ -35,5 +35,5 @@ MINOR 5
 PATCH 0
 PRERELEASE 0
 PRERELEASE_PATCH 0
-ABI_VERSION 6
+ABI_VERSION 7
 OLDEST_SUPPORTED_ABI_VERSION 5