[vm/kernel/aot] Generate list of precompiler entry points during build

This CL adds auto-generation of entry points JSON file for standalone VM
during build (in runtime_kernel target, as it is only used for
kernel-based transformation).

pkg/vm/tool/precompiler2 script is changed to automatically pass
auto-generated entry points to gen_kernel.

New option --entry-points is added to pkg/vm/bin/gen_kernel.dart in order to
specify custom entry points files.

Checked-in entry_points.json is still used for testing and in gen_kernel
if --entry-points is not specified.

Issue: https://github.com/dart-lang/sdk/issues/30480
Change-Id: I39c4461e48763f40d7565656dc33e23fc5d9c2cb
Reviewed-on: https://dart-review.googlesource.com/42521
Commit-Queue: Alexander Markov <alexmarkov@google.com>
Reviewed-by: Zach Anderson <zra@google.com>

BUILD.gn

@@ -55,6 +55,7 @@ group("runtime_kernel") {
   }
   deps = [
     ":runtime",
+    "runtime/bin:precompiler_entry_points_json",
     "runtime/vm:kernel_platform_files($host_toolchain)",
 
     # TODO(rmacnak): Link this into 'dart'.

pkg/vm/bin/gen_kernel.dart

@@ -30,7 +30,10 @@ final ArgParser _argParser = new ArgParser(allowTrailingOptions: true)
   ..addFlag('sync-async', help: 'Start `async` functions synchronously')
   ..addFlag('embed-sources',
       help: 'Embed source files in the generated kernel program',
-      defaultsTo: true);
+      defaultsTo: true)
+  ..addOption('entry-points',
+      help: 'Path to JSON file with the list of entry points',
+      allowMultiple: true);
 
 final String _usage = '''
 Usage: dart pkg/vm/bin/gen_kernel.dart --platform vm_platform_strong.dill [options] input.dart
@@ -67,6 +70,14 @@ Future<int> compile(List<String> arguments) async {
   final bool aot = options['aot'];
   final bool syncAsync = options['sync-async'];
 
+  final List<String> entryPoints = options['entry-points'] ?? <String>[];
+  if (entryPoints.isEmpty) {
+    entryPoints.addAll([
+      'pkg/vm/lib/transformations/type_flow/entry_points.json',
+      'pkg/vm/lib/transformations/type_flow/entry_points_extra.json',
+    ]);
+  }
+
   ErrorDetector errorDetector = new ErrorDetector();
 
   final CompilerOptions compilerOptions = new CompilerOptions()
@@ -81,7 +92,7 @@ Future<int> compile(List<String> arguments) async {
 
   Program program = await compileToKernel(
       Uri.base.resolve(filename), compilerOptions,
-      aot: aot);
+      aot: aot, entryPoints: entryPoints);
 
   if (errorDetector.hasCompilationErrors || (program == null)) {
     return _compileTimeErrorExitCode;

pkg/vm/lib/kernel_front_end.dart

@@ -33,7 +33,7 @@ const kUseGlobalTypeFlow = const bool.fromEnvironment('use.global.type.flow');
 /// VM-specific replacement of [kernelForProgram].
 ///
 Future<Program> compileToKernel(Uri source, CompilerOptions options,
-    {bool aot: false}) async {
+    {bool aot: false, List<String> entryPoints}) async {
   // Replace error handler to detect if there are compilation errors.
   final errorDetector =
       new ErrorDetector(previousErrorHandler: options.onError);
@@ -46,18 +46,19 @@ Future<Program> compileToKernel(Uri source, CompilerOptions options,
 
   // Run global transformations only if program is correct.
   if (aot && (program != null) && !errorDetector.hasCompilationErrors) {
-    _runGlobalTransformations(program, options.strongMode);
+    _runGlobalTransformations(program, options.strongMode, entryPoints);
   }
 
   return program;
 }
 
-_runGlobalTransformations(Program program, bool strongMode) {
+_runGlobalTransformations(
+    Program program, bool strongMode, List<String> entryPoints) {
   if (strongMode) {
     final coreTypes = new CoreTypes(program);
 
     if (kUseGlobalTypeFlow) {
-      globalTypeFlow.transformProgram(coreTypes, program);
+      globalTypeFlow.transformProgram(coreTypes, program, entryPoints);
     } else {
       devirtualization.transformProgram(coreTypes, program);
     }

pkg/vm/lib/transformations/type_flow/analysis.dart

@@ -27,7 +27,6 @@ import 'utils.dart';
 // === Correctness ===
 // * Verify incremental re-calculation by fresh analysis starting with known
 //   allocated classes.
-// * Auto-generate entry_points.json during build.
 //
 // === Precision ===
 // * Handle '==' with null.

pkg/vm/lib/transformations/type_flow/transformer.dart

@@ -30,12 +30,12 @@ const bool kDumpClassHierarchy =
 
 /// Whole-program type flow analysis and transformation.
 /// Assumes strong mode and closed world.
-Program transformProgram(CoreTypes coreTypes, Program program,
-    // TODO(alexmarkov): Pass entry points descriptors from command line.
-    {List<String> entryPointsJSONFiles: const [
-      'pkg/vm/lib/transformations/type_flow/entry_points.json',
-      'pkg/vm/lib/transformations/type_flow/entry_points_extra.json',
-    ]}) {
+Program transformProgram(
+    CoreTypes coreTypes, Program program, List<String> entryPoints) {
+  if ((entryPoints == null) || entryPoints.isEmpty) {
+    throw 'Error: unable to perform global type flow analysis without entry points.';
+  }
+
   void ignoreAmbiguousSupertypes(Class cls, Supertype a, Supertype b) {}
   final hierarchy = new ClassHierarchy(program,
       onAmbiguousSupertypes: ignoreAmbiguousSupertypes);
@@ -52,7 +52,7 @@ Program transformProgram(CoreTypes coreTypes, Program program,
   final analysisStopWatch = new Stopwatch()..start();
 
   final typeFlowAnalysis = new TypeFlowAnalysis(hierarchy, types, libraryIndex,
-      entryPointsJSONFiles: entryPointsJSONFiles);
+      entryPointsJSONFiles: entryPoints);
 
   Procedure main = program.mainMethod;
   final Selector mainSelector = new DirectSelector(main);

pkg/vm/test/transformations/type_flow/transformer_test.dart

@@ -25,10 +25,12 @@ runTestCase(Uri source) async {
 
   final coreTypes = new CoreTypes(program);
 
-  program = transformProgram(coreTypes, program, entryPointsJSONFiles: [
+  final entryPoints = [
     pkgVmDir + '/lib/transformations/type_flow/entry_points.json',
     pkgVmDir + '/lib/transformations/type_flow/entry_points_extra.json',
-  ]);
+  ];
+
+  program = transformProgram(coreTypes, program, entryPoints);
 
   final StringBuffer buffer = new StringBuffer();
   new Printer(buffer, showExternal: false, showMetadata: true)

pkg/vm/tool/precompiler2

@@ -70,6 +70,9 @@ fi
 export DART_CONFIGURATION=${DART_CONFIGURATION:-ReleaseX64}
 BIN_DIR="$OUT_DIR/$DART_CONFIGURATION"
 
+ENTRY_POINTS="--entry-points ${BIN_DIR}/gen/runtime/bin/precompiler_entry_points.json \
+              --entry-points ${SDK_DIR}/pkg/vm/lib/transformations/type_flow/entry_points_extra.json"
+
 # Step 1: Generate Kernel binary from the input Dart source.
 "$BIN_DIR"/dart                                                                \
      --limit-ints-to-64-bits                                                   \
@@ -78,6 +81,7 @@ BIN_DIR="$OUT_DIR/$DART_CONFIGURATION"
      --aot                                                                     \
      $SYNC_ASYNC                                                               \
      $PACKAGES                                                                 \
+     $ENTRY_POINTS                                                             \
      -o "$SNAPSHOT_FILE.dill"                                                  \
      "$SOURCE_FILE"
 

runtime/bin/BUILD.gn

@@ -1205,3 +1205,30 @@ shared_library("sample_extension") {
     ldflags = [ "/LIBPATH:$abs_root_out_dir" ]
   }
 }
+
+compiled_action("precompiler_entry_points_json") {
+  # Printing precompiler entry points is folded into precompilation, so dart_bootstrap is invoked
+  # with correct arguments to generate app-aot snapshot.
+
+  # Dart script to precompile (any valid script would suffice).
+  input = "../vm/snapshot_test.dart"
+
+  # Output snapshot for precompilation.
+  snapshot = "$target_gen_dir/dummy.snapshot"
+
+  output = "$target_gen_dir/precompiler_entry_points.json"
+  tool = ":dart_bootstrap"
+  inputs = [
+    input,
+  ]
+  outputs = [
+    output,
+  ]
+  args = [
+    "--print-precompiler-entry-points=" + rebase_path(output),
+    "--snapshot=" + rebase_path(snapshot),
+    "--snapshot-kind=app-aot",
+    "--use-blobs",
+    rebase_path(input),
+  ]
+}

runtime/vm/compiler/aot/precompiler.cc

@@ -691,7 +691,27 @@ void PrecompilerEntryPointsPrinter::Print() {
 
   writer.CloseObject();  // top-level
 
-  THR_Print("%s\n", writer.ToCString());
+  const char* contents = writer.ToCString();
+
+  Dart_FileOpenCallback file_open = Dart::file_open_callback();
+  Dart_FileWriteCallback file_write = Dart::file_write_callback();
+  Dart_FileCloseCallback file_close = Dart::file_close_callback();
+  if ((file_open == NULL) || (file_write == NULL) || (file_close == NULL)) {
+    FATAL(
+        "Unable to print precompiler entry points:"
+        " file callbacks are not provided by embedder");
+  }
+
+  void* out_stream =
+      file_open(FLAG_print_precompiler_entry_points, /* write = */ true);
+  if (out_stream == NULL) {
+    FATAL1(
+        "Unable to print precompiler entry points:"
+        " failed to open file \"%s\" for writing",
+        FLAG_print_precompiler_entry_points);
+  }
+  file_write(contents, strlen(contents), out_stream);
+  file_close(out_stream);
 }
 
 void PrecompilerEntryPointsPrinter::DescribeClass(JSONWriter* writer,

runtime/vm/flag_list.h

@@ -133,7 +133,7 @@
   P(polymorphic_with_deopt, bool, true,                                        \
     "Polymorphic calls with deoptimization / megamorphic call")                \
   P(precompiled_mode, bool, false, "Precompilation compiler mode")             \
-  P(print_precompiler_entry_points, bool, false,                               \
+  P(print_precompiler_entry_points, charp, NULL,                               \
     "Print entry points and info about recognized methods used by "            \
     "precompiler.")                                                            \
   P(print_snapshot_sizes, bool, false, "Print sizes of generated snapshots.")  \