[vm] Add --minimal-kernel option to kernel compilers

New option can be used instead of --aot to produce a tree-shaken
minimal .dill file which can run on the VM in JIT mode.

Can be used along with --no-link-platform and --gen-bytecode to further
decrease size of the .dill file.

Change-Id: I35e8d81263a5e223decf160d79322c06cfd27321
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/140740
Commit-Queue: Alexander Markov <alexmarkov@google.com>
Reviewed-by: Alexander Aprelev <aam@google.com>

pkg/frontend_server/lib/frontend_server.dart

@@ -61,6 +61,8 @@ ArgParser argParser = ArgParser(allowTrailingOptions: true)
   ..addFlag('protobuf-tree-shaker',
       help: 'Enable protobuf tree shaker transformation in AOT mode.',
       defaultsTo: false)
+  ..addFlag('minimal-kernel',
+      help: 'Produce minimal tree-shaken kernel file.', defaultsTo: false)
   ..addFlag('link-platform',
       help:
           'When in batch mode, link platform kernel file into result kernel file.'
@@ -517,7 +519,8 @@ class FrontendCompiler implements CompilerInterface {
           useGlobalTypeFlowAnalysis: options['tfa'],
           environmentDefines: environmentDefines,
           enableAsserts: options['enable-asserts'],
-          useProtobufTreeShaker: options['protobuf-tree-shaker']));
+          useProtobufTreeShaker: options['protobuf-tree-shaker'],
+          minimalKernel: options['minimal-kernel']));
     }
     if (results.component != null) {
       transformer?.transform(results.component);
@@ -527,7 +530,7 @@ class FrontendCompiler implements CompilerInterface {
             options['filesystem-scheme'], options['dartdevc-module-format']);
       }
       await writeDillFile(results, _kernelBinaryFilename,
-          filterExternal: importDill != null,
+          filterExternal: importDill != null || options['minimal-kernel'],
           incrementalSerializer: incrementalSerializer);
 
       _outputStream.writeln(boundaryKey);

pkg/vm/lib/kernel_front_end.dart

@@ -78,6 +78,8 @@ void declareCompilerOptions(ArgParser args) {
   args.addOption('depfile', help: 'Path to output Ninja depfile');
   args.addFlag('link-platform',
       help: 'Include platform into resulting kernel file.', defaultsTo: true);
+  args.addFlag('minimal-kernel',
+      help: 'Produce minimal tree-shaken kernel file.', defaultsTo: false);
   args.addFlag('embed-sources',
       help: 'Embed source files in the generated kernel component',
       defaultsTo: true);
@@ -167,6 +169,7 @@ Future<int> runCompiler(ArgResults options, String usage) async {
   final bool nullSafety = options['null-safety'];
   final bool useProtobufTreeShaker = options['protobuf-tree-shaker'];
   final bool splitOutputByPackages = options['split-output-by-packages'];
+  final bool minimalKernel = options['minimal-kernel'];
   final List<String> experimentalFlags = options['enable-experiment'];
   final Map<String, String> environmentDefines = {};
 
@@ -242,7 +245,8 @@ Future<int> runCompiler(ArgResults options, String usage) async {
       genBytecode: genBytecode,
       bytecodeOptions: bytecodeOptions,
       dropAST: dropAST && !splitOutputByPackages,
-      useProtobufTreeShaker: useProtobufTreeShaker);
+      useProtobufTreeShaker: useProtobufTreeShaker,
+      minimalKernel: minimalKernel);
 
   errorPrinter.printCompilationMessages();
 
@@ -255,7 +259,10 @@ Future<int> runCompiler(ArgResults options, String usage) async {
   }
 
   final IOSink sink = new File(outputFileName).openWrite();
-  final BinaryPrinter printer = new BinaryPrinter(sink);
+  final BinaryPrinter printer = new BinaryPrinter(sink,
+      libraryFilter: minimalKernel
+          ? ((lib) => !results.loadedLibraries.contains(lib))
+          : null);
   printer.writeComponentFile(results.component);
   await sink.close();
 
@@ -314,7 +321,8 @@ Future<KernelCompilationResults> compileToKernel(
     bool genBytecode: false,
     BytecodeOptions bytecodeOptions,
     bool dropAST: false,
-    bool useProtobufTreeShaker: false}) async {
+    bool useProtobufTreeShaker: false,
+    bool minimalKernel: false}) async {
   // Replace error handler to detect if there are compilation errors.
   final errorDetector =
       new ErrorDetector(previousErrorHandler: options.onDiagnostic);
@@ -325,21 +333,32 @@ Future<KernelCompilationResults> compileToKernel(
 
   CompilerResult compilerResult = await kernelForProgram(source, options);
   Component component = compilerResult?.component;
-  final compiledSources = component?.uriToSource?.keys;
+  Iterable<Uri> compiledSources = component?.uriToSource?.keys;
 
   Set<Library> loadedLibraries = createLoadedLibrariesSet(
       compilerResult?.loadedComponents, compilerResult?.sdkComponent,
       includePlatform: includePlatform);
 
   // Run global transformations only if component is correct.
-  if (aot && component != null) {
+  if ((aot || minimalKernel) && component != null) {
     await runGlobalTransformations(
         options.target,
         component,
         useGlobalTypeFlowAnalysis,
         enableAsserts,
         useProtobufTreeShaker,
-        errorDetector);
+        errorDetector,
+        minimalKernel: minimalKernel);
+
+    if (minimalKernel) {
+      // compiledSources is component.uriToSource.keys.
+      // Make a copy of compiledSources to detach it from
+      // component.uriToSource which is cleared below.
+      compiledSources = compiledSources.toList();
+
+      component.metadata.clear();
+      component.uriToSource.clear();
+    }
   }
 
   if (genBytecode && !errorDetector.hasCompilationErrors && component != null) {
@@ -404,7 +423,8 @@ Future runGlobalTransformations(
     bool useGlobalTypeFlowAnalysis,
     bool enableAsserts,
     bool useProtobufTreeShaker,
-    ErrorDetector errorDetector) async {
+    ErrorDetector errorDetector,
+    {bool minimalKernel: false}) async {
   if (errorDetector.hasCompilationErrors) return;
 
   final coreTypes = new CoreTypes(component);
@@ -422,7 +442,8 @@ Future runGlobalTransformations(
   unreachable_code_elimination.transformComponent(component, enableAsserts);
 
   if (useGlobalTypeFlowAnalysis) {
-    globalTypeFlow.transformComponent(target, coreTypes, component);
+    globalTypeFlow.transformComponent(target, coreTypes, component,
+        treeShakeSignatures: !minimalKernel);
   } else {
     devirtualization.transformComponent(coreTypes, component);
     no_dynamic_invocations_annotator.transformComponent(component);
@@ -436,7 +457,8 @@ Future runGlobalTransformations(
     protobuf_tree_shaker.removeUnusedProtoReferences(
         component, coreTypes, null);
 
-    globalTypeFlow.transformComponent(target, coreTypes, component);
+    globalTypeFlow.transformComponent(target, coreTypes, component,
+        treeShakeSignatures: !minimalKernel);
   }
 
   // TODO(35069): avoid recomputing CSA by reading it from the platform files.

pkg/vm/tool/precompiler2

@@ -36,6 +36,9 @@ for arg in "$@"; do
     --no-tfa | \
     --gen-bytecode | \
     --no-gen-bytecode | \
+    --bytecode-options=* | \
+    --minimal-kernel | \
+    --no-embed-sources | \
     -D* )
     GEN_KERNEL_OPTIONS+=("$arg")
     ;;

runtime/tests/vm/dart/minimal_kernel_bytecode_test.dart

@@ -0,0 +1,12 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// Tests that dill file produced with --minimal-kernel --gen-bytecode
+// options works as expected.
+
+import 'minimal_kernel_test.dart' as test;
+
+main() async {
+  await test.compileAndRunMinimalDillTest(['--gen-bytecode']);
+}

runtime/tests/vm/dart/minimal_kernel_test.dart

@@ -0,0 +1,33 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// Tests that dill file produced with --minimal-kernel option
+// works as expected.
+
+import 'package:path/path.dart' as path;
+import 'snapshot_test_helper.dart';
+
+compileAndRunMinimalDillTest(List<String> extraCompilationArgs) async {
+  await withTempDir((String temp) async {
+    final expectedOutput =
+        (await runDart('RUN FROM SOURCE', [genKernel, '--help'])).output;
+
+    final minimalDillPath = path.join(temp, 'minimal.dill');
+    await runGenKernel('BUILD MINIMAL DILL FILE', [
+      '--minimal-kernel',
+      '--no-link-platform',
+      ...extraCompilationArgs,
+      '--output=$minimalDillPath',
+      genKernel,
+    ]);
+
+    final result1 = await runDart(
+        'RUN FROM MINIMAL DILL FILE', [minimalDillPath, '--help']);
+    expectOutput(expectedOutput, result1);
+  });
+}
+
+main() async {
+  await compileAndRunMinimalDillTest([]);
+}

runtime/tests/vm/dart/snapshot_test_helper.dart

@@ -14,6 +14,8 @@ class Result {
   final ProcessResult processResult;
 
   Result(this.cmdline, this.processResult);
+
+  String get output => processResult.stdout.trim();
 }
 
 void reportError(Result result, String msg) {
@@ -36,7 +38,7 @@ ${result.processResult.stderr}''');
 }
 
 void expectOutput(String what, Result result) {
-  if (result.processResult.stdout.trim() != what) {
+  if (result.output != what) {
     reportError(result, 'Expected test to print \'${what}\' to stdout');
   }
 }

runtime/tests/vm/vm.status

@@ -117,6 +117,8 @@ dart/entrypoints/jit/*: SkipByDesign # These tests should only run on JIT.
 dart/appjit*: SkipByDesign # Test needs to run from source
 dart/bytecode_and_ast_mix_test: SkipByDesign # Test needs to run from source
 dart/kernel_determinism_test: SkipByDesign # Test needs to run from source
+dart/minimal_kernel_bytecode_test: SkipByDesign # Test needs to run from source
+dart/minimal_kernel_test: SkipByDesign # Test needs to run from source
 dart/snapshot_depfile_test: SkipByDesign # Test needs to run from source
 
 [ $compiler == dartkp && ($runtime == dart_precompiled || $runtime == vm) ]