[frontend_server] Wire up incremental serializer into frontend server / package VM

Change-Id: I4d8dbc2d9b0915c654eccae9e35357d7ec9fa13d
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/120787
Commit-Queue: Jens Johansen <jensj@google.com>
Reviewed-by: Alexander Markov <alexmarkov@google.com>
Reviewed-by: Johnni Winther <johnniwinther@google.com>

pkg/front_end/lib/src/api_prototype/incremental_kernel_generator.dart

@@ -44,7 +44,7 @@ abstract class IncrementalKernelGenerator {
   /// platform will be loaded.
   factory IncrementalKernelGenerator.fromComponent(
       CompilerOptions options, Uri entryPoint, Component component,
-      [bool outlineOnly]) {
+      [bool outlineOnly, IncrementalSerializer incrementalSerializer]) {
     return new IncrementalCompiler.fromComponent(
         new CompilerContext(
             new ProcessedOptions(options: options, inputs: [entryPoint])),

pkg/frontend_server/lib/frontend_server.dart

@@ -130,13 +130,21 @@ ArgParser argParser = new ArgParser(allowTrailingOptions: true)
           ' application, produces better stack traces on exceptions.',
       defaultsTo: true)
   ..addFlag('unsafe-package-serialization',
-      help: 'Potentially unsafe: Does not allow for invalidating packages, '
+      help: '*Deprecated* '
+          'Potentially unsafe: Does not allow for invalidating packages, '
           'additionally the output dill file might include more libraries than '
           'needed. The use case is test-runs, where invalidation is not really '
           'used, and where dill filesize does not matter, and the gain is '
           'improved speed.',
       defaultsTo: false,
       hide: true)
+  ..addFlag('incremental-serialization',
+      help: 'Re-use previously serialized data when serializing. '
+          'The output dill file might include more libraries than strictly '
+          'needed, but the serialization phase will generally be much faster.',
+      defaultsTo: false,
+      negatable: true,
+      hide: true)
   ..addFlag('track-widget-creation',
       help: 'Run a kernel transformer to track creation locations for widgets.',
       defaultsTo: false)
@@ -262,7 +270,8 @@ class FrontendCompiler implements CompilerInterface {
   FrontendCompiler(this._outputStream,
       {this.printerFactory,
       this.transformer,
-      this.unsafePackageSerialization}) {
+      this.unsafePackageSerialization,
+      this.incrementalSerialization}) {
     _outputStream ??= stdout;
     printerFactory ??= new BinaryPrinterFactory();
     // Initialize supported kernel targets.
@@ -277,6 +286,7 @@ class FrontendCompiler implements CompilerInterface {
   StringSink _outputStream;
   BinaryPrinterFactory printerFactory;
   bool unsafePackageSerialization;
+  bool incrementalSerialization;
 
   CompilerOptions _compilerOptions;
   BytecodeOptions _bytecodeOptions;
@@ -400,6 +410,7 @@ class FrontendCompiler implements CompilerInterface {
     _bytecodeOptions = bytecodeOptions;
 
     KernelCompilationResults results;
+    IncrementalSerializer incrementalSerializer;
     if (options['incremental']) {
       setVMEnvironmentDefines(environmentDefines, _compilerOptions);
 
@@ -414,6 +425,8 @@ class FrontendCompiler implements CompilerInterface {
           _generator.getClassHierarchy(),
           _generator.getCoreTypes(),
           component.uriToSource.keys);
+
+      incrementalSerializer = _generator.incrementalSerializer;
     } else {
       if (options['link-platform']) {
         // TODO(aam): Remove linkedDependencies once platform is directly embedded
@@ -436,7 +449,8 @@ class FrontendCompiler implements CompilerInterface {
       }
 
       await writeDillFile(results, _kernelBinaryFilename,
-          filterExternal: importDill != null);
+          filterExternal: importDill != null,
+          incrementalSerializer: incrementalSerializer);
 
       _outputStream.writeln(boundaryKey);
       await _outputDependenciesDelta(results.compiledSources);
@@ -501,7 +515,8 @@ class FrontendCompiler implements CompilerInterface {
   }
 
   writeDillFile(KernelCompilationResults results, String filename,
-      {bool filterExternal: false}) async {
+      {bool filterExternal: false,
+      IncrementalSerializer incrementalSerializer}) async {
     final Component component = results.component;
     // Remove the cache that came either from this function or from
     // initializing from a kernel file.
@@ -560,7 +575,10 @@ class FrontendCompiler implements CompilerInterface {
 
       sortComponent(component);
 
-      if (unsafePackageSerialization == true) {
+      if (incrementalSerializer != null) {
+        incrementalSerializer.writePackagesToSinkAndTrimComponent(
+            component, sink);
+      } else if (unsafePackageSerialization == true) {
         writePackagesToSinkAndTrimComponent(component, sink);
       }
 
@@ -687,7 +705,8 @@ class FrontendCompiler implements CompilerInterface {
         _generator.getCoreTypes(),
         deltaProgram.uriToSource.keys);
 
-    await writeDillFile(results, _kernelBinaryFilename);
+    await writeDillFile(results, _kernelBinaryFilename,
+        incrementalSerializer: _generator.incrementalSerializer);
 
     _outputStream.writeln(boundaryKey);
     await _outputDependenciesDelta(results.compiledSources);
@@ -852,7 +871,8 @@ class FrontendCompiler implements CompilerInterface {
 
   IncrementalCompiler _createGenerator(Uri initializeFromDillUri) {
     return new IncrementalCompiler(_compilerOptions, _mainSource,
-        initializeFromDillUri: initializeFromDillUri);
+        initializeFromDillUri: initializeFromDillUri,
+        incrementalSerialization: incrementalSerialization);
   }
 
   Uri _ensureFolderPath(String path) {
@@ -1072,7 +1092,8 @@ Future<int> starter(
 
   compiler ??= new FrontendCompiler(output,
       printerFactory: binaryPrinterFactory,
-      unsafePackageSerialization: options["unsafe-package-serialization"]);
+      unsafePackageSerialization: options["unsafe-package-serialization"],
+      incrementalSerialization: options["incremental-serialization"]);
 
   if (options.rest.isNotEmpty) {
     return await compiler.compile(options.rest[0], options,

pkg/frontend_server/test/frontend_server_test.dart

@@ -896,6 +896,115 @@ true
       inputStreamController.close();
     });
 
+    test('incremental-serialization', () async {
+      // Package A.
+      var file = new File('${tempDir.path}/pkgA/a.dart')
+        ..createSync(recursive: true);
+      file.writeAsStringSync("pkgA() {}");
+
+      // Package B.
+      file = new File('${tempDir.path}/pkgB/.packages')
+        ..createSync(recursive: true);
+      file.writeAsStringSync("pkgA: ../pkgA");
+      file = new File('${tempDir.path}/pkgB/a.dart')
+        ..createSync(recursive: true);
+      file.writeAsStringSync("pkgB_a() {}");
+      file = new File('${tempDir.path}/pkgB/b.dart')
+        ..createSync(recursive: true);
+      file.writeAsStringSync("import 'package:pkgA/a.dart';"
+          "pkgB_b() { pkgA(); }");
+
+      // Application.
+      file = new File('${tempDir.path}/app/.packages')
+        ..createSync(recursive: true);
+      file.writeAsStringSync("pkgA:../pkgA\n"
+          "pkgB:../pkgB");
+
+      // Entry point A uses both package A and B.
+      file = new File('${tempDir.path}/app/a.dart')
+        ..createSync(recursive: true);
+      file.writeAsStringSync("import 'package:pkgB/b.dart';"
+          "import 'package:pkgB/a.dart';"
+          "appA() { pkgB_a(); pkgB_b(); }");
+
+      // Entry point B uses only package B.
+      var fileB = new File('${tempDir.path}/app/B.dart')
+        ..createSync(recursive: true);
+      fileB.writeAsStringSync("import 'package:pkgB/a.dart';"
+          "appB() { pkgB_a(); }");
+
+      // Other setup.
+      var dillFile = new File('${tempDir.path}/app.dill');
+      expect(dillFile.existsSync(), equals(false));
+
+      // First compile app entry point A.
+      final List<String> args = <String>[
+        '--sdk-root=${sdkRoot.toFilePath()}',
+        '--incremental',
+        '--platform=${platformKernel.path}',
+        '--output-dill=${dillFile.path}',
+        '--incremental-serialization',
+      ];
+
+      final StreamController<List<int>> inputStreamController =
+          new StreamController<List<int>>();
+      final StreamController<List<int>> stdoutStreamController =
+          new StreamController<List<int>>();
+      final IOSink ioSink = new IOSink(stdoutStreamController.sink);
+      StreamController<Result> receivedResults = new StreamController<Result>();
+      final outputParser = new OutputParser(receivedResults);
+      stdoutStreamController.stream
+          .transform(utf8.decoder)
+          .transform(const LineSplitter())
+          .listen(outputParser.listener);
+
+      Future<int> result =
+          starter(args, input: inputStreamController.stream, output: ioSink);
+      inputStreamController.add('compile ${file.path}\n'.codeUnits);
+      int count = 0;
+      receivedResults.stream.listen((Result compiledResult) {
+        CompilationResult result =
+            new CompilationResult.parse(compiledResult.status);
+        switch (count) {
+          case 0:
+            expect(dillFile.existsSync(), equals(true));
+            expect(result.filename, dillFile.path);
+            expect(result.errorsCount, 0);
+            count += 1;
+            inputStreamController.add('accept\n'.codeUnits);
+            inputStreamController.add('reset\n'.codeUnits);
+
+            inputStreamController.add('recompile ${fileB.path} abc\n'
+                    '${fileB.path}\n'
+                    'abc\n'
+                .codeUnits);
+            break;
+          case 1:
+            expect(result.filename, dillFile.path);
+            expect(result.errorsCount, 0);
+            inputStreamController.add('quit\n'.codeUnits);
+
+            // Loadable.
+            Component component = loadComponentFromBinary(dillFile.path);
+
+            // Contains (at least) the 2 files we want.
+            component.libraries
+                    .where((l) =>
+                        l.importUri.toString() == "package:pkgB/a.dart" ||
+                        l.fileUri.toString().contains(fileB.path))
+                    .length ==
+                2;
+
+            // Verifiable (together with the platform file).
+            component =
+                loadComponentFromBinary(platformKernel.toFilePath(), component);
+            verifyComponent(component);
+        }
+      });
+      expect(await result, 0);
+      inputStreamController.close();
+    });
+
     test('compile and recompile report non-zero error count', () async {
       var file = new File('${tempDir.path}/foo.dart')..createSync();
       file.writeAsStringSync("main() { foo(); bar(); }\n");

pkg/vm/lib/incremental_compiler.dart

@@ -19,6 +19,7 @@ const String kDebugProcedureName = ":Eval";
 /// accepted.
 class IncrementalCompiler {
   IncrementalKernelGenerator _generator;
+  IncrementalSerializer incrementalSerializer;
 
   // Component that reflect the state that was most recently accepted by the
   // client. Is [null], if no compilation results were accepted by the client.
@@ -34,9 +35,12 @@ class IncrementalCompiler {
   Uri get entryPoint => _entryPoint;
 
   IncrementalCompiler(this._compilerOptions, this._entryPoint,
-      {this.initializeFromDillUri}) {
-    _generator = new IncrementalKernelGenerator(
-        _compilerOptions, _entryPoint, initializeFromDillUri);
+      {this.initializeFromDillUri, bool incrementalSerialization}) {
+    if (incrementalSerialization == true) {
+      incrementalSerializer = new IncrementalSerializer();
+    }
+    _generator = new IncrementalKernelGenerator(_compilerOptions, _entryPoint,
+        initializeFromDillUri, false, incrementalSerializer);
     _pendingDeltas = <Component>[];
   }
 
@@ -127,8 +131,11 @@ class IncrementalCompiler {
     _pendingDeltas.clear();
     // Need to reset and warm up compiler so that expression evaluation requests
     // are processed in that known good state.
-    _generator = new IncrementalKernelGenerator.fromComponent(
-        _compilerOptions, _entryPoint, _lastKnownGood);
+    if (incrementalSerializer != null) {
+      incrementalSerializer = new IncrementalSerializer();
+    }
+    _generator = new IncrementalKernelGenerator.fromComponent(_compilerOptions,
+        _entryPoint, _lastKnownGood, false, incrementalSerializer);
     await _generator.computeDelta(entryPoints: [_entryPoint]);
   }