Add LimitedBinaryPrinter, tests and switch incremental generator to it.

R=ahe@google.com, kmillikin@google.com, paulberry@google.com, sigmund@google.com
BUG=

Review-Url: https://codereview.chromium.org/2896493002 .

pkg/front_end/lib/src/incremental_kernel_generator_impl.dart

@@ -18,7 +18,7 @@ import 'package:front_end/src/fasta/translate_uri.dart';
 import 'package:front_end/src/incremental/byte_store.dart';
 import 'package:front_end/src/incremental/file_state.dart';
 import 'package:kernel/binary/ast_from_binary.dart';
-import 'package:kernel/binary/ast_to_binary.dart';
+import 'package:kernel/binary/limited_ast_to_binary.dart';
 import 'package:kernel/kernel.dart' hide Source;
 
 dynamic unimplemented() {
@@ -121,6 +121,7 @@ class IncrementalKernelGeneratorImpl implements IncrementalKernelGenerator {
           if (_latestSignature[uri] != result.signature) {
             _latestSignature[uri] = result.signature;
             program.libraries.add(library);
+            library.parent = program;
           }
         }
       }
@@ -225,6 +226,7 @@ class IncrementalKernelGeneratorImpl implements IncrementalKernelGenerator {
 
       _logger.run('Serialize ${kernelLibraries.length} libraries', () {
         program.unbindCanonicalNames();
+        program.uriToSource.clear();
         List<int> bytes = _writeProgramBytes(program, kernelLibraries.contains);
         _byteStore.put(kernelKey, bytes);
         _logger.writeln('Stored ${bytes.length} bytes.');
@@ -284,8 +286,7 @@ class IncrementalKernelGeneratorImpl implements IncrementalKernelGenerator {
 
   List<int> _writeProgramBytes(Program program, bool filter(Library library)) {
     ByteSink byteSink = new ByteSink();
-    new LibraryFilteringBinaryPrinter(byteSink, filter)
-        .writeProgramFile(program);
+    new LimitedBinaryPrinter(byteSink, filter).writeProgramFile(program);
     return byteSink.builder.takeBytes();
   }
 }

pkg/front_end/test/incremental_kernel_generator_test.dart

@@ -8,8 +8,12 @@ import 'package:front_end/compiler_options.dart';
 import 'package:front_end/incremental_kernel_generator.dart';
 import 'package:front_end/memory_file_system.dart';
 import 'package:front_end/src/incremental/byte_store.dart';
+import 'package:front_end/src/incremental_kernel_generator_impl.dart';
 import 'package:kernel/ast.dart';
+import 'package:kernel/binary/ast_from_binary.dart';
+import 'package:kernel/binary/limited_ast_to_binary.dart';
 import 'package:kernel/text/ast_to_text.dart';
+import 'package:kernel/verifier.dart';
 import 'package:test/test.dart';
 import 'package:test_reflective_loader/test_reflective_loader.dart';
 
@@ -316,6 +320,104 @@ static field (core::String) → core::int f;
 ''');
   }
 
+  test_limited_ast_to_binary() async {
+    writeFile('/test/.packages', 'test:lib/');
+    String aPath = '/test/lib/a.dart';
+    String bPath = '/test/lib/b.dart';
+    writeFile(
+        aPath,
+        r'''
+int topField = 0;
+int get topGetter => 0;
+int topFunction({p}) => 0;
+        
+class A {
+  static int staticField;
+  static int get staticGetter => 0;
+  static int staticMethod() => 0;
+
+  int instanceField;
+  int get instanceGetter => 0;
+  int instanceMethod() => 0;
+  
+  A();
+  A.named();
+}
+''');
+    Uri bUri = writeFile(
+        bPath,
+        r'''
+import 'a.dart';
+
+class B extends A {
+  B() : super();
+  B.named() : super.named();
+  
+  void foo() {
+    super.instanceMethod();
+    instanceMethod();
+  }
+  
+  int instanceMethod() => 0;
+}
+
+main() {
+  topField;
+  topField = 0;
+  var v1 = topGetter;
+  var v2 = topFunction(p: 0);
+
+  A.staticField;
+  A.staticField = 0;
+  var v3 = A.staticGetter;
+  var v4 = A.staticMethod();
+
+  var a = new A();
+  a.instanceField;
+  a.instanceField = 0;
+  var v5 = a.instanceGetter;
+  var v6 = a.instanceMethod();
+}
+''');
+
+    Program program = await getInitialState(bUri);
+
+    String initialKernelText;
+    List<int> bytes;
+    {
+      Library initialLibrary = _getLibrary(program, bUri);
+      initialKernelText = _getLibraryText(initialLibrary);
+
+      var byteSink = new ByteSink();
+      var printer = new LimitedBinaryPrinter(
+          byteSink, (library) => library.importUri == bUri);
+      printer.writeProgramFile(program);
+      bytes = byteSink.builder.takeBytes();
+
+      // Remove b.dart from the program.
+      // So, the program is now ready for re-adding the library.
+      program.libraries.remove(initialLibrary);
+      program.root.removeChild(initialLibrary.importUri.toString());
+    }
+
+    // Load b.dart from bytes using the initial name root, so that
+    // serialized canonical names can be linked to corresponding nodes.
+    Library loadedLibrary;
+    {
+      var programForLoading = new Program(nameRoot: program.root);
+      var reader = new BinaryBuilder(bytes);
+      reader.readProgram(programForLoading);
+      loadedLibrary = _getLibrary(programForLoading, bUri);
+    }
+
+    // Add the library into the program.
+    program.libraries.add(loadedLibrary);
+    loadedLibrary.parent = program;
+
+    expect(_getLibraryText(loadedLibrary), initialKernelText);
+    verifyProgram(program);
+  }
+
   test_updateEntryPoint() async {
     writeFile('/test/.packages', 'test:lib/');
     String path = '/test/lib/test.dart';

pkg/kernel/lib/binary/ast_to_binary.dart

@@ -19,7 +19,7 @@ class BinaryPrinter extends Visitor {
   LabelIndexer _labelIndexer;
   SwitchCaseIndexer _switchCaseIndexer;
   final TypeParameterIndexer _typeParameterIndexer = new TypeParameterIndexer();
-  final StringIndexer _stringIndexer = new StringIndexer();
+  final StringIndexer stringIndexer;
   final StringIndexer _sourceUriIndexer = new StringIndexer();
   Map<LibraryDependency, int> _libraryDependencyIndex =
       <LibraryDependency, int>{};
@@ -34,7 +34,9 @@ class BinaryPrinter extends Visitor {
   /// If multiple binaries are to be written based on the same IR, a shared
   /// [globalIndexer] may be passed in to avoid rebuilding the same indices
   /// in every printer.
-  BinaryPrinter(Sink<List<int>> sink) : _sink = new BufferedSink(sink);
+  BinaryPrinter(Sink<List<int>> sink, {StringIndexer stringIndexer})
+      : _sink = new BufferedSink(sink),
+        stringIndexer = stringIndexer ?? new StringIndexer();
 
   void _flush() {
     _sink.flushAndDestroy();
@@ -90,7 +92,7 @@ class BinaryPrinter extends Visitor {
   }
 
   void writeStringReference(String string) {
-    writeUInt30(_stringIndexer[string]);
+    writeUInt30(stringIndexer[string]);
   }
 
   void writeStringReferenceList(List<String> strings) {
@@ -137,11 +139,24 @@ class BinaryPrinter extends Visitor {
     }
 
     for (var library in program.libraries) {
+      if (!shouldWriteLibraryCanonicalNames(library)) continue;
       visitCanonicalName(library.canonicalName);
     }
+    addCanonicalNamesForLinkTable(list);
     writeList(list, writeCanonicalNameEntry);
   }
 
+  /// Return `true` if all canonical names of the [library] should be written
+  /// into the link table.  If some libraries of the program are skipped,
+  /// then [addCanonicalNamesForLinkTable] should append all the additional
+  /// names referenced by the libraries that are written by [writeLibraries].
+  bool shouldWriteLibraryCanonicalNames(Library library) => true;
+
+  /// Append additional names for entities that are referenced by the
+  /// libraries that are written by [writeLibraries], but declared outside
+  /// of these libraries.
+  void addCanonicalNamesForLinkTable(List<CanonicalName> list) {}
+
   void writeCanonicalNameEntry(CanonicalName node) {
     var parent = node.parent;
     if (parent.isRoot) {
@@ -155,15 +170,25 @@ class BinaryPrinter extends Visitor {
   void writeProgramFile(Program program) {
     program.computeCanonicalNames();
     writeMagicWord(Tag.ProgramFile);
-    _stringIndexer.scanProgram(program);
-    writeStringTable(_stringIndexer);
+    buildStringIndex(program);
+    writeStringTable(stringIndexer);
     writeUriToSource(program);
     writeLinkTable(program);
-    writeList(program.libraries, writeNode);
+    writeLibraries(program);
     writeMemberReference(program.mainMethod, allowNull: true);
     _flush();
   }
 
+  /// Fill the [stringIndexer] with all strings we are going to reference.
+  void buildStringIndex(Program program) {
+    stringIndexer.scanProgram(program);
+  }
+
+  /// Write all of some of the libraries of the [program].
+  void writeLibraries(Program program) {
+    writeList(program.libraries, writeNode);
+  }
+
   void writeUriToSource(Program program) {
     program.uriToSource.keys.forEach((uri) {
       _sourceUriIndexer.put(uri);
@@ -1089,8 +1114,8 @@ class LibraryFilteringBinaryPrinter extends BinaryPrinter {
   void writeProgramFile(Program program) {
     program.computeCanonicalNames();
     writeMagicWord(Tag.ProgramFile);
-    _stringIndexer.scanProgram(program);
-    writeStringTable(_stringIndexer);
+    stringIndexer.scanProgram(program);
+    writeStringTable(stringIndexer);
     writeUriToSource(program);
     writeLinkTable(program);
     writeList(program.libraries.where(predicate).toList(), writeNode);
@@ -1197,8 +1222,19 @@ class StringIndexer extends RecursiveVisitor<Null> {
 
   int get numberOfStrings => index.length;
 
-  void scanProgram(Node node) {
-    node.accept(this);
+  /// Scan all the [program] libraries and [finish] indexing.
+  void scanProgram(Program program) {
+    program.accept(this);
+    finish();
+  }
+
+  /// Scan the given library, but don't [finish] indexing yet.
+  void scanLibrary(Library library) {
+    library.accept(this);
+  }
+
+  /// Finish building of the index - sort and assign indices for entries.
+  void finish() {
     entries.sort();
     for (int i = 0; i < entries.length; ++i) {
       index[entries[i].value] = i;

pkg/kernel/lib/binary/limited_ast_to_binary.dart

@@ -0,0 +1,75 @@
+// Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:kernel/ast.dart';
+import 'package:kernel/binary/ast_to_binary.dart';
+
+/// Writes libraries that satisfy the [predicate].
+///
+/// Only the referenced subset of canonical names is indexed and written,
+/// so we don't waste time indexing all libraries of a program, when only
+/// a tiny subset is used.
+class LimitedBinaryPrinter extends BinaryPrinter {
+  final LibraryFilter predicate;
+
+  LimitedBinaryPrinter(Sink<List<int>> sink, this.predicate)
+      : super(sink, stringIndexer: new ReferencesStringIndexer());
+
+  @override
+  void addCanonicalNamesForLinkTable(List<CanonicalName> list) {
+    ReferencesStringIndexer stringIndexer = this.stringIndexer;
+    stringIndexer.referencedNames.forEach((name) {
+      if (name.index != -1) return;
+      name.index = list.length;
+      list.add(name);
+    });
+  }
+
+  @override
+  void buildStringIndex(Program program) {
+    program.libraries.where(predicate).forEach((library) {
+      stringIndexer.scanLibrary(library);
+    });
+    stringIndexer.finish();
+  }
+
+  @override
+  bool shouldWriteLibraryCanonicalNames(Library library) {
+    return predicate(library);
+  }
+
+  @override
+  void writeLibraries(Program program) {
+    var librariesToWrite = program.libraries.where(predicate).toList();
+    writeList(librariesToWrite, writeNode);
+  }
+
+  @override
+  void writeNode(Node node) {
+    if (node is Library && !predicate(node)) return;
+    node.accept(this);
+  }
+}
+
+/// Extension of [StringIndexer] that also indexes canonical names of
+/// referenced classes and members.
+class ReferencesStringIndexer extends StringIndexer {
+  final List<CanonicalName> referencedNames = <CanonicalName>[];
+
+  defaultMemberReference(Member node) {
+    _handleReferencedName(node.canonicalName);
+  }
+
+  visitClassReference(Class node) {
+    _handleReferencedName(node.canonicalName);
+  }
+
+  void _handleReferencedName(CanonicalName name) {
+    if (name == null || name.parent == null) return;
+    _handleReferencedName(name.parent);
+    referencedNames.add(name);
+    name.index = -1;
+    put(name.name);
+  }
+}

pkg/kernel/lib/canonical_name.dart

@@ -108,6 +108,10 @@ class CanonicalName {
     return getChild('@typedefs').getChild(typedef_.name);
   }
 
+  void removeChild(String name) {
+    _children?.remove(name);
+  }
+
   void bindTo(Reference target) {
     if (reference == target) return;
     if (reference != null) {