Allow bazels kernel_worker to output used inputs

This change basically consists of these steps:
* Enable the incremental compiler to trace used inputs.
* Translate used libraries into used dill inputs.
* Output the list of used dills.

Bug: #37788
Change-Id: I08cffe299166cf10e990c9e261f190afd25da8b1
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/112384
Commit-Queue: Jens Johansen <jensj@google.com>
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Reviewed-by: Jake Macdonald <jakemac@google.com>

pkg/front_end/lib/src/api_unstable/bazel_worker.dart

@@ -63,7 +63,8 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
     Target target,
     FileSystem fileSystem,
     Iterable<String> experiments,
-    bool outlineOnly) async {
+    bool outlineOnly,
+    {bool trackNeededDillLibraries: false}) async {
   final List<int> sdkDigest = workerInputDigests[sdkSummary];
   if (sdkDigest == null) {
     throw new StateError("Expected to get digest for $sdkSummary");
@@ -121,6 +122,7 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
         new CompilerContext(processedOpts),
         cachedSdkInput.component,
         outlineOnly);
+    incrementalCompiler.trackNeededDillLibraries = trackNeededDillLibraries;
   } else {
     options = oldState.options;
     processedOpts = oldState.processedOpts;
@@ -147,6 +149,7 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
     // Reuse the incremental compiler, but reset as needed.
     incrementalCompiler = oldState.incrementalCompiler;
     incrementalCompiler.invalidateAllSources();
+    incrementalCompiler.trackNeededDillLibraries = trackNeededDillLibraries;
     options.packagesFileUri = packagesFile;
     options.fileSystem = fileSystem;
     processedOpts.clearFileSystemCache();
@@ -155,6 +158,10 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
   // Then read all the input summary components.
   CanonicalName nameRoot = cachedSdkInput.component.root;
   final inputSummaries = <Component>[];
+  Map<Uri, Uri> libraryToInputDill;
+  if (trackNeededDillLibraries) {
+    libraryToInputDill = new Map<Uri, Uri>();
+  }
   List<Uri> loadFromDill = new List<Uri>();
   for (Uri summary in summaryInputs) {
     var cachedInput = workerInputCache[summary];
@@ -171,6 +178,9 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
       var component = cachedInput.component;
       for (var lib in component.libraries) {
         lib.isExternal = cachedInput.externalLibs.contains(lib.importUri);
+        if (trackNeededDillLibraries) {
+          libraryToInputDill[lib.importUri] = summary;
+        }
       }
       inputSummaries.add(component);
     }
@@ -189,13 +199,19 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
             alwaysCreateNewNamedNodes: true));
     workerInputCache[summary] = cachedInput;
     inputSummaries.add(cachedInput.component);
+    if (trackNeededDillLibraries) {
+      for (var lib in cachedInput.component.libraries) {
+        libraryToInputDill[lib.importUri] = summary;
+      }
+    }
   }
 
   incrementalCompiler.setModulesToLoadOnNextComputeDelta(inputSummaries);
 
   return new InitializedCompilerState(options, processedOpts,
       workerInputCache: workerInputCache,
-      incrementalCompiler: incrementalCompiler);
+      incrementalCompiler: incrementalCompiler,
+      libraryToInputDill: libraryToInputDill);
 }
 
 Future<InitializedCompilerState> initializeCompiler(

pkg/front_end/lib/src/api_unstable/compiler_state.dart

@@ -16,9 +16,12 @@ class InitializedCompilerState {
   final ProcessedOptions processedOpts;
   final Map<Uri, WorkerInputComponent> workerInputCache;
   final IncrementalCompiler incrementalCompiler;
+  final Map<Uri, Uri> libraryToInputDill;
 
   InitializedCompilerState(this.options, this.processedOpts,
-      {this.workerInputCache, this.incrementalCompiler});
+      {this.workerInputCache,
+      this.incrementalCompiler,
+      this.libraryToInputDill});
 }
 
 /// A cached [Component] for a summary input file.

utils/bazel/kernel_worker.dart

@@ -139,6 +139,7 @@ final summaryArgsParser = new ArgParser()
   ..addOption('output')
   ..addFlag('reuse-compiler-result', defaultsTo: false)
   ..addFlag('use-incremental-compiler', defaultsTo: false)
+  ..addOption('used-inputs')
   ..addFlag('track-widget-creation', defaultsTo: false)
   ..addMultiOption('enable-experiment',
       help: 'Enable a language experiment when invoking the CFE.');
@@ -242,6 +243,7 @@ Future<ComputeKernelResult> computeKernel(List<String> args,
 
   fe.InitializedCompilerState state;
   bool usingIncrementalCompiler = false;
+  bool recordUsedInputs = parsedArgs["used-inputs"] != null;
   if (parsedArgs['use-incremental-compiler'] &&
       linkedInputs.isEmpty &&
       isWorker) {
@@ -264,7 +266,8 @@ Future<ComputeKernelResult> computeKernel(List<String> args,
         target,
         fileSystem,
         (parsedArgs['enable-experiment'] as List<String>),
-        summaryOnly);
+        summaryOnly,
+        trackNeededDillLibraries: recordUsedInputs);
   } else {
     state = await fe.initializeCompiler(
         // TODO(sigmund): pass an old state once we can make use of it.
@@ -285,11 +288,29 @@ Future<ComputeKernelResult> computeKernel(List<String> args,
   }
 
   List<int> kernel;
+  bool wroteUsedDills = false;
   if (usingIncrementalCompiler) {
     state.options.onDiagnostic = onDiagnostic;
     Component incrementalComponent = await state.incrementalCompiler
         .computeDelta(entryPoints: sources, fullComponent: true);
 
+    if (recordUsedInputs) {
+      Set<Uri> usedOutlines = {};
+      for (Library lib in state.incrementalCompiler.neededDillLibraries) {
+        if (lib.importUri.scheme == "dart") continue;
+        Uri uri = state.libraryToInputDill[lib.importUri];
+        if (uri == null) {
+          throw new StateError("Library ${lib.importUri} was recorded as used, "
+              "but was not in the list of known libraries.");
+        }
+        usedOutlines.add(uri);
+      }
+      var outputUsedFile = new File(parsedArgs["used-inputs"]);
+      outputUsedFile.createSync(recursive: true);
+      outputUsedFile.writeAsStringSync(usedOutlines.join("\n"));
+      wroteUsedDills = true;
+    }
+
     kernel = await state.incrementalCompiler.context.runInContext((_) {
       if (summaryOnly) {
         incrementalComponent.uriToSource.clear();
@@ -316,6 +337,15 @@ Future<ComputeKernelResult> computeKernel(List<String> args,
   }
   state.options.onDiagnostic = null; // See http://dartbug.com/36983.
 
+  if (!wroteUsedDills && recordUsedInputs) {
+    // The path taken didn't record inputs used: Say we used everything.
+    var outputUsedFile = new File(parsedArgs["used-inputs"]);
+    outputUsedFile.createSync(recursive: true);
+    Set<Uri> allFiles = {...summaryInputs, ...linkedInputs};
+    outputUsedFile.writeAsStringSync(allFiles.join("\n"));
+    wroteUsedDills = true;
+  }
+
   if (kernel != null) {
     var outputFile = new File(parsedArgs['output']);
     outputFile.createSync(recursive: true);