[dart2js] Support trimming modular analysis data.

Change-Id: I0e0902813fd71c0bda5de6171487fa0e941f6bad
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/241246
Reviewed-by: Sigmund Cherem <sigmund@google.com>
Commit-Queue: Joshua Litt <joshualitt@google.com>

pkg/compiler/lib/src/compiler.dart

@@ -246,9 +246,8 @@ class Compiler {
   /// Dumps a list of unused [ir.Library]'s in the [KernelResult]. This *must*
   /// be called before [setMainAndTrimComponent], because that method will
   /// discard the unused [ir.Library]s.
-  void dumpUnusedLibraries(ir.Component component, List<Uri> libraries) {
-    var usedUris = libraries.toSet();
-    bool isUnused(ir.Library l) => !usedUris.contains(l.importUri);
+  void dumpUnusedLibraries(ir.Component component, Set<Uri> libraries) {
+    bool isUnused(ir.Library l) => !libraries.contains(l.importUri);
     String libraryString(ir.Library library) {
       return '${library.importUri}(${library.fileUri})';
     }
@@ -270,7 +269,7 @@ class Compiler {
 
   /// Trims a component down to only the provided library uris.
   ir.Component trimComponent(
-      ir.Component component, List<Uri> librariesToInclude) {
+      ir.Component component, Set<Uri> librariesToInclude) {
     var irLibraryMap = <Uri, ir.Library>{};
     var irLibraries = <ir.Library>[];
     for (var library in component.libraries) {
@@ -325,11 +324,8 @@ class Compiler {
     }
   }
 
-  JClosedWorld computeClosedWorld(
-      ir.Component component,
-      List<ModuleData> moduleData,
-      Uri rootLibraryUri,
-      Iterable<Uri> libraries) {
+  JClosedWorld computeClosedWorld(ir.Component component, ModuleData moduleData,
+      Uri rootLibraryUri, Iterable<Uri> libraries) {
     frontendStrategy.registerLoadedLibraries(component, libraries);
     frontendStrategy.registerModuleData(moduleData);
     ResolutionEnqueuer resolutionEnqueuer = frontendStrategy
@@ -402,16 +398,31 @@ class Compiler {
         untrimmedComponentForDumpInfo = component;
       }
       if (options.cfeOnly) {
+        // [ModuleData] must be deserialized with the full component, i.e.
+        // before trimming.
+        ModuleData moduleData;
+        if (options.modularAnalysisInputs != null) {
+          moduleData = await serializationTask.deserializeModuleData(component);
+        }
+
+        Set<Uri> includedLibraries = output.libraries.toSet();
         if (options.fromDill) {
-          List<Uri> libraries = output.libraries;
           if (options.dumpUnusedLibraries) {
-            dumpUnusedLibraries(component, libraries);
+            dumpUnusedLibraries(component, includedLibraries);
           }
           if (options.entryUri != null) {
-            component = trimComponent(component, libraries);
+            component = trimComponent(component, includedLibraries);
           }
         }
-        await serializationTask.serializeComponent(component);
+        if (moduleData == null) {
+          await serializationTask.serializeComponent(component);
+        } else {
+          // Trim [moduleData] down to only the included libraries.
+          moduleData.impactData
+              .removeWhere((uri, _) => !includedLibraries.contains(uri));
+          await serializationTask.serializeModuleData(
+              moduleData, component, includedLibraries);
+        }
       }
       return output.withNewComponent(component);
     } else {
@@ -434,7 +445,7 @@ class Compiler {
         'runModularAnalysis', () async => modular_analysis.run(input));
   }
 
-  Future<List<ModuleData>> produceModuleData(load_kernel.Output output) async {
+  Future<ModuleData> produceModuleData(load_kernel.Output output) async {
     ir.Component component = output.component;
     if (options.modularMode) {
       Set<Uri> moduleLibraries = output.moduleLibraries.toSet();
@@ -444,7 +455,7 @@ class Compiler {
         serializationTask.serializeModuleData(
             moduleData, component, moduleLibraries);
       }
-      return [moduleData];
+      return moduleData;
     } else {
       return await serializationTask.deserializeModuleData(component);
     }
@@ -530,7 +541,7 @@ class Compiler {
   }
 
   Future<ClosedWorldAndIndices> produceClosedWorld(
-      load_kernel.Output output, List<ModuleData> moduleData) async {
+      load_kernel.Output output, ModuleData moduleData) async {
     ir.Component component = output.component;
     ClosedWorldAndIndices closedWorldAndIndices;
     if (options.readClosedWorldUri == null) {
@@ -626,7 +637,7 @@ class Compiler {
 
     // Run modular analysis. This may be null if modular analysis was not
     // requested for this pipeline.
-    List<ModuleData> moduleData;
+    ModuleData moduleData;
     if (options.modularMode || options.hasModularAnalysisInputs) {
       moduleData = await produceModuleData(output);
     }

pkg/compiler/lib/src/dart2js.dart

@@ -318,15 +318,13 @@ Future<api.CompilationResult> compile(List<String> argv,
       fail("Cannot use ${Flags.writeModularAnalysis} "
           "and write serialized codegen simultaneously.");
     }
-    if (writeStrategy == WriteStrategy.toKernel) {
-      fail("Cannot use ${Flags.writeModularAnalysis} "
-          "and run the CFE simultaneously.");
-    }
     if (argument != Flags.writeModularAnalysis) {
       writeModularAnalysisUri =
           fe.nativeToUri(extractPath(argument, isDirectory: false));
     }
-    writeStrategy = WriteStrategy.toModularAnalysis;
+    writeStrategy = writeStrategy == WriteStrategy.toKernel
+        ? WriteStrategy.toKernelWithModularAnalysis
+        : WriteStrategy.toModularAnalysis;
   }
 
   void setReadData(String argument) {
@@ -371,10 +369,6 @@ Future<api.CompilationResult> compile(List<String> argv,
   }
 
   void setCfeOnly(String argument) {
-    if (writeStrategy == WriteStrategy.toModularAnalysis) {
-      fail("Cannot use ${Flags.cfeOnly} "
-          "and write serialized modular analysis simultaneously.");
-    }
     if (writeStrategy == WriteStrategy.toClosedWorld) {
       fail("Cannot use ${Flags.cfeOnly} "
           "and write serialized closed world simultaneously.");
@@ -387,7 +381,9 @@ Future<api.CompilationResult> compile(List<String> argv,
       fail("Cannot use ${Flags.cfeOnly} "
           "and write serialized codegen simultaneously.");
     }
-    writeStrategy = WriteStrategy.toKernel;
+    writeStrategy = writeStrategy == WriteStrategy.toModularAnalysis
+        ? WriteStrategy.toKernelWithModularAnalysis
+        : WriteStrategy.toKernel;
   }
 
   void setReadCodegen(String argument) {
@@ -825,6 +821,12 @@ Future<api.CompilationResult> compile(List<String> argv,
             "and read serialized codegen simultaneously.");
       }
       break;
+    case WriteStrategy.toKernelWithModularAnalysis:
+      out ??= Uri.base.resolve('out.dill');
+      options.add(Flags.cfeOnly);
+      writeModularAnalysisUri ??= Uri.base.resolve('$out.mdata');
+      options.add('${Flags.writeModularAnalysis}=${writeModularAnalysisUri}');
+      break;
     case WriteStrategy.toModularAnalysis:
       writeModularAnalysisUri ??= Uri.base.resolve('$out.mdata');
       options.add('${Flags.writeModularAnalysis}=${writeModularAnalysisUri}');
@@ -1007,6 +1009,15 @@ Future<api.CompilationResult> compile(List<String> argv,
         String output = fe.relativizeUri(Uri.base, out, Platform.isWindows);
         summary += 'compiled to dill: ${output}.';
         break;
+      case WriteStrategy.toKernelWithModularAnalysis:
+        processName = 'Compiled';
+        outputName = 'kernel and bytes data';
+        outputSize = outputProvider.totalDataWritten;
+        String output = fe.relativizeUri(Uri.base, out, Platform.isWindows);
+        String dataOutput = fe.relativizeUri(
+            Uri.base, writeModularAnalysisUri, Platform.isWindows);
+        summary += 'compiled to dill and data: ${output} and ${dataOutput}.';
+        break;
       case WriteStrategy.toModularAnalysis:
         processName = 'Serialized';
         outputName = 'bytes data';
@@ -1547,6 +1558,7 @@ enum ReadStrategy {
 
 enum WriteStrategy {
   toKernel,
+  toKernelWithModularAnalysis,
   toModularAnalysis,
   toClosedWorld,
   toData,

pkg/compiler/lib/src/ir/modular.dart

@@ -46,27 +46,45 @@ abstract class ModularStrategy {
       ir.Member node, EnumSet<PragmaAnnotation> pragmaAnnotations);
 }
 
-/// Data computed for an entire compilation module.
+/// [ModuleData] is the data computed modularly, i.e. modularly computed impact
+/// data. Currently, we aggregate this data when computing the closed world, so it
+/// reflects all of the modularly computed data across the entire program.
 class ModuleData {
   static const String tag = 'ModuleData';
 
   // TODO(joshualitt) Support serializing ModularMemberData;
-  final Map<ir.Member, ImpactBuilderData> impactData;
+  final Map<Uri, Map<ir.Member, ImpactBuilderData>> impactData;
 
-  ModuleData(this.impactData);
+  ModuleData([Map<Uri, Map<ir.Member, ImpactBuilderData>> impactData])
+      : this.impactData = impactData ?? {};
 
-  factory ModuleData.fromDataSource(DataSourceReader source) {
+  factory ModuleData.fromImpactData(
+          Map<Uri, Map<ir.Member, ImpactBuilderData>> impactData) =>
+      ModuleData(impactData);
+
+  ModuleData readMoreFromDataSource(DataSourceReader source) {
     source.begin(tag);
-    var impactData = source
-        .readMemberNodeMap(() => ImpactBuilderData.fromDataSource(source));
+    int uriCount = source.readInt();
+    for (int i = 0; i < uriCount; i++) {
+      Uri uri = source.readUri();
+      impactData[uri] = source
+          .readMemberNodeMap(() => ImpactBuilderData.fromDataSource(source));
+    }
     source.end(tag);
-    return ModuleData(impactData);
+    return this;
   }
 
+  factory ModuleData.fromDataSource(DataSourceReader source) =>
+      ModuleData().readMoreFromDataSource(source);
+
   void toDataSink(DataSinkWriter sink) {
     sink.begin(tag);
-    sink.writeMemberNodeMap<ImpactBuilderData>(
-        impactData, (e) => e.toDataSink(sink));
+    sink.writeInt(impactData.keys.length);
+    impactData.forEach((uri, data) {
+      sink.writeUri(uri);
+      sink.writeMemberNodeMap<ImpactBuilderData>(
+          data, (e) => e.toDataSink(sink));
+    });
     sink.end(tag);
   }
 }

pkg/compiler/lib/src/kernel/dart2js_target.dart

@@ -235,6 +235,14 @@ class Dart2jsTarget extends Target {
       const Dart2jsDartLibrarySupport();
 }
 
+const implicitlyUsedLibraries = <String>[
+  'dart:_foreign_helper',
+  'dart:_interceptors',
+  'dart:_js_helper',
+  'dart:_late_helper',
+  'dart:js_util'
+];
+
 // TODO(sigmund): this "extraRequiredLibraries" needs to be removed...
 // compile-platform should just specify which libraries to compile instead.
 const requiredLibraries = <String, List<String>>{

pkg/compiler/lib/src/kernel/kernel_strategy.dart

@@ -229,7 +229,7 @@ class KernelFrontendStrategy {
     }
   }
 
-  void registerModuleData(List<ModuleData> data) {
+  void registerModuleData(ModuleData data) {
     if (data == null) {
       _modularStrategy = KernelModularStrategy(_compilerTask, _elementMap);
     } else {
@@ -452,9 +452,10 @@ class DeserializedModularStrategy extends ModularStrategy {
   final Map<ir.Member, ImpactBuilderData> _cache = {};
 
   DeserializedModularStrategy(
-      this._compilerTask, this._elementMap, List<ModuleData> data) {
-    for (var module in data) {
-      _cache.addAll(module.impactData);
+      this._compilerTask, this._elementMap, ModuleData data) {
+    for (Map<ir.Member, ImpactBuilderData> moduleData
+        in data.impactData.values) {
+      _cache.addAll(moduleData);
     }
   }
 

pkg/compiler/lib/src/options.dart

@@ -208,7 +208,7 @@ class CompilerOptions implements DiagnosticOptions {
   Uri? writeModularAnalysisUri;
 
   /// Helper to determine if compiler is being run just for modular analysis.
-  bool get modularMode => writeModularAnalysisUri != null;
+  bool get modularMode => writeModularAnalysisUri != null && !cfeOnly;
 
   List<Uri>? modularAnalysisInputs;
 

pkg/compiler/lib/src/phase/load_kernel.dart

@@ -18,7 +18,8 @@ import '../../compiler.dart' as api;
 import '../commandline_options.dart';
 import '../common.dart';
 import '../kernel/front_end_adapter.dart';
-import '../kernel/dart2js_target.dart' show Dart2jsTarget;
+import '../kernel/dart2js_target.dart'
+    show Dart2jsTarget, implicitlyUsedLibraries;
 import '../kernel/transformations/clone_mixin_methods_with_super.dart'
     as transformMixins show transformLibraries;
 import '../options.dart';
@@ -164,9 +165,11 @@ Future<_LoadFromKernelResult> _loadFromKernel(CompilerOptions options,
   _inferNullSafetyMode(options, isStrongDill);
   _validateNullSafetyMode(options);
 
-  // Modular compiles do not include the platform on the input dill
-  // either.
-  if (options.platformBinaries != null) {
+  // When compiling modularly, a dill for the SDK will be provided. In those
+  // cases we ignore the implicit platform binary.
+  bool platformBinariesIncluded =
+      options.modularMode || options.hasModularAnalysisInputs;
+  if (options.platformBinaries != null && !platformBinariesIncluded) {
     var platformUri = options.platformBinaries
         .resolve(_getPlatformFilename(options, targetName));
     // Modular analysis can be run on the sdk by providing directly the
@@ -331,11 +334,20 @@ Output _createOutput(
 
     search(root);
 
-    // Libraries dependencies do not show implicit imports to `dart:core`.
-    var dartCore = component.libraries.firstWhere((lib) {
-      return lib.importUri.isScheme('dart') && lib.importUri.path == 'core';
-    });
-    search(dartCore);
+    // Libraries dependencies do not show implicit imports to certain internal
+    // libraries.
+    const Set<String> alwaysInclude = {
+      'dart:_internal',
+      'dart:core',
+      'dart:async',
+      ...implicitlyUsedLibraries,
+    };
+    for (String uri in alwaysInclude) {
+      Library library = component.libraries.firstWhere((lib) {
+        return '${lib.importUri}' == uri;
+      });
+      search(library);
+    }
 
     libraries = libraries.where(seen.contains);
   }

pkg/compiler/lib/src/phase/modular_analysis.dart

@@ -53,12 +53,12 @@ KernelToElementMap _createElementMap(
   return elementMap;
 }
 
-ModuleData run(Input input) {
-  final options = input.options;
-  final reporter = input.reporter;
-  final elementMap = _createElementMap(
-      options, reporter, input.environment, input.component, input.libraries);
-  final result = <ir.Member, ImpactBuilderData>{};
+Map<ir.Member, ImpactBuilderData> _computeForLibrary(
+    CompilerOptions options,
+    DiagnosticReporter reporter,
+    KernelToElementMap elementMap,
+    ir.Library library) {
+  Map<ir.Member, ImpactBuilderData> result = {};
   void computeForMember(ir.Member member) {
     final scopeModel = ScopeModel.from(member, elementMap.constantEvaluator);
     final annotations = processMemberAnnotations(
@@ -68,12 +68,23 @@ ModuleData run(Input input) {
             .impactBuilderData;
   }
 
+  library.members.forEach(computeForMember);
+  for (final cls in library.classes) {
+    cls.members.forEach(computeForMember);
+  }
+  return result;
+}
+
+ModuleData run(Input input) {
+  final options = input.options;
+  final reporter = input.reporter;
+  final elementMap = _createElementMap(
+      options, reporter, input.environment, input.component, input.libraries);
+  Map<Uri, Map<ir.Member, ImpactBuilderData>> result = {};
   for (final library in input.component.libraries) {
     if (!input.moduleLibraries.contains(library.importUri)) continue;
-    library.members.forEach(computeForMember);
-    for (final cls in library.classes) {
-      cls.members.forEach(computeForMember);
-    }
+    result[library.importUri] =
+        _computeForLibrary(options, reporter, elementMap, library);
   }
-  return ModuleData(result);
+  return ModuleData.fromImpactData(result);
 }

pkg/compiler/lib/src/serialization/task.dart

@@ -198,7 +198,6 @@ class SerializationTask extends CompilerTask {
       //   DataSource source = new ObjectSource(encoding, useDataKinds: true);
       //   source.registerComponentLookup(new ComponentLookup(component));
       //   ModuleData.fromDataSource(source);
-
       BytesSink bytes = BytesSink();
       DataSinkWriter binarySink =
           DataSinkWriter(BinaryDataSink(bytes), useDataKinds: true);
@@ -211,17 +210,17 @@ class SerializationTask extends CompilerTask {
     }
   }
 
-  Future<List<ModuleData>> deserializeModuleData(ir.Component component) async {
+  Future<ModuleData> deserializeModuleData(ir.Component component) async {
     return await measureIoSubtask('deserialize module data', () async {
       _reporter.log('Reading data from ${_options.modularAnalysisInputs}');
-      List<ModuleData> results = [];
+      final results = ModuleData();
       for (Uri uri in _options.modularAnalysisInputs) {
         api.Input<List<int>> dataInput =
             await _provider.readFromUri(uri, inputKind: api.InputKind.binary);
         DataSourceReader source =
             DataSourceReader(BinaryDataSource(dataInput.data));
         source.registerComponentLookup(ComponentLookup(component));
-        results.add(ModuleData.fromDataSource(source));
+        results.readMoreFromDataSource(source);
       }
       return results;
     });

pkg/compiler/tool/modular_test_suite.dart

@@ -26,9 +26,7 @@ main(List<String> args) async {
           FullDillCompilationStep(onlyOnSdk: true),
           ModularAnalysisStep(onlyOnSdk: true),
           ModularAnalysisStep(),
-          // TODO(joshualitt): Re-enable ConcatenateDillStep after it works
-          // correctly alongside modular analysis.
-          // ConcatenateDillsStep(useModularAnalysis: true),
+          ConcatenateDillsStep(useModularAnalysis: true),
           ComputeClosedWorldStep(useModularAnalysis: true),
           GlobalAnalysisStep(),
           Dart2jsCodegenStep(codeId0),

pkg/compiler/tool/modular_test_suite_helper.dart

@@ -32,6 +32,7 @@ const dillId = DataId("full.dill");
 const fullDillId = DataId("concatenate.dill");
 const modularUpdatedDillId = DataId("modular.dill");
 const modularDataId = DataId("modular.data");
+const modularFullDataId = DataId("concatenate.modular.data");
 const closedWorldId = DataId("world");
 const globalUpdatedDillId = DataId("global.dill");
 const globalDataId = DataId("global.data");
@@ -304,10 +305,16 @@ class ConcatenateDillsStep extends IOModularStep {
 
   DataId get idForDill => useModularAnalysis ? modularUpdatedDillId : dillId;
 
-  List<DataId> get dependencies => [idForDill];
+  List<DataId> get dependencies => [
+        idForDill,
+        if (useModularAnalysis) modularDataId,
+      ];
 
   @override
-  List<DataId> get resultData => const [fullDillId];
+  List<DataId> get resultData => [
+        fullDillId,
+        if (useModularAnalysis) modularFullDataId,
+      ];
 
   @override
   bool get needsSources => false;
@@ -344,6 +351,10 @@ class ConcatenateDillsStep extends IOModularStep {
       '${Flags.inputDill}=${toUri(module, dillId)}',
       for (String flag in flags) '--enable-experiment=$flag',
       '${Flags.dillDependencies}=${dillDependencies.join(',')}',
+      if (useModularAnalysis) ...[
+        '${Flags.readModularAnalysis}=${dataDependencies.join(',')}',
+        '${Flags.writeModularAnalysis}=${toUri(module, modularFullDataId)}',
+      ],
       '${Flags.cfeOnly}',
       '--out=${toUri(module, fullDillId)}',
     ];
@@ -364,14 +375,11 @@ class ConcatenateDillsStep extends IOModularStep {
 class ComputeClosedWorldStep extends IOModularStep {
   final bool useModularAnalysis;
 
-  DataId get idForDill =>
-      useModularAnalysis ? modularUpdatedDillId : fullDillId;
-
   ComputeClosedWorldStep({this.useModularAnalysis});
 
   List<DataId> get dependencies => [
-        idForDill,
-        if (useModularAnalysis) modularDataId,
+        fullDillId,
+        if (useModularAnalysis) modularFullDataId,
       ];
 
   @override
@@ -394,25 +402,16 @@ class ComputeClosedWorldStep extends IOModularStep {
       List<String> flags) async {
     if (_options.verbose)
       print("\nstep: dart2js compute closed world on $module");
-    Set<Module> transitiveDependencies = computeTransitiveDependencies(module);
-    Iterable<String> dillDependencies =
-        transitiveDependencies.map((m) => '${toUri(m, idForDill)}');
-    List<String> dataDependencies = transitiveDependencies
-        .map((m) => '${toUri(m, modularDataId)}')
-        .toList();
-    dataDependencies.add('${toUri(module, modularDataId)}');
     List<String> args = [
       '--packages=${sdkRoot.toFilePath()}/$packageConfigJsonPath',
       _dart2jsScript,
       // TODO(sigmund): remove this dependency on libraries.json
       if (_options.useSdk) '--libraries-spec=$_librarySpecForSnapshot',
       '${Flags.entryUri}=$fakeRoot${module.mainSource}',
-      '${Flags.inputDill}=${toUri(module, idForDill)}',
+      '${Flags.inputDill}=${toUri(module, fullDillId)}',
       for (String flag in flags) '--enable-experiment=$flag',
-      if (useModularAnalysis) ...[
-        '${Flags.dillDependencies}=${dillDependencies.join(',')}',
-        '${Flags.readModularAnalysis}=${dataDependencies.join(',')}',
-      ],
+      if (useModularAnalysis)
+        '${Flags.readModularAnalysis}=${toUri(module, modularFullDataId)}',
       '${Flags.writeClosedWorld}=${toUri(module, closedWorldId)}',
       Flags.noClosedWorldInData,
       '--out=${toUri(module, globalUpdatedDillId)}',

tests/modular/unused_library_and_module/main.dart

@@ -0,0 +1,11 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:expect/expect.dart';
+
+import 'used_module/used_library.dart';
+
+main() {
+  Expect.equals(usedConstant, 1);
+}

tests/modular/unused_library_and_module/modules.yaml

@@ -0,0 +1,8 @@
+# Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+#
+# Regression test: integral numbers should be treated as int and not double
+# after serialization across modules.
+dependencies:
+  main: [expect, used_module, unused_module]

tests/modular/unused_library_and_module/unused_module.dart

@@ -0,0 +1,4 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+const int unusedConstant2 = 3;

tests/modular/unused_library_and_module/used_module/unused_library.dart

@@ -0,0 +1,4 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+const int unusedConstant1 = 2;

tests/modular/unused_library_and_module/used_module/used_library.dart

@@ -0,0 +1,4 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+const int usedConstant = 1;