Support prebuilt macros in the incremental compiler mode for DDC

Also adds support for modular tests with precompiled macros, and one test exercising that.

Change-Id: Ie372ea7fcb270ecd55baa54c4ed1c4ae5a527df4
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/361261
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Commit-Queue: Johnni Winther <johnniwinther@google.com>
Auto-Submit: Jake Macdonald <jakemac@google.com>
Reviewed-by: Nicholas Shahan <nshahan@google.com>

pkg/compiler/tool/modular_test_suite.dart

@@ -9,6 +9,7 @@ import 'dart:async';
 
 import 'package:modular_test/src/io_pipeline.dart';
 import 'package:modular_test/src/runner.dart';
+import 'package:modular_test/src/steps/macro_precompile_aot.dart';
 import 'modular_test_suite_helper.dart';
 
 Future<void> main(List<String> args) async {
@@ -20,6 +21,7 @@ Future<void> main(List<String> args) async {
         'tests/modular',
         options,
         IOPipeline([
+          PrecompileMacroAotStep(verbose: options.verbose),
           OutlineDillCompilationStep(),
           FullDillCompilationStep(),
           ConcatenateDillsStep(),

pkg/compiler/tool/modular_test_suite_helper.dart

@@ -18,6 +18,8 @@ import 'package:modular_test/src/io_pipeline.dart';
 import 'package:modular_test/src/pipeline.dart';
 import 'package:modular_test/src/runner.dart';
 import 'package:modular_test/src/suite.dart';
+import 'package:modular_test/src/steps/macro_precompile_aot.dart';
+import 'package:modular_test/src/steps/util.dart';
 
 String packageConfigJsonPath = ".dart_tool/package_config.json";
 Uri sdkRoot = Platform.script.resolve("../../../");
@@ -130,13 +132,17 @@ abstract class CFEStep extends IOModularStep {
       '${toUri(module, outputData)}',
       ...(transitiveDependencies
           .expand((m) => ['--input-summary', '${toUri(m, inputData)}'])),
+      ...transitiveDependencies
+          .where((m) => m.macroConstructors.isNotEmpty)
+          .expand((m) =>
+              ['--precompiled-macro', '${precompiledMacroArg(m, toUri)};']),
       ...(sources.expand((String uri) => ['--source', uri])),
       ...(flags.expand((String flag) => ['--enable-experiment', flag])),
     ];
 
-    var result =
-        await _runProcess(Platform.resolvedExecutable, args, root.toFilePath());
-    _checkExitCode(result, this, module);
+    var result = await runProcess(
+        Platform.resolvedExecutable, args, root.toFilePath(), _options.verbose);
+    checkExitCode(result, this, module, _options.verbose);
   }
 
   List<String> get stepArguments;
@@ -160,7 +166,8 @@ class OutlineDillCompilationStep extends CFEStep {
   bool get needsSources => true;
 
   @override
-  List<DataId> get dependencyDataNeeded => const [dillSummaryId];
+  List<DataId> get dependencyDataNeeded =>
+      const [dillSummaryId, precompiledMacroId];
 
   @override
   List<DataId> get moduleDataNeeded => const [];
@@ -187,7 +194,8 @@ class FullDillCompilationStep extends CFEStep {
   bool get needsSources => true;
 
   @override
-  List<DataId> get dependencyDataNeeded => const [dillSummaryId];
+  List<DataId> get dependencyDataNeeded =>
+      const [dillSummaryId, precompiledMacroId];
 
   @override
   List<DataId> get moduleDataNeeded => const [];
@@ -245,10 +253,10 @@ class ConcatenateDillsStep extends IOModularStep {
       '${Flags.stage}=cfe',
       '--out=${toUri(module, fullDillId)}',
     ];
-    var result =
-        await _runProcess(Platform.resolvedExecutable, args, root.toFilePath());
+    var result = await runProcess(
+        Platform.resolvedExecutable, args, root.toFilePath(), _options.verbose);
 
-    _checkExitCode(result, this, module);
+    checkExitCode(result, this, module, _options.verbose);
   }
 
   @override
@@ -297,10 +305,10 @@ class ComputeClosedWorldStep extends IOModularStep {
       '${Flags.closedWorldUri}=${toUri(module, closedWorldId)}',
       '${Flags.stage}=closed-world',
     ];
-    var result =
-        await _runProcess(Platform.resolvedExecutable, args, root.toFilePath());
+    var result = await runProcess(
+        Platform.resolvedExecutable, args, root.toFilePath(), _options.verbose);
 
-    _checkExitCode(result, this, module);
+    checkExitCode(result, this, module, _options.verbose);
   }
 
   @override
@@ -345,10 +353,10 @@ class GlobalAnalysisStep extends IOModularStep {
       '${Flags.globalInferenceUri}=${toUri(module, globalDataId)}',
       '${Flags.stage}=global-inference',
     ];
-    var result =
-        await _runProcess(Platform.resolvedExecutable, args, root.toFilePath());
+    var result = await runProcess(
+        Platform.resolvedExecutable, args, root.toFilePath(), _options.verbose);
 
-    _checkExitCode(result, this, module);
+    checkExitCode(result, this, module, _options.verbose);
   }
 
   @override
@@ -402,10 +410,10 @@ class Dart2jsCodegenStep extends IOModularStep {
       '${Flags.codegenShards}=${codeId.dataId.shards}',
       '${Flags.stage}=codegen',
     ];
-    var result =
-        await _runProcess(Platform.resolvedExecutable, args, root.toFilePath());
+    var result = await runProcess(
+        Platform.resolvedExecutable, args, root.toFilePath(), _options.verbose);
 
-    _checkExitCode(result, this, module);
+    checkExitCode(result, this, module, _options.verbose);
   }
 
   @override
@@ -454,10 +462,10 @@ class Dart2jsEmissionStep extends IOModularStep {
       '${Flags.stage}=emit-js',
       '--out=${toUri(module, jsId)}',
     ];
-    var result =
-        await _runProcess(Platform.resolvedExecutable, args, root.toFilePath());
+    var result = await runProcess(
+        Platform.resolvedExecutable, args, root.toFilePath(), _options.verbose);
 
-    _checkExitCode(result, this, module);
+    checkExitCode(result, this, module, _options.verbose);
   }
 
   @override
@@ -512,10 +520,10 @@ class Dart2jsDumpInfoStep extends IOModularStep {
       '${Flags.stage}=dump-info',
       '--out=${toUri(module, jsId)}',
     ];
-    var result =
-        await _runProcess(Platform.resolvedExecutable, args, root.toFilePath());
+    var result = await runProcess(
+        Platform.resolvedExecutable, args, root.toFilePath(), _options.verbose);
 
-    _checkExitCode(result, this, module);
+    checkExitCode(result, this, module, _options.verbose);
   }
 
   @override
@@ -551,10 +559,10 @@ class RunD8 extends IOModularStep {
           .toFilePath(),
       root.resolveUri(toUri(module, jsId)).toFilePath(),
     ];
-    var result = await _runProcess(
-        sdkRoot.resolve(_d8executable).toFilePath(), d8Args, root.toFilePath());
+    var result = await runProcess(sdkRoot.resolve(_d8executable).toFilePath(),
+        d8Args, root.toFilePath(), _options.verbose);
 
-    _checkExitCode(result, this, module);
+    checkExitCode(result, this, module, _options.verbose);
 
     await File.fromUri(root.resolveUri(toUri(module, txtId)))
         .writeAsString(result.stdout);
@@ -566,26 +574,6 @@ class RunD8 extends IOModularStep {
   }
 }
 
-void _checkExitCode(ProcessResult result, IOModularStep step, Module module) {
-  if (result.exitCode != 0 || _options.verbose) {
-    stdout.write(result.stdout);
-    stderr.write(result.stderr);
-  }
-  if (result.exitCode != 0) {
-    throw "${step.runtimeType} failed on $module:\n\n"
-        "stdout:\n${result.stdout}\n\n"
-        "stderr:\n${result.stderr}";
-  }
-}
-
-Future<ProcessResult> _runProcess(
-    String command, List<String> arguments, String workingDirectory) {
-  if (_options.verbose) {
-    print('command:\n$command ${arguments.join(' ')} from $workingDirectory');
-  }
-  return Process.run(command, arguments, workingDirectory: workingDirectory);
-}
-
 String get _d8executable {
   final arch = Abi.current().toString().split('_')[1];
   if (Platform.isWindows) {

pkg/dev_compiler/lib/src/kernel/command.dart

@@ -332,7 +332,8 @@ Future<CompilerResult> _compile(List<String> args,
         environmentDefines: declaredVariables,
         trackNeededDillLibraries: recordUsedInputs,
         nnbdMode:
-            options.soundNullSafety ? fe.NnbdMode.Strong : fe.NnbdMode.Weak);
+            options.soundNullSafety ? fe.NnbdMode.Strong : fe.NnbdMode.Weak,
+        precompiledMacros: options.precompiledMacros);
     var incrementalCompiler = compilerState.incrementalCompiler!;
     var cachedSdkInput = compileSdk
         ? null

pkg/dev_compiler/test/modular_helpers.dart

@@ -9,6 +9,8 @@ import 'package:modular_test/src/create_package_config.dart';
 import 'package:modular_test/src/io_pipeline.dart';
 import 'package:modular_test/src/pipeline.dart';
 import 'package:modular_test/src/runner.dart';
+import 'package:modular_test/src/steps/macro_precompile_aot.dart';
+import 'package:modular_test/src/steps/util.dart';
 import 'package:modular_test/src/suite.dart';
 
 String packageConfigJsonPath = '.dart_tool/package_config.json';
@@ -34,7 +36,7 @@ class SourceToSummaryDillStep implements IOModularStep {
   bool get needsSources => true;
 
   @override
-  List<DataId> get dependencyDataNeeded => const [dillId];
+  List<DataId> get dependencyDataNeeded => const [dillId, precompiledMacroId];
 
   @override
   List<DataId> get moduleDataNeeded => const [];
@@ -109,19 +111,26 @@ class SourceToSummaryDillStep implements IOModularStep {
       ...transitiveDependencies
           .where((m) => !m.isSdk)
           .expand((m) => ['--input-summary', '${toUri(m, dillId)}']),
+      ...transitiveDependencies
+          .where((m) => m.macroConstructors.isNotEmpty)
+          .expand((m) =>
+              ['--precompiled-macro', '${precompiledMacroArg(m, toUri)};']),
       ...sources.expand((String uri) => ['--source', uri]),
       ...flags.expand((String flag) => ['--enable-experiment', flag]),
     ];
 
-    var result =
-        await _runProcess(Platform.resolvedExecutable, args, root.toFilePath());
-    _checkExitCode(result, this, module);
+    var result = await runProcess(
+        Platform.resolvedExecutable, args, root.toFilePath(), _options.verbose);
+    checkExitCode(result, this, module, _options.verbose);
   }
 
   @override
   void notifyCached(Module module) {
     if (_options.verbose) print('\ncached step: source-to-dill on $module');
   }
+
+  @override
+  bool shouldExecute(Module module) => true;
 }
 
 class DDCStep implements IOModularStep {
@@ -137,7 +146,7 @@ class DDCStep implements IOModularStep {
   bool get needsSources => true;
 
   @override
-  List<DataId> get dependencyDataNeeded => const [dillId];
+  List<DataId> get dependencyDataNeeded => const [dillId, precompiledMacroId];
 
   @override
   List<DataId> get moduleDataNeeded => const [dillId];
@@ -202,18 +211,25 @@ class DDCStep implements IOModularStep {
       ...transitiveDependencies
           .where((m) => !m.isSdk)
           .expand((m) => ['-s', '${toUri(m, dillId)}=${m.name}']),
+      ...transitiveDependencies
+          .where((m) => m.macroConstructors.isNotEmpty)
+          .expand((m) =>
+              ['--precompiled-macro', '${precompiledMacroArg(m, toUri)};']),
       '-o',
       '$output',
     ];
-    var result =
-        await _runProcess(Platform.resolvedExecutable, args, root.toFilePath());
-    _checkExitCode(result, this, module);
+    var result = await runProcess(
+        Platform.resolvedExecutable, args, root.toFilePath(), _options.verbose);
+    checkExitCode(result, this, module, _options.verbose);
   }
 
   @override
   void notifyCached(Module module) {
     if (_options.verbose) print('\ncached step: ddc on $module');
   }
+
+  @override
+  bool shouldExecute(Module module) => true;
 }
 
 class RunD8 implements IOModularStep {
@@ -263,10 +279,10 @@ class RunD8 implements IOModularStep {
         '${root.resolveUri(toUri(module, jsId)).toFilePath()}.wrapper.js';
     await File(wrapper).writeAsString(runjs);
     var d8Args = ['--module', wrapper];
-    var result = await _runProcess(
-        sdkRoot.resolve(_d8executable).toFilePath(), d8Args, root.toFilePath());
+    var result = await runProcess(sdkRoot.resolve(_d8executable).toFilePath(),
+        d8Args, root.toFilePath(), _options.verbose);
 
-    _checkExitCode(result, this, module);
+    checkExitCode(result, this, module, _options.verbose);
 
     await File.fromUri(root.resolveUri(toUri(module, txtId)))
         .writeAsString(result.stdout as String);
@@ -276,26 +292,9 @@ class RunD8 implements IOModularStep {
   void notifyCached(Module module) {
     if (_options.verbose) print('\ncached step: d8 on $module');
   }
-}
 
-void _checkExitCode(ProcessResult result, IOModularStep step, Module module) {
-  if (result.exitCode != 0 || _options.verbose) {
-    stdout.write(result.stdout);
-    stderr.write(result.stderr);
-  }
-  if (result.exitCode != 0) {
-    throw '${step.runtimeType} failed on $module:\n\n'
-        'stdout:\n${result.stdout}\n\n'
-        'stderr:\n${result.stderr}';
-  }
-}
-
-Future<ProcessResult> _runProcess(
-    String command, List<String> arguments, String workingDirectory) {
-  if (_options.verbose) {
-    print('command:\n$command ${arguments.join(' ')} from $workingDirectory');
-  }
-  return Process.run(command, arguments, workingDirectory: workingDirectory);
+  @override
+  bool shouldExecute(Module module) => true;
 }
 
 String get _d8executable {

pkg/dev_compiler/test/modular_suite.dart

@@ -9,6 +9,7 @@ library;
 
 import 'package:modular_test/src/io_pipeline.dart';
 import 'package:modular_test/src/runner.dart';
+import 'package:modular_test/src/steps/macro_precompile_aot.dart';
 
 import 'modular_helpers.dart';
 
@@ -21,6 +22,7 @@ void main(List<String> args) async {
       'tests/modular',
       options,
       IOPipeline([
+        PrecompileMacroAotStep(verbose: options.verbose),
         SourceToSummaryDillStep(soundNullSafety: soundNullSafety),
         DDCStep(soundNullSafety: soundNullSafety, canaryFeatures: false),
         RunD8(),

pkg/dev_compiler/test/modular_suite_canary_unsound.dart

@@ -10,6 +10,7 @@ library;
 
 import 'package:modular_test/src/io_pipeline.dart';
 import 'package:modular_test/src/runner.dart';
+import 'package:modular_test/src/steps/macro_precompile_aot.dart';
 
 import 'modular_helpers.dart';
 
@@ -22,6 +23,7 @@ void main(List<String> args) async {
       'tests/modular',
       options,
       IOPipeline([
+        PrecompileMacroAotStep(verbose: options.verbose),
         SourceToSummaryDillStep(soundNullSafety: soundNullSafety),
         DDCStep(soundNullSafety: soundNullSafety, canaryFeatures: true),
         RunD8(),

pkg/dev_compiler/test/modular_suite_unsound.dart

@@ -9,6 +9,7 @@ library;
 
 import 'package:modular_test/src/io_pipeline.dart';
 import 'package:modular_test/src/runner.dart';
+import 'package:modular_test/src/steps/macro_precompile_aot.dart';
 
 import 'modular_helpers.dart';
 
@@ -21,6 +22,7 @@ void main(List<String> args) async {
       'tests/modular',
       options,
       IOPipeline([
+        PrecompileMacroAotStep(verbose: options.verbose),
         SourceToSummaryDillStep(soundNullSafety: soundNullSafety),
         DDCStep(soundNullSafety: soundNullSafety, canaryFeatures: false),
         RunD8(),

pkg/front_end/lib/src/api_unstable/ddc.dart

@@ -104,6 +104,7 @@ InitializedCompilerState initializeCompiler(
     Map<ExperimentalFlag, bool>? explicitExperimentalFlags,
     Map<String, String>? environmentDefines,
     required NnbdMode nnbdMode,
+    bool requirePrebuiltMacros = false,
     List<String>? precompiledMacros,
     String? macroSerializationMode}) {
   additionalDills.sort((a, b) => a.toString().compareTo(b.toString()));
@@ -117,7 +118,9 @@ InitializedCompilerState initializeCompiler(
       equalLists(oldState.options.additionalDills, additionalDills) &&
       equalMaps(oldState.options.explicitExperimentalFlags,
           explicitExperimentalFlags) &&
-      equalMaps(oldState.options.environmentDefines, environmentDefines)) {
+      equalMaps(oldState.options.environmentDefines, environmentDefines) &&
+      equalLists(oldState.options.precompiledMacros, precompiledMacros) &&
+      oldState.options.requirePrebuiltMacros == requirePrebuiltMacros) {
     // Reuse old state.
     return oldState;
   }
@@ -166,6 +169,8 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
     required Map<ExperimentalFlag, bool> explicitExperimentalFlags,
     required Map<String, String> environmentDefines,
     bool trackNeededDillLibraries = false,
+    bool requirePrebuiltMacros = false,
+    List<String> precompiledMacros = const [],
     required NnbdMode nnbdMode}) {
   return modular.initializeIncrementalCompiler(
       oldState,
@@ -184,6 +189,8 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
       environmentDefines: environmentDefines,
       outlineOnly: false,
       omitPlatform: false,
+      requirePrebuiltMacros: requirePrebuiltMacros,
+      precompiledMacros: precompiledMacros,
       trackNeededDillLibraries: trackNeededDillLibraries,
       nnbdMode: nnbdMode);
 }

pkg/modular_test/lib/src/create_package_config.dart

@@ -9,8 +9,10 @@ import 'package:package_config/package_config.dart';
 import 'find_sdk_root.dart';
 import 'suite.dart';
 
-Future<void> writePackageConfig(
-    Module module, Set<Module> transitiveDependencies, Uri root) async {
+/// Writes a package config under [root] and returns the [Uri] pointing to it.
+Future<Uri> writePackageConfig(
+    Module module, Set<Module> transitiveDependencies, Uri root,
+    {bool useRealPaths = false}) async {
   const packageConfigJsonPath = ".dart_tool/package_config.json";
   var sdkRoot = await findRoot();
   Uri packageConfigUri = sdkRoot.resolve(packageConfigJsonPath);
@@ -39,24 +41,27 @@ Future<void> writePackageConfig(
     if (dependency.isPackage) {
       // rootUri should be ignored for dependent modules, so we pass in a
       // bogus value.
-      var rootUri = Uri.parse('unused$unusedNum');
-      unusedNum++;
+      var rootUri =
+          useRealPaths ? dependency.rootUri : Uri.parse('unused${unusedNum++}');
 
       var dependentPackage = packageConfig[dependency.name];
       var packageJson = dependentPackage == null
-          ? _packageConfigEntry(dependency.name, rootUri)
+          ? _packageConfigEntry(dependency.name, rootUri,
+              packageRoot: dependency.packageBase)
           : _packageConfigEntry(dependentPackage.name, rootUri,
+              packageRoot: dependency.packageBase,
               version: dependentPackage.languageVersion);
       packagesJson.add(packageJson);
     }
   }
 
-  await File.fromUri(root.resolve(packageConfigJsonPath))
-      .create(recursive: true);
-  await File.fromUri(root.resolve(packageConfigJsonPath)).writeAsString('{'
+  var packageConfigFile = File.fromUri(root.resolve(packageConfigJsonPath));
+  await packageConfigFile.create(recursive: true);
+  await packageConfigFile.writeAsString('{'
       '  "configVersion": ${packageConfig.version},'
       '  "packages": [ ${packagesJson.join(',')} ]'
       '}');
+  return packageConfigFile.uri;
 }
 
 String _packageConfigEntry(String name, Uri root,

pkg/modular_test/lib/src/io_pipeline.dart

@@ -26,6 +26,10 @@ abstract class IOModularStep extends ModularStep {
   /// should be stored under `root.resolveUri(toUri(module, resultKind))`.
   Future<void> execute(Module module, Uri root, ModuleDataToRelativeUri toUri,
       List<String> flags);
+
+  /// Whether this step should apply to [module]. Most steps apply to all
+  /// modules, but not all.
+  bool shouldExecute(Module module) => true;
 }
 
 class IOPipeline extends Pipeline<IOModularStep> {
@@ -100,6 +104,9 @@ class IOPipeline extends Pipeline<IOModularStep> {
   @override
   Future<void> runStep(IOModularStep step, Module module,
       Map<Module, Set<DataId>> visibleData, List<String> flags) async {
+    // Skip it if we aren't expected to run.
+    if (!step.shouldExecute(module)) return;
+
     final resultsFolderUri = _resultsFolderUri!;
     if (cacheSharedModules && module.isShared) {
       // If all expected outputs are already available, skip the step.
@@ -127,8 +134,18 @@ class IOPipeline extends Pipeline<IOModularStep> {
       for (var dataId in visibleData[module]!) {
         var assetUri = resultsFolderUri
             .resolve(_toFileName(module, dataId, configSpecific: true));
-        await File.fromUri(assetUri).copy(
-            stepFolder.uri.resolve(_toFileName(module, dataId)).toFilePath());
+        var originalFile = File.fromUri(assetUri);
+        // Some steps don't actually have an output, if they implement
+        // shouldExecute.
+        if (!(await originalFile.exists())) continue;
+        var newPath =
+            stepFolder.uri.resolve(_toFileName(module, dataId)).toFilePath();
+        await originalFile.copy(newPath);
+        // If the input was executable, ensure it still is.
+        var originalMode = (await originalFile.stat()).modeString();
+        if (originalMode.contains('x')) {
+          await Process.run('chmod', [originalMode, newPath]);
+        }
       }
     }
     if (step.needsSources) {
@@ -148,11 +165,18 @@ class IOPipeline extends Pipeline<IOModularStep> {
           File.fromUri(stepFolder.uri.resolve(_toFileName(module, dataId)));
       if (!await outputFile.exists()) {
         throw StateError(
-            "Step '${step.runtimeType}' didn't produce an output file");
+            "Step '${step.runtimeType}' on module '${module.name}' didn't "
+            "produce an output file");
       }
-      await outputFile.copy(resultsFolderUri
+      var newPath = resultsFolderUri
           .resolve(_toFileName(module, dataId, configSpecific: true))
-          .toFilePath());
+          .toFilePath();
+      await outputFile.copy(newPath);
+      // If the output was executable, ensure it still is.
+      var originalMode = (await outputFile.stat()).modeString();
+      if (originalMode.contains('x')) {
+        await Process.run('chmod', [originalMode, newPath]);
+      }
     }
     await stepFolder.delete(recursive: true);
   }

pkg/modular_test/lib/src/loader.dart

@@ -60,7 +60,7 @@ Future<ModularTest> loadTest(Uri uri) async {
         }
         var relativeUri = Uri.parse(fileName);
         var isMain = moduleName == 'main';
-        var module = Module(moduleName, [], testUri, [relativeUri],
+        var module = Module(moduleName, [], testUri, [relativeUri], {},
             mainSource: isMain ? relativeUri : null,
             isMain: isMain,
             packageBase: Uri.parse('.'));
@@ -87,7 +87,7 @@ Future<ModularTest> loadTest(Uri uri) async {
         return _moduleConflict(moduleName, modules[moduleName]!, testUri);
       }
       var sources = await _listModuleSources(entryUri);
-      modules[moduleName] = Module(moduleName, [], testUri, sources,
+      modules[moduleName] = Module(moduleName, [], testUri, sources, {},
           packageBase: Uri.parse('$moduleName/'));
     }
   }
@@ -109,6 +109,7 @@ Future<ModularTest> loadTest(Uri uri) async {
   await _addModulePerPackage(spec.packages, testUri, modules);
   _attachDependencies(spec.dependencies, modules);
   _attachDependencies(defaultTestSpecification.dependencies, modules);
+  _attachMacros(spec.macros, modules);
   _addSdkDependencies(modules, sdkModule);
   _detectCyclesAndRemoveUnreachable(modules, mainModule);
   var sortedModules = modules.values.toList()
@@ -154,6 +155,20 @@ void _attachDependencies(
   });
 }
 
+void _attachMacros(Map<String, Map<String, Map<String, List<String>>>> macros,
+    Map<String, Module> modules) {
+  macros.forEach((name, macroConstructors) {
+    final module = modules[name];
+    if (module == null) {
+      _invalidTest("declared macros for a nonexistent module named '$name'");
+    }
+    if (module.macroConstructors.isNotEmpty) {
+      _invalidTest("Module macros have already been declared on $name.");
+    }
+    module.macroConstructors.addAll(macroConstructors);
+  });
+}
+
 /// Make every module depend on the sdk module.
 void _addSdkDependencies(Map<String, Module> modules, Module sdkModule) {
   for (var module in modules.values) {
@@ -177,7 +192,8 @@ Future<void> _addModulePerPackage(Map<String, String> packages, Uri configRoot,
       // TODO(sigmund): validate that we don't use a different alias for a
       // module that is part of the test (package name and module name should
       // match).
-      modules[packageName] = Module(packageName, [], packageRootUri, sources,
+      modules[packageName] = Module(
+          packageName, [], packageRootUri, sources, {},
           isPackage: true, packageBase: Uri.parse('lib/'), isShared: true);
     }
   }
@@ -205,7 +221,7 @@ Future<Module> _createSdkModule(Uri root) async {
     }
   }
   sources.sort((a, b) => a.path.compareTo(b.path));
-  return Module('sdk', [], root, sources, isSdk: true, isShared: true);
+  return Module('sdk', [], root, sources, {}, isSdk: true, isShared: true);
 }
 
 /// Trim the set of modules, and detect cycles while we are at it.

pkg/modular_test/lib/src/steps/macro_precompile_aot.dart

@@ -0,0 +1,100 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+// ignore: implementation_imports
+import 'package:macros/src/bootstrap.dart';
+// ignore: implementation_imports
+import 'package:macros/src/executor/serialization.dart';
+
+import 'util.dart';
+import '../create_package_config.dart';
+import '../io_pipeline.dart';
+import '../pipeline.dart';
+import '../suite.dart';
+
+const precompiledMacroId = DataId('macro.exe');
+
+/// Bootstraps a macro program and compiles it to an AOT executable.
+class PrecompileMacroAotStep implements IOModularStep {
+  final bool verbose;
+
+  PrecompileMacroAotStep({required this.verbose});
+
+  @override
+  List<DataId> get resultData => const [precompiledMacroId];
+
+  @override
+  bool get needsSources => true;
+
+  @override
+  List<DataId> get dependencyDataNeeded => const [];
+
+  @override
+  List<DataId> get moduleDataNeeded => const [];
+
+  @override
+  bool get onlyOnMain => false;
+
+  @override
+  bool get onlyOnSdk => false;
+
+  @override
+  bool get notOnSdk => true;
+
+  @override
+  Future<void> execute(Module module, Uri root, ModuleDataToRelativeUri toUri,
+      List<String> flags) async {
+    if (verbose) {
+      print('\nstep: precompile-macro-aot on $module');
+    }
+
+    var transitiveDependencies = computeTransitiveDependencies(module);
+    var packageConfigUri = await writePackageConfig(
+        module, transitiveDependencies, root,
+        useRealPaths: true);
+
+    var bootstrapContent = bootstrapMacroIsolate(
+            module.macroConstructors, SerializationMode.byteData)
+        // TODO: Don't do this https://github.com/dart-lang/sdk/issues/55388
+        .replaceFirst('dev-dart-app:/', '');
+    var bootstrapFile = File.fromUri(
+        root.replace(path: '${root.path}/${module.name}.macro.bootstrap.dart'));
+    await bootstrapFile.create(recursive: true);
+    await bootstrapFile.writeAsString(bootstrapContent);
+
+    var args = [
+      'compile',
+      'exe',
+      '--packages',
+      packageConfigUri.toFilePath(),
+      '--output',
+      '${toUri(module, precompiledMacroId)}',
+      ...flags.expand((String flag) => ['--enable-experiment', flag]),
+      bootstrapFile.path,
+    ];
+
+    var result = await runProcess(
+        Platform.resolvedExecutable, args, root.toFilePath(), verbose);
+    checkExitCode(result, this, module, verbose);
+  }
+
+  @override
+  void notifyCached(Module module) {
+    if (verbose) {
+      print('\ncached step: precompile-macro-aot on $module');
+    }
+  }
+
+  @override
+  bool shouldExecute(Module module) => module.macroConstructors.isNotEmpty;
+}
+
+// The value of the --precompiled-macro argument for macros coming from
+// `module`.
+String precompiledMacroArg(Module module, ModuleDataToRelativeUri toUri) {
+  var executableUri = toUri(module, precompiledMacroId);
+  return '$executableUri;${module.macroConstructors.keys.join(';')}';
+}

pkg/modular_test/lib/src/steps/util.dart

@@ -0,0 +1,35 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:modular_test/src/io_pipeline.dart';
+import 'package:modular_test/src/suite.dart';
+
+/// Checks the [exitCode] of [result], and forwards its [stdout] and [stderr] if
+/// the exit code is non-zero or [verbose] is `true`.
+///
+/// Also throws an error if the [exitCode] is non-zero.
+void checkExitCode(
+    ProcessResult result, IOModularStep step, Module module, bool verbose) {
+  if (result.exitCode != 0 || verbose) {
+    stdout.write(result.stdout);
+    stderr.write(result.stderr);
+  }
+  if (result.exitCode != 0) {
+    throw '${step.runtimeType} failed on $module:\n\n'
+        'stdout:\n${result.stdout}\n\n'
+        'stderr:\n${result.stderr}';
+  }
+}
+
+/// Runs [command] with [arguments] in [workingDirectory], and if [verbose] is
+/// `true` then it logs the full command.
+Future<ProcessResult> runProcess(String command, List<String> arguments,
+    String workingDirectory, bool verbose) {
+  if (verbose) {
+    print('command:\n$command ${arguments.join(' ')} from $workingDirectory');
+  }
+  return Process.run(command, arguments, workingDirectory: workingDirectory);
+}

pkg/modular_test/lib/src/suite.dart

@@ -65,7 +65,17 @@ class Module {
   /// will be true only for the SDK and shared packages like `package:expect`.
   bool isShared;
 
+  /// A map containing information about the macros defined in the module.
+  ///
+  /// The outer map is keyed on the source URI, and the values are the macros
+  /// defined in that source.
+  ///
+  /// The inner map is keyed on class names, and the values are lists of the
+  /// names of the public constructors for those macros.
+  Map<String, Map<String, List<String>>> macroConstructors;
+
   Module(this.name, this.dependencies, this.rootUri, this.sources,
+      this.macroConstructors,
       {this.mainSource,
       this.isPackage = false,
       this.isMain = false,

pkg/modular_test/lib/src/test_specification_parser.dart

@@ -110,7 +110,60 @@ TestSpecification parseTestSpecification(String contents) {
       _invalidSpecification("packages is not a map");
     }
   }
-  return TestSpecification(normalizedFlags, normalizedMap, normalizedPackages);
+
+  final Map<String, Map<String, Map<String, List<String>>>> normalizedMacros =
+      {};
+  var macros = spec['macros'];
+  if (macros != null) {
+    if (macros is Map) {
+      macros.forEach((module, macroConfig) {
+        if (module is! String) {
+          _invalidSpecification("macros key $module was not a String");
+        }
+        if (macroConfig is! Map) {
+          _invalidSpecification(
+              "macros[$module] value $macroConfig was not a Map");
+        }
+        var normalizedConfig = normalizedMacros[module] = {};
+        macroConfig.forEach((library, macroConstructors) {
+          if (library is! String) {
+            _invalidSpecification(
+                "macros[$module] key `$library` was not a String");
+          }
+          if (macroConstructors is! Map) {
+            _invalidSpecification(
+                "macros[$module][$library] value `$macroConstructors` was not "
+                "a Map");
+          }
+          var normalizedConstructors = normalizedConfig[library] = {};
+          macroConstructors.forEach((clazz, constructors) {
+            if (clazz is! String) {
+              _invalidSpecification(
+                  "macros[$module][$library] key `$clazz` was not a String");
+            }
+            if (constructors is! List) {
+              _invalidSpecification(
+                  "macro[$module][$library][$clazz] value `$constructors` was "
+                  "not a List");
+            }
+            var constructorNames = normalizedConstructors[clazz] = [];
+            for (var constructor in constructors) {
+              if (constructor is! String) {
+                _invalidSpecification(
+                    "macros[$module][$library][$clazz] element `$constructor` "
+                    "was not a String");
+              }
+              constructorNames.add(constructor);
+            }
+          });
+        });
+      });
+    } else {
+      _invalidSpecification("macros is not a Map");
+    }
+  }
+  return TestSpecification(
+      normalizedFlags, normalizedMap, normalizedPackages, normalizedMacros);
 }
 
 /// Data specifying details about a modular test including dependencies and
@@ -136,10 +189,18 @@ class TestSpecification {
   /// where this test specification was defined.
   final Map<String, String> packages;
 
-  TestSpecification(this.flags, this.dependencies, this.packages);
+  /// A map containing information about the macros defined in each module.
+  ///
+  /// The keys are the macro names, and the values describe the macros in each
+  /// module.
+  ///
+  /// See the `Module` class for more information about the values.
+  final Map<String, Map<String, Map<String, List<String>>>> macros;
+
+  TestSpecification(this.flags, this.dependencies, this.packages, this.macros);
 }
 
-_invalidSpecification(String message) {
+Never _invalidSpecification(String message) {
   throw InvalidSpecificationError(message);
 }
 

pkg/modular_test/pubspec.yaml

@@ -12,6 +12,7 @@ environment:
 # Use 'any' constraints here; we get our versions from the DEPS file.
 dependencies:
   args: any
+  macros: any
   package_config: any
   yaml: any
 

pkg/modular_test/test/io_pipeline_test.dart

@@ -136,6 +136,9 @@ class SourceOnlyStep implements IOModularStep {
 
   @override
   void notifyCached(Module module) {}
+
+  @override
+  bool shouldExecute(Module module) => true;
 }
 
 class ModuleDataStep implements IOModularStep {
@@ -172,6 +175,9 @@ class ModuleDataStep implements IOModularStep {
 
   @override
   void notifyCached(Module module) {}
+
+  @override
+  bool shouldExecute(Module module) => true;
 }
 
 class TwoOutputStep implements IOModularStep {
@@ -214,6 +220,9 @@ class TwoOutputStep implements IOModularStep {
 
   @override
   void notifyCached(Module module) {}
+
+  @override
+  bool shouldExecute(Module module) => true;
 }
 
 class LinkStep implements IOModularStep {
@@ -255,6 +264,9 @@ class LinkStep implements IOModularStep {
 
   @override
   void notifyCached(Module module) {}
+
+  @override
+  bool shouldExecute(Module module) => true;
 }
 
 class MainOnlyStep implements IOModularStep {
@@ -296,6 +308,9 @@ class MainOnlyStep implements IOModularStep {
 
   @override
   void notifyCached(Module module) {}
+
+  @override
+  bool shouldExecute(Module module) => true;
 }
 
 Future<String?> _readHelper(Module module, Uri root, DataId dataId,

pkg/modular_test/test/pipeline_common.dart

@@ -101,11 +101,12 @@ runPipelineTest<S extends ModularStep>(PipelineTestStrategy<S> testStrategy) {
   };
 
   var m1 = Module("a", const [], testStrategy.testRootUri,
-      [Uri.parse("a1.dart"), Uri.parse("a2.dart")],
+      [Uri.parse("a1.dart"), Uri.parse("a2.dart")], {},
       isShared: true);
   var m2 = Module("b", [m1], testStrategy.testRootUri,
-      [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")]);
-  var m3 = Module("c", [m2], testStrategy.testRootUri, [Uri.parse("c.dart")],
+      [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")], {});
+  var m3 = Module(
+      "c", [m2], testStrategy.testRootUri, [Uri.parse("c.dart")], {},
       isMain: true);
 
   var singleModuleInput = ModularTest([m1], m1, []);

pkg/modular_test/test/validate_suite_test.dart

@@ -10,23 +10,23 @@ import 'package:modular_test/src/suite.dart';
 
 main() {
   test('module test is not empty', () {
-    var m = Module("a", [], Uri.parse("app:/"), []);
+    var m = Module("a", [], Uri.parse("app:/"), [], {});
     expect(() => ModularTest([], m, []), throwsA(TypeMatcher<ArgumentError>()));
   });
 
   test('package must depend on package', () {
     var m1a = Module("a", const [], Uri.parse("app:/"),
-        [Uri.parse("a1.dart"), Uri.parse("a2.dart")],
+        [Uri.parse("a1.dart"), Uri.parse("a2.dart")], {},
         isPackage: false);
     var m1b = Module("a", const [], Uri.parse("app:/"),
-        [Uri.parse("a1.dart"), Uri.parse("a2.dart")],
+        [Uri.parse("a1.dart"), Uri.parse("a2.dart")], {},
         isPackage: true);
 
     var m2a = Module("b", [m1a], Uri.parse("app:/"),
-        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")],
+        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")], {},
         isPackage: true);
     var m2b = Module("b", [m1b], Uri.parse("app:/"),
-        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")],
+        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")], {},
         isPackage: true);
     expect(() => ModularTest([m1a, m2a], m2a, []),
         throwsA(TypeMatcher<InvalidModularTestError>()));
@@ -35,17 +35,17 @@ main() {
 
   test('shared module must depend on shared modules', () {
     var m1a = Module("a", const [], Uri.parse("app:/"),
-        [Uri.parse("a1.dart"), Uri.parse("a2.dart")],
+        [Uri.parse("a1.dart"), Uri.parse("a2.dart")], {},
         isShared: false);
     var m1b = Module("a", const [], Uri.parse("app:/"),
-        [Uri.parse("a1.dart"), Uri.parse("a2.dart")],
+        [Uri.parse("a1.dart"), Uri.parse("a2.dart")], {},
         isShared: true);
 
     var m2a = Module("b", [m1a], Uri.parse("app:/"),
-        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")],
+        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")], {},
         isShared: true);
     var m2b = Module("b", [m1b], Uri.parse("app:/"),
-        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")],
+        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")], {},
         isShared: true);
     expect(() => ModularTest([m1a, m2a], m2a, []),
         throwsA(TypeMatcher<InvalidModularTestError>()));
@@ -54,17 +54,17 @@ main() {
 
   test('sdk module must not have dependencies', () {
     var m1a = Module("a", const [], Uri.parse("app:/"),
-        [Uri.parse("a1.dart"), Uri.parse("a2.dart")],
+        [Uri.parse("a1.dart"), Uri.parse("a2.dart")], {},
         isSdk: false);
     var m1b = Module("a", const [], Uri.parse("app:/"),
-        [Uri.parse("a1.dart"), Uri.parse("a2.dart")],
+        [Uri.parse("a1.dart"), Uri.parse("a2.dart")], {},
         isSdk: true);
 
     var m2a = Module("b", [m1a], Uri.parse("app:/"),
-        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")],
+        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")], {},
         isSdk: true);
     var m2b = Module("b", [m1b], Uri.parse("app:/"),
-        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")],
+        [Uri.parse("b/b1.dart"), Uri.parse("b/b2.dart")], {},
         isSdk: true);
     expect(() => ModularTest([m1a, m2a], m2a, []),
         throwsA(TypeMatcher<InvalidModularTestError>()));
@@ -74,7 +74,7 @@ main() {
 
   test('sdk module cannot be package module', () {
     var m = Module("a", const [], Uri.parse("app:/"),
-        [Uri.parse("a1.dart"), Uri.parse("a2.dart")],
+        [Uri.parse("a1.dart"), Uri.parse("a2.dart")], {},
         isSdk: true);
     expect(ModularTest([m], m, []), isNotNull);
 

tests/modular/const_extension_type/modules.yaml

@@ -7,4 +7,4 @@
 dependencies:
   main: [def, expect]
 flags:
-  - inline-class
+  - inline-class

tests/modular/macro/add_function.dart

@@ -0,0 +1,20 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:macros/macros.dart';
+
+/// Adds a function with a specified name to a class, with no body and a void
+/// return type.
+macro class AddFunction implements ClassDeclarationsMacro {
+  /// The name of the function to add.
+  final String name;
+
+  const AddFunction(this.name);
+
+  @override
+  void buildDeclarationsForClass(
+      ClassDeclaration clazz, MemberDeclarationBuilder builder) {
+    builder.declareInType(DeclarationCode.fromString('void $name() {}'));
+  }
+}

tests/modular/macro/analysis_options.yaml

@@ -0,0 +1,3 @@
+analyzer:
+  enable-experiment:
+    - macros

tests/modular/macro/main.dart

@@ -0,0 +1,12 @@
+// Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'add_function.dart';
+
+void main() {
+  A().myFunc();
+}
+
+@AddFunction('myFunc')
+class A {}

tests/modular/macro/modules.yaml

@@ -0,0 +1,19 @@
+# Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+#
+# Tests running precompiled macros from dependent modules.
+dependencies:
+  main: [add_function]
+  add_function: [macros]
+  macros: [_macros]
+macros:
+  add_function:
+    dev-dart-app:/add_function.dart:
+      AddFunction:
+        - ""
+flags:
+  - macros
+packages:
+  macros: ../../../pkg/macros/lib
+  _macros: ../../../pkg/_macros/lib