[macros] Add macro build tests.

These are a new type of test that runs with the SDK under test against what
looks like an external package; they start by running "pub get" then run SDK
build commands to build the package, and check that macros applied correctly.

Add tests for various builds that already pass, plus one that doesn't:
cfe_sdk_cli_test fails because it was switched to run from an AOT snapshot
and that doesn't support macros yet.

Change-Id: Ic801cb61bd414d4876566452e01dd8c8203e9013
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/353100
Reviewed-by: Jake Macdonald <jakemac@google.com>
Reviewed-by: William Hesse <whesse@google.com>
Commit-Queue: Morgan :) <davidmorgan@google.com>

pkg/test_runner/lib/src/test_configurations.dart

@@ -37,6 +37,7 @@ final testSuiteDirectories = [
   Path('tests/dartdevc'),
   Path('tests/ffi'),
   Path('tests/language'),
+  Path('tests/macro_build'),
   Path('tests/lib'),
   Path('tests/standalone'),
   Path('tests/web'),

tests/macro_build/README.md

@@ -0,0 +1,5 @@
+Each test in this folder tests that macros build via a particular SDK
+entrypoint, for example via `dart run`.
+
+In the test names, `lt` stands for "language test", meaning that the build
+matches the command(s) used by some type of language test.ss

tests/macro_build/analyzer_lt_test.dart

@@ -0,0 +1,19 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'tester/tester.dart';
+
+void main() {
+  testMacroBuild([
+    r'$DART pub get',
+    r'$DART '
+        r'$DART_SDK/out/ReleaseX64/gen/dartanalyzer.dart.snapshot '
+        '-Dtest_runner.configuration=analyzer-asserts-linux '
+        '--enable-experiment=macros '
+        '--ignore-unrecognized-flags '
+        '--packages=.dart_tool/package_config.json '
+        '--format=json test/main.dart ',
+    // Analysis passed; there isn't any other output to verify.
+  ]);
+}

tests/macro_build/cfe_lt_test.dart

@@ -0,0 +1,23 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'tester/tester.dart';
+
+void main() {
+  testMacroBuild([
+    r'$DART pub get',
+    r'$DART '
+        r'$DART_SDK/pkg/front_end/tool/_fasta/compile.dart '
+        '--verify '
+        '--skip-platform-verification -o out.dill '
+        '--platform '
+        r'$DART_SDK/out/ReleaseX64/vm_platform_strong.dill '
+        '-Dtest_runner.configuration=cfe-strong-linux '
+        '--enable-experiment=macros '
+        '--nnbd-strong '
+        '--packages=.dart_tool/package_config.json '
+        'test/main.dart',
+    r'$DART out.dill',
+  ]);
+}

tests/macro_build/cfe_sdk_cli_test.dart

@@ -0,0 +1,16 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'tester/tester.dart';
+
+void main() {
+  testMacroBuild([
+    r'$DART pub get',
+    r'$DART compile kernel '
+        '--enable-experiment=macros '
+        '-o out.kernel '
+        'test/main.dart',
+    r'$DART run out.kernel',
+  ]);
+}

tests/macro_build/jit_lt_test.dart

@@ -0,0 +1,18 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'tester/tester.dart';
+
+void main() {
+  testMacroBuild([
+    r'$DART pub get',
+    r'$DART '
+        '--sound-null-safety '
+        '-Dtest_runner.configuration=vm-linux-release-x64 '
+        '--enable-experiment=macros '
+        '--ignore-unrecognized-flags '
+        '--packages=.dart_tool/package_config.json '
+        'test/main.dart',
+  ]);
+}

tests/macro_build/jit_sdk_cli_compile_test.dart

@@ -0,0 +1,16 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'tester/tester.dart';
+
+void main() {
+  testMacroBuild([
+    r'$DART pub get',
+    r'$DART compile jit-snapshot '
+        '--enable-experiment=macros '
+        '-o out.jit '
+        'test/main.dart',
+    r'$DART run out.jit',
+  ]);
+}

tests/macro_build/jit_sdk_cli_run_test.dart

@@ -0,0 +1,15 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'tester/tester.dart';
+
+void main() {
+  testMacroBuild([
+    r'$DART pub get',
+    r'$DART '
+        '--enable-experiment=macros '
+        'run '
+        'test/main.dart',
+  ]);
+}

tests/macro_build/macro_build.status

@@ -0,0 +1,3 @@
+# Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.

tests/macro_build/package_under_test/lib/declare_x.dart

@@ -0,0 +1,16 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:_fe_analyzer_shared/src/macros/api.dart';
+
+/// Macro that adds a `String` getter called `x` that return `OK`.
+macro class DeclareX implements ClassDeclarationsMacro  {
+  const DeclareX();
+
+  @override
+  Future<void> buildDeclarationsForClass(
+      ClassDeclaration clazz, MemberDeclarationBuilder builder) async {
+    builder.declareInType(DeclarationCode.fromString('String get x => "OK";'));
+  }
+}

tests/macro_build/package_under_test/pubspec.yaml

@@ -0,0 +1,18 @@
+name: package_under_test
+version: 0.0.1
+description: Package under test for macros build tests.
+publish_to: none
+
+environment:
+  sdk: '>=3.3.0 <4.0.0'
+
+dependencies:
+  _fe_analyzer_shared: any
+  analyzer: any
+  test: any
+
+dependency_overrides:
+  _fe_analyzer_shared:
+    path: ../../../pkg/_fe_analyzer_shared
+  analyzer:
+    path: ../../../pkg/analyzer

tests/macro_build/package_under_test/test/main.dart

@@ -0,0 +1,19 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:package_under_test/declare_x.dart';
+import 'package:test/test.dart';
+
+@DeclareX()
+class ClassWithMacroApplied {}
+
+/// Checks that the macro applied correctly.
+///
+/// Not named `*_test.dart` because the test runner would pick it up and run
+/// it, when what we want is the outer build test to run it.
+void main() {
+  test('macro was applied correctly', () {
+    expect(ClassWithMacroApplied().x, 'OK');
+  });
+}

tests/macro_build/tester/tester.dart

@@ -0,0 +1,113 @@
+// Copyright (c) 2024, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:expect/expect.dart';
+import 'package:path/path.dart' as p;
+import 'package:test/test.dart';
+
+/// Tests a macro build specified by [commands].
+///
+/// The commands are launched with current directory set to a temp folder with
+/// a fresh copy of the package `package_under_test`.
+///
+/// The commands should build and run `bin/main.dart`. It is a test that will
+/// string `OK\n` showing that the macro output ran.
+///
+/// In commands, the string `$DART` is replaced to refer to the `dart` command
+/// in the Dart SDK under test; and `$DART_SDK` to the root of the built Dart
+/// SDK under test.
+///
+/// The test passes if all commands return exit code 0.
+Future<void> testMacroBuild(List<String> commands) async {
+  var temp = Directory.systemTemp.createTempSync('macro_build_test');
+  var sourceDirectory =
+      Directory.current.path + '/tests/macro_build/package_under_test';
+  var workingDirectory = '${temp.path}/package_under_test';
+  await _copyPath(sourceDirectory, workingDirectory);
+
+  var dartSdkPath = Directory.current.path;
+  final dartPath = Platform.resolvedExecutable;
+
+  // TODO(davidmorgan): run on more platforms.
+  final configuration = String.fromEnvironment('test_runner.configuration');
+  if (configuration.isEmpty) {
+    print(r'''
+Hint: this test is an e2e test of SDK tools, consider running using the test
+runner to ensure they are built and not stale:
+
+./tools/test.py -v -nunittest-asserts-release-linux-x64 \
+    --build 'tests/macro_build/*'
+''');
+  } else if (configuration != 'unittest-asserts-release-linux-x64') {
+    print('Skipping test, not yet supported on '
+        '-Dtest_runner.configuration=$configuration, '
+        'use unittest-asserts-release-linux-x64.');
+    return;
+  }
+
+  _fixPubspec(
+      pubspecPath: '$workingDirectory/pubspec.yaml', dartSdkPath: dartSdkPath);
+
+  var failed = false;
+  var timedOut = false;
+  for (var command in commands) {
+    final commandParts = command
+        .replaceAll(r'$DART_SDK', dartSdkPath)
+        .replaceAll(r'$DART', dartPath)
+        .split(' ');
+    print('Running: ${commandParts.join(' ')}');
+    final process = await Process.start(
+        commandParts.first, commandParts.skip(1).toList(),
+        workingDirectory: workingDirectory);
+    try {
+      final result = await process.exitCode.timeout(Duration(seconds: 30));
+      if (result != 0) {
+        failed = true;
+      }
+    } on TimeoutException catch (_) {
+      timedOut = true;
+      process.kill();
+    }
+
+    final stdout =
+        (await process.stdout.transform(utf8.decoder).toList()).join('');
+    final stderr =
+        (await process.stderr.transform(utf8.decoder).toList()).join('');
+    print('--- stdout ---\n$stdout--- stderr ---\n$stderr---\n');
+
+    if (timedOut || failed) break;
+  }
+
+  if (failed) {
+    Expect.fail('Command exited with non-zero exit code.');
+  }
+  if (timedOut) {
+    Expect.fail('Command ran for more than 30s.');
+  }
+}
+
+Future<void> _copyPath(String from, String to) async {
+  await Directory(to).create(recursive: true);
+  await for (final file in Directory(from).list(recursive: true)) {
+    final copyTo = p.join(to, p.relative(file.path, from: from));
+    if (file is Directory) {
+      await Directory(copyTo).create(recursive: true);
+    } else if (file is File) {
+      await File(file.path).copy(copyTo);
+    }
+  }
+}
+
+/// Fixes relative paths in the pubspec at [pubspecPath] to refer to the
+/// Dart SDK path [dartSdkPath].
+void _fixPubspec({required String pubspecPath, required String dartSdkPath}) {
+  print('Updated $pubspecPath to point to SDK under $dartSdkPath.');
+  var file = File(pubspecPath);
+  file.writeAsStringSync(
+      file.readAsStringSync().replaceAll('../../..', dartSdkPath));
+}

tools/bots/test_matrix.json

@@ -205,6 +205,7 @@
       "tests/language/",
       "tests/lib/",
       "tests/light_unittest.dart",
+      "tests/macro_build/",
       "tests/search/",
       "tests/standalone/",
       "tests/ffi/",
@@ -2932,6 +2933,7 @@
           "script": "tools/build.py",
           "arguments": [
             "create_sdk",
+            "dartanalyzer",
             "ddc_stable_test"
           ]
         },
@@ -3016,10 +3018,11 @@
           ]
         },
         {
-          "name": "package unit tests",
+          "name": "package unit tests and macro build tests",
           "arguments": [
             "-nunittest-asserts-${mode}-${system}-${arch}",
-            "pkg/pkg/(?!(analyzer*|analysis_server|compiler|dev_compiler|js_runtime|front_end|kernel|frontend_server|nnbd_migration|dartdev/test/native_assets|vm_service)/)"
+            "pkg/pkg/(?!(analyzer*|analysis_server|compiler|dev_compiler|js_runtime|front_end|kernel|frontend_server|nnbd_migration|dartdev/test/native_assets|vm_service)/)",
+            "macro_build"
           ],
           "shards": 3,
           "fileset": "vm"