Let experimental features specify an example for tooling to have generic

tests.

Bug: https://github.com/dart-lang/sdk/issues/44937
Change-Id: I309e6e4299b09705f653d59c7842a05fdf7cdaa5
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/184790
Commit-Queue: Sigurd Meldgaard <sigurdm@google.com>
Reviewed-by: Lasse R.H. Nielsen <lrn@google.com>
Reviewed-by: Ben Konyi <bkonyi@google.com>

pkg/dartdev/pubspec.yaml

@@ -33,4 +33,6 @@ dependencies:
   usage: ^3.4.0
 
 dev_dependencies:
+  pub_semver: any
   test: ^1.0.0
+  yaml: any

pkg/dartdev/test/analytics_test.dart

@@ -9,6 +9,7 @@ import 'package:dartdev/dartdev.dart';
 import 'package:dartdev/src/analytics.dart';
 import 'package:test/test.dart';
 
+import 'experiment_util.dart';
 import 'utils.dart';
 
 List<Map> extractAnalytics(ProcessResult result) {
@@ -18,7 +19,7 @@ List<Map> extractAnalytics(ProcessResult result) {
       .toList();
 }
 
-void main() {
+Future<void> main() async {
   group('DisabledAnalytics', disabledAnalyticsObject);
 
   group('VM -> CLI --analytics flag smoke test:', () {
@@ -66,6 +67,7 @@ void main() {
 '''));
   });
 
+  final experiments = await experimentsWithValidation();
   group('Sending analytics', () {
     test('help', () {
       final p = project(logAnalytics: true);
@@ -226,42 +228,47 @@ void main() {
         }
       ]);
     });
-
-    test('run --enable-experiments', () {
-      final p = project(
-          mainSrc: 'void main(List<String> args) => print(args);',
-          logAnalytics: true);
-      final result = p.runSync([
-        'run',
-        '--enable-experiment=non-nullable',
-        'lib/main.dart',
-      ]);
-      expect(extractAnalytics(result), [
-        {
-          'hitType': 'screenView',
-          'message': {'viewName': 'run'}
-        },
-        {
-          'hitType': 'event',
-          'message': {
-            'category': 'dartdev',
-            'action': 'run',
-            'label': null,
-            'value': null,
-            'cd1': '0',
-            'cd2': ' non-nullable ',
+    group('run --enable-experiments', () {
+      for (final experiment in experiments) {
+        test(experiment.name, () {
+          final p = project(mainSrc: experiment.validation, logAnalytics: true);
+          {
+            for (final no in ['', 'no-']) {
+              final result = p.runSync([
+                'run',
+                '--enable-experiment=$no${experiment.name}',
+                'lib/main.dart',
+              ]);
+              expect(extractAnalytics(result), [
+                {
+                  'hitType': 'screenView',
+                  'message': {'viewName': 'run'}
+                },
+                {
+                  'hitType': 'event',
+                  'message': {
+                    'category': 'dartdev',
+                    'action': 'run',
+                    'label': null,
+                    'value': null,
+                    'cd1': '0',
+                    'cd2': ' $no${experiment.name} ',
+                  }
+                },
+                {
+                  'hitType': 'timing',
+                  'message': {
+                    'variableName': 'run',
+                    'time': isA<int>(),
+                    'category': 'commands',
+                    'label': null
+                  }
+                }
+              ]);
+            }
           }
-        },
-        {
-          'hitType': 'timing',
-          'message': {
-            'variableName': 'run',
-            'time': isA<int>(),
-            'category': 'commands',
-            'label': null
-          }
-        }
-      ]);
+        });
+      }
     });
 
     test('compile', () {

pkg/dartdev/test/commands/test_test.dart

@@ -5,15 +5,18 @@
 import 'dart:io';
 
 import 'package:path/path.dart' as path;
+import 'package:pub_semver/pub_semver.dart';
 import 'package:test/test.dart';
 
+import '../experiment_util.dart';
 import '../utils.dart';
 
-void main() {
-  group('test', defineTest, timeout: longTimeout);
+Future<void> main() async {
+  final experiments = await experimentsWithValidation();
+  group('test', () => defineTest(experiments), timeout: longTimeout);
 }
 
-void defineTest() {
+void defineTest(List<Experiment> experiments) {
   TestProject p;
 
   tearDown(() => p?.dispose());
@@ -145,29 +148,56 @@ void main() {
     expect(result.stderr, isEmpty);
   });
 
-  test('--enable-experiment', () {
-    p = project(mainSrc: 'int get foo => 1;\n');
-    p.file('test/foo_test.dart', '''
-import 'package:test/test.dart';
-
-void main() {
-  test('', () {
-    int a;
-    a = null;
-    print('a is \$a.');
-  });
-}
-''');
-
-    final result = p.runSync(
-      [
-        '--enable-experiment=non-nullable',
+  group('--enable-experiment', () {
+    ProcessResult runTestWithExperimentFlag(String flag) {
+      return p.runSync([
+        if (flag != null) flag,
         'test',
         '--no-color',
         '--reporter',
         'expanded',
-      ],
-    );
-    expect(result.exitCode, 1);
+      ]);
+    }
+
+    void expectSuccess(String flag) {
+      final result = runTestWithExperimentFlag(flag);
+      expect(result.stdout, contains('feature enabled'),
+          reason: 'stderr: ${result.stderr}');
+      expect(result.exitCode, 0,
+          reason: 'stdout: ${result.stdout} stderr: ${result.stderr}');
+    }
+
+    void expectFailure(String flag) {
+      final result = runTestWithExperimentFlag(flag);
+      expect(result.exitCode, isNot(0));
+    }
+
+    for (final experiment in experiments) {
+      test(experiment.name, () {
+        final currentSdk = Version.parse(Platform.version.split(' ').first);
+        p = project(
+            mainSrc: experiment.validation,
+            sdkConstraint: VersionConstraint.compatibleWith(currentSdk));
+        p.file('test/experiment_test.dart', '''
+import 'package:dartdev_temp/main.dart' as imported;
+import 'package:test/test.dart';
+
+void main() {
+  test('testing feature', () {
+    imported.main();
+  });
+}
+''');
+        if (experiment.enabledIn != null) {
+          // The experiment has been released - enabling it should have no effect.
+          expectSuccess(null);
+          expectSuccess('--enable-experiment=${experiment.name}');
+        } else {
+          expectFailure(null);
+          expectFailure('--enable-experiment=no-${experiment.name}');
+          expectSuccess('--enable-experiment=${experiment.name}');
+        }
+      });
+    }
   });
 }

pkg/dartdev/test/experiment_util.dart

@@ -0,0 +1,45 @@
+// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+import 'dart:isolate';
+
+import 'package:pub_semver/pub_semver.dart';
+import 'package:yaml/yaml.dart' as yaml;
+
+/// The unexpired experiments from `experimental_features.yaml` that have an
+/// associated `validation` program.
+Future<List<Experiment>> experimentsWithValidation() async {
+  final url = (await Isolate.resolvePackageUri(
+          Uri.parse('package:dartdev/dartdev.dart')))
+      .resolve('../../../tools/experimental_features.yaml');
+  final experiments =
+      yaml.loadYaml(File.fromUri(url).readAsStringSync(), sourceUrl: url);
+  return [
+    for (final e in experiments['features'].entries)
+      if (e.value['expired'] != true && e.value['validation'] != null)
+        Experiment(
+          e.key,
+          e.value['validation'],
+          tryParseVersion(e.value['enabledIn']),
+          tryParseVersion(e.value['experimentalReleaseVersion']),
+        )
+  ];
+}
+
+Version tryParseVersion(String version) =>
+    version == null ? null : Version.parse(version);
+
+class Experiment {
+  final String name;
+  final String validation;
+  final Version enabledIn;
+  final Version experimentalReleaseVersion;
+  Experiment(
+    this.name,
+    this.validation,
+    this.enabledIn,
+    this.experimentalReleaseVersion,
+  );
+}

pkg/dartdev/test/utils.dart

@@ -6,6 +6,8 @@ import 'dart:convert';
 import 'dart:io';
 
 import 'package:path/path.dart' as path;
+import 'package:pub_semver/pub_semver.dart';
+
 import 'package:test/test.dart';
 
 /// A long [Timeout] is provided for tests that start a process on
@@ -21,11 +23,13 @@ TestProject project(
         {String mainSrc,
         String analysisOptions,
         bool logAnalytics = false,
-        String name = TestProject._defaultProjectName}) =>
+        String name = TestProject._defaultProjectName,
+        VersionConstraint sdkConstraint}) =>
     TestProject(
         mainSrc: mainSrc,
         analysisOptions: analysisOptions,
-        logAnalytics: logAnalytics);
+        logAnalytics: logAnalytics,
+        sdkConstraint: sdkConstraint);
 
 class TestProject {
   static const String _defaultProjectName = 'dartdev_temp';
@@ -40,17 +44,19 @@ class TestProject {
 
   final bool logAnalytics;
 
-  TestProject({
-    String mainSrc,
-    String analysisOptions,
-    this.name = _defaultProjectName,
-    this.logAnalytics = false,
-  }) {
+  final VersionConstraint sdkConstraint;
+
+  TestProject(
+      {String mainSrc,
+      String analysisOptions,
+      this.name = _defaultProjectName,
+      this.logAnalytics = false,
+      this.sdkConstraint}) {
     dir = Directory.systemTemp.createTempSync(name);
     file('pubspec.yaml', '''
 name: $name
 environment:
-  sdk: '>=2.10.0 <3.0.0'
+  sdk: '${sdkConstraint ?? '>=2.10.0 <3.0.0'}'
 
 dev_dependencies:
   test: any

tools/experimental_features.yaml

@@ -50,6 +50,12 @@
 #    eventual removal from this file. If this field is omitted, then 'expired'
 #    is considered to be false.
 #
+# 4. validation: (optional string)
+#    If provided this should be a program that prints "feature enabled" on
+#    stdout if the feature is enabled, and throws or fails to compile otherwise.
+#    The intended use for this is to be able to run generic tests for each
+#    experiment.
+#
 # Using the above fields, experiments pass through several states:
 #
 # Disabled:
@@ -102,6 +108,13 @@ current-version: '2.13.0'
 features:
   triple-shift:
     help: "Triple-shift operator"
+    validation: |
+      class A {
+        operator>>>(int k) => 42;
+      }
+      void main() {
+        if ((A() >>> 1) == 42) print('feature enabled');
+      }
 
   variance:
     help: "Sound variance"
@@ -134,6 +147,11 @@ features:
     help: "Non Nullable by default"
     experimentalReleaseVersion: '2.10.0'
     enabledIn: '2.12.0'
+    validation: |
+      void main() {
+        int? a = null;
+        print('feature enabled');
+      }
 
   extension-methods:
     help: "Extension Methods"