From bc0ec85717de14677a0253814faead368ac2abe8 Mon Sep 17 00:00:00 2001
From: Jenn Magder <magder@google.com>
Date: Mon, 4 Apr 2022 17:27:18 -0700
Subject: [PATCH] Only run test harness tests once instead of every shard
 (#101218)

---
 dev/bots/test.dart           | 24 +++++++++---------------
 dev/bots/test/test_test.dart |  6 +++---
 2 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/dev/bots/test.dart b/dev/bots/test.dart
index 98e919e84ff..f58b9ebf993 100644
--- a/dev/bots/test.dart
+++ b/dev/bots/test.dart
@@ -137,7 +137,7 @@ const Map<String, List<String>> kWebTestFileKnownFailures = <String, List<String
   ],
 };
 
-const String kSmokeTestShardName = 'smoke_tests';
+const String kTestHarnessShardName = 'test_harness_tests';
 const List<String> _kAllBuildModes = <String>['debug', 'profile', 'release'];
 
 // The seed used to shuffle tests.  If not passed with
@@ -171,7 +171,6 @@ Future<void> main(List<String> args) async {
   try {
     flutterTestArgs.addAll(args);
     final Set<String> removeArgs = <String>{};
-    bool runSmokeTests = true;
     for (final String arg in args) {
       if (arg.startsWith('--local-engine=')) {
         localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length);
@@ -184,7 +183,7 @@ Future<void> main(List<String> args) async {
         removeArgs.add(arg);
       }
       if (arg == '--no-smoke-tests') {
-        runSmokeTests = false;
+        // This flag is deprecated, ignore it.
         removeArgs.add(arg);
       }
     }
@@ -192,10 +191,6 @@ Future<void> main(List<String> args) async {
     if (Platform.environment.containsKey(CIRRUS_TASK_NAME))
       print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
     print('═' * 80);
-    if (runSmokeTests) {
-      await _runSmokeTests();
-    }
-    print('═' * 80);
     await selectShard(<String, ShardRunner>{
       'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
       'build_tests': _runBuildTests,
@@ -213,7 +208,7 @@ Future<void> main(List<String> args) async {
       'web_long_running_tests': _runWebLongRunningTests,
       'flutter_plugins': _runFlutterPluginsTests,
       'skp_generator': _runSkpGeneratorTests,
-      kSmokeTestShardName: () async {}, // No-op, the smoke tests already ran. Used for testing this script.
+      kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script.
     });
   } on ExitException catch (error) {
     error.apply();
@@ -248,8 +243,8 @@ Future<void> _validateEngineHash() async {
   }
 }
 
-Future<void> _runSmokeTests() async {
-  print('${green}Running smoketests...$reset');
+Future<void> _runTestHarnessTests() async {
+  print('${green}Running test harness tests...$reset');
 
   await _validateEngineHash();
 
@@ -257,7 +252,7 @@ Future<void> _runSmokeTests() async {
   // success.
   final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
 
-  // We want to run the smoketests in parallel, because they each take some time
+  // We want to run these tests in parallel, because they each take some time
   // to run (e.g. compiling), so we don't want to run them in series, especially
   // on 20-core machines. However, we have a race condition, so for now...
   // Race condition issue: https://github.com/flutter/flutter/issues/90026
@@ -319,11 +314,9 @@ Future<void> _runSmokeTests() async {
 
   List<ShardRunner> testsToRun;
 
-  // Smoke tests are special and run first for all test shards.
-  // Run all smoke tests for other shards.
-  // Only shard smoke tests when explicitly specified.
+  // Run all tests unless sharding is explicitly specified.
   final String? shardName = Platform.environment[kShardKey];
-  if (shardName == kSmokeTestShardName) {
+  if (shardName == kTestHarnessShardName) {
     testsToRun = _selectIndexOfTotalSubshard<ShardRunner>(tests);
   } else {
     testsToRun = tests;
@@ -871,6 +864,7 @@ Future<void> _runFrameworkTests() async {
 
   Future<void> runMisc() async {
     print('${green}Running package tests$reset for directories other than packages/flutter');
+    await _runTestHarnessTests();
     await runExampleTests();
     await _dartRunTest(path.join(flutterRoot, 'dev', 'bots'));
     await _dartRunTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
diff --git a/dev/bots/test/test_test.dart b/dev/bots/test/test_test.dart
index ed194b08d90..9e3589bf60b 100644
--- a/dev/bots/test/test_test.dart
+++ b/dev/bots/test/test_test.dart
@@ -112,13 +112,13 @@ void main() {
       // When updating this test, try to pick shard numbers that ensure we're checking
       // that unequal test distributions don't miss tests.
       ProcessResult result = await runScript(
-        <String, String>{'SHARD': 'smoke_tests', 'SUBSHARD': '1_3'},
+        <String, String>{'SHARD': 'test_harness_tests', 'SUBSHARD': '1_3'},
       );
       expectExitCode(result, 0);
       expect(result.stdout, contains('Selecting subshard 1 of 3 (range 1-3 of 8)'));
 
       result = await runScript(
-        <String, String>{'SHARD': 'smoke_tests', 'SUBSHARD': '3_3'},
+        <String, String>{'SHARD': 'test_harness_tests', 'SUBSHARD': '3_3'},
       );
       expectExitCode(result, 0);
       expect(result.stdout, contains('Selecting subshard 3 of 3 (range 7-8 of 8)'));
@@ -126,7 +126,7 @@ void main() {
 
     test('exits with code 1 when SUBSHARD index greater than total', () async {
       final ProcessResult result = await runScript(
-        <String, String>{'SHARD': 'smoke_tests', 'SUBSHARD': '100_99'},
+        <String, String>{'SHARD': 'test_harness_tests', 'SUBSHARD': '100_99'},
       );
       expectExitCode(result, 1);
       expect(result.stdout, contains('Invalid subshard name'));