[infra] Make front_end tests unapprovable using a blacklist

Before, the front_end tests were run in a step disguised as not using test.py so that the test results would not be analzyed and could not be approved.

This CL uses a blacklist instead to prevent approve_results.dart from appoving failures in these tests.

Change-Id: I9c4d1f236e3cde3143e3092100a5553caf4b1487
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/116781
Commit-Queue: Karl Klose <karlklose@google.com>
Reviewed-by: Jonas Termansen <sortie@google.com>

tools/approve_results.dart

@@ -18,6 +18,15 @@ import 'package:glob/glob.dart';
 
 import 'bots/results.dart';
 
+bool isUnapprovable(Test test) {
+  String name = test.name;
+  return name == "pkg/analyzer/test/generated/parser_fasta_test" ||
+      name == "pkg/compiler/tool/generate_kernel_test" ||
+      name == "pkg/vm/test/kernel_front_end_test" ||
+      ["pkg/kernel/test/", "pkg/front_end/", "pkg/analyzer/test/src/fasta/"]
+          .any((path) => test.name.startsWith(path));
+}
+
 /// Returns whether two decoded JSON objects are identical.
 bool isIdenticalJson(dynamic a, dynamic b) {
   if (a is Map<String, dynamic> && b is Map<String, dynamic>) {
@@ -618,8 +627,19 @@ ${parser.usage}""");
           ? options["failures-only"]
           : options["successes-only"]))
       .toList();
-  final fixedTests = selectedTests.where((test) => test.matches).toList();
-  final brokenTests = selectedTests.where((test) => !test.matches).toList();
+
+  final fixedTests = [];
+  final brokenTests = [];
+  final blackListedTests = [];
+  for (Test test in selectedTests) {
+    if (isUnapprovable(test) && !test.matches) {
+      blackListedTests.add(test);
+    } else if (test.matches) {
+      fixedTests.add(test);
+    } else {
+      brokenTests.add(test);
+    }
+  }
 
   // Find out which bots have multiple configurations.
   final configurationsForBots = <String, Set<String>>{};
@@ -720,6 +740,16 @@ ${parser.usage}""");
     print("");
   }
 
+  if (blackListedTests.isNotEmpty) {
+    print("Warning: The following failing test(s) cannot be approved:");
+    print("");
+    blackListedTests.forEach((test) => print("  ${test.name}"));
+    print("");
+    print(
+        "Try fixing or reverting the change that caused the failure(s) instead.");
+    print("");
+  }
+
   // Provide statistics on how well the bots are doing.
   void statistic(int numerator, int denominator, String what) {
     double percent = numerator / denominator * 100.0;
@@ -922,6 +952,7 @@ ${parser.usage}""");
 
     // Approve the changes in test results.
     for (final test in selectedTests) {
+      if (blackListedTests.contains(test)) continue;
       final newApprovals = newApprovalsForBuilders.putIfAbsent(
           test.bot, () => new SplayTreeMap<String, Map<String, dynamic>>());
       final approvalData =

tools/bots/test_matrix.json

@@ -546,7 +546,6 @@
         },
         {
           "name": "unit tests",
-          "script": "tools/disguised_test.py",
           "arguments": [
             "-nunittest-asserts-${mode}-${system}",
              "pkg/(kernel|front_end|fasta)"

tools/disguised_test.py

@@ -1,23 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
-# for details. All rights reserved. Use of this source code is governed by a
-# BSD-style license that can be found in the LICENSE file.
-
-# This script disguises test.py so it doesn't look like test.py to the testing
-# infrastructure, which handles test.py specially. Any testing done through
-# this script will not show up on testing dashboards and in the test results
-# database.
-
-# The front-end-* builders are currently using this script to run their unit
-# tests, which must always pass and should not be approvable through the status
-# file free workflow.
-
-import os
-import subprocess
-import sys
-
-exit(
-    subprocess.call([
-        sys.executable,
-        os.path.join(os.path.dirname(os.path.abspath(__file__)), "test.py")
-    ] + sys.argv[1:]))