[presubmit] Add presubmit check for valid test matrix

This change adds a rule to the SDK PRESUBMIT.py to validate that the
test matrix used by the CI/CQ builders does not contain errors.

Change-Id: Ie967e71dda76677f4db84c1e9e613d702b57069f
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/147547
Commit-Queue: Karl Klose <karlklose@google.com>
Reviewed-by: Alexander Thomas <athom@google.com>

PRESUBMIT.py

@@ -337,6 +337,36 @@ def _CheckClangTidy(input_api, output_api):
     ]
 
 
+def _CheckTestMatrixValid(input_api, output_api):
+    """Run script to check that the test matrix has no errors."""
+
+    def test_matrix_filter(affected_file):
+        """Only run test if either the test matrix or the code that
+           validates it was modified."""
+        path = affected_file.LocalPath()
+        return (path == 'tools/bots/test_matrix.json' or
+                path == 'tools/validate_test_matrix.dart' or
+                path.startswith('pkg/smith/'))
+
+    if len(
+            input_api.AffectedFiles(
+                include_deletes=False, file_filter=test_matrix_filter)) == 0:
+        return []
+
+    command = [
+        'tools/sdks/dart-sdk/bin/dart',
+        'tools/validate_test_matrix.dart',
+    ]
+    stdout = input_api.subprocess.check_output(command).strip()
+    if not stdout:
+        return []
+    else:
+        return [
+            output_api.PresubmitError(
+                'The test matrix is not valid:', long_text=stdout)
+        ]
+
+
 def _CommonChecks(input_api, output_api):
     results = []
     results.extend(_CheckNnbdSdkSync(input_api, output_api))
@@ -347,6 +377,7 @@ def _CommonChecks(input_api, output_api):
     results.extend(_CheckStatusFiles(input_api, output_api))
     results.extend(_CheckLayering(input_api, output_api))
     results.extend(_CheckClangTidy(input_api, output_api))
+    results.extend(_CheckTestMatrixValid(input_api, output_api))
     results.extend(
         input_api.canned_checks.CheckPatchFormatted(input_api, output_api))
     return results

tools/validate_test_matrix.dart

@@ -0,0 +1,26 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// Test that the test matrix in the SDK can be parsed correctly.
+
+import 'dart:convert' show jsonDecode;
+import 'dart:io' show File, Platform;
+import 'package:smith/smith.dart' show TestMatrix;
+
+main() {
+  var path = Platform.script.resolve("bots/test_matrix.json").toFilePath();
+  var json;
+  try {
+    json = jsonDecode(File(path).readAsStringSync());
+  } catch (e) {
+    print("The test matrix at $path is not valid JSON!\n\n$e");
+    return;
+  }
+  try {
+    TestMatrix.fromJson(json);
+  } catch (e) {
+    print("The test matrix at $path is invalid!\n\n$e");
+    return;
+  }
+}