Modify test runner to use the new dartanalyzer --format=json feature.

This allows the test runner to support context messages, for example
the context error messages associated with the new "why not promoted"
feature.

Bug: https://github.com/dart-lang/sdk/issues/44905
Change-Id: Ie342bde21c43641eafc5d117f328e9fde23c49bc
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/193740
Reviewed-by: Bob Nystrom <rnystrom@google.com>
Commit-Queue: Paul Berry <paulberry@google.com>

pkg/test_runner/lib/src/command_output.dart

@@ -551,18 +551,57 @@ class AnalyzerError implements Comparable<AnalyzerError> {
 class AnalysisCommandOutput extends CommandOutput with _StaticErrorOutput {
   static void parseErrors(String stderr, List<StaticError> errors,
       [List<StaticError> warnings]) {
-    for (var error in AnalyzerError.parseStderr(stderr)) {
-      var staticError = StaticError(ErrorSource.analyzer, error.errorCode,
-          line: error.line, column: error.column, length: error.length);
+    var jsonData = json.decode(stderr) as Map<String, dynamic>;
+    var version = jsonData['version'];
+    if (version != 1) {
+      DebugLogger.error('Unexpected analyzer JSON data version: $version');
+      throw UnimplementedError();
+    }
+    for (var diagnostic in jsonData['diagnostics'] as List<dynamic>) {
+      var diagnosticMap = diagnostic as Map<String, dynamic>;
+      var location = diagnosticMap['location'] as Map<String, dynamic>;
+      var file = location['file'] as String;
+      var type = diagnosticMap['type'] as String;
+      var code = (diagnosticMap['code'] as String).toUpperCase();
+      var staticError =
+          _decodeStaticError(ErrorSource.analyzer, '$type.$code', location);
+      var contextMessages = diagnosticMap['contextMessages'] as List<dynamic>;
+      for (var contextMessage in contextMessages ?? const []) {
+        var contextMessageMap = contextMessage as Map<String, dynamic>;
+        var contextLocation =
+            contextMessageMap['location'] as Map<String, dynamic>;
+        if (contextLocation['file'] == file) {
+          staticError.contextMessages.add(_decodeStaticError(
+              ErrorSource.context,
+              contextMessageMap['message'] as String,
+              contextLocation));
+        } else {
+          DebugLogger.warning(
+              "Context messages in other files not currently supported.");
+        }
+      }
 
-      if (error.severity == 'ERROR') {
+      var severity = diagnosticMap['severity'] as String;
+      if (severity == 'ERROR') {
         errors.add(staticError);
-      } else if (error.severity == 'WARNING') {
+      } else if (severity == 'WARNING') {
         warnings?.add(staticError);
       }
     }
   }
 
+  static StaticError _decodeStaticError(
+      ErrorSource errorSource, String message, Map<String, dynamic> location) {
+    var locationRange = location['range'] as Map<String, dynamic>;
+    var locationRangeStart = locationRange['start'] as Map<String, dynamic>;
+    var locationRangeEnd = locationRange['end'] as Map<String, dynamic>;
+    return StaticError(errorSource, message,
+        line: locationRangeStart['line'] as int,
+        column: locationRangeStart['column'] as int,
+        length: (locationRangeEnd['offset'] as int) -
+            (locationRangeStart['offset'] as int));
+  }
+
   AnalysisCommandOutput(
       Command command,
       int exitCode,
@@ -584,6 +623,23 @@ class AnalysisCommandOutput extends CommandOutput with _StaticErrorOutput {
     if (hasTimedOut) return Expectation.timeout;
     if (hasNonUtf8) return Expectation.nonUtf8Error;
 
+    List<StaticError> errors;
+    try {
+      errors = this.errors;
+    } catch (_) {
+      // This can happen in at least two scenarios:
+      // - The analyzer output was too long so it got truncated (so the
+      //   resulting output was ill-formed).  See also
+      //   https://github.com/dart-lang/sdk/issues/44493.
+      // - The analyzer did not find a file to analyze at the specified
+      //   location, so it generated the output "No dart files found at
+      //   <path>.dart" (which is not valid JSON).  See
+      //   https://github.com/dart-lang/sdk/issues/45556.
+      // TODO(paulberry,rnystrom): remove this logic once the two above bugs are
+      // fixed.
+      return Expectation.crash;
+    }
+
     // If it's a static error test, validate the exact errors.
     if (testCase.testFile.isStaticErrorTest) {
       return _validateExpectedErrors(testCase);
@@ -628,6 +684,23 @@ class AnalysisCommandOutput extends CommandOutput with _StaticErrorOutput {
     if (hasTimedOut) return Expectation.timeout;
     if (hasNonUtf8) return Expectation.nonUtf8Error;
 
+    List<StaticError> errors;
+    try {
+      errors = this.errors;
+    } catch (_) {
+      // This can happen in at least two scenarios:
+      // - The analyzer output was too long so it got truncated (so the
+      //   resulting output was ill-formed).  See also
+      //   https://github.com/dart-lang/sdk/issues/44493.
+      // - The analyzer did not find a file to analyze at the specified
+      //   location, so it generated the output "No dart files found at
+      //   <path>.dart" (which is not valid JSON).  See
+      //   https://github.com/dart-lang/sdk/issues/45556.
+      // TODO(paulberry,rnystrom): remove this logic once the two above bugs are
+      // fixed.
+      return Expectation.crash;
+    }
+
     // If it's a static error test, validate the exact errors.
     if (testCase.testFile.isStaticErrorTest) {
       return _validateExpectedErrors(testCase);
@@ -1363,7 +1436,14 @@ mixin _StaticErrorOutput on CommandOutput {
     // Handle static error test output specially. We don't want to show the raw
     // stdout if we can give the user the parsed expectations instead.
     if (testCase.testFile.isStaticErrorTest && !hasCrashed && !hasTimedOut) {
-      _validateExpectedErrors(testCase, output);
+      try {
+        _validateExpectedErrors(testCase, output);
+      } catch (_) {
+        // In the event of a crash trying to compute errors, go ahead and give
+        // the raw output.
+        super.describe(testCase, progress, output);
+        return;
+      }
     }
 
     // Don't show the "raw" output unless something strange happened or the

pkg/test_runner/lib/src/test_suite.dart

@@ -980,7 +980,7 @@ class StandardTestSuite extends TestSuite {
     }
     args.addAll(additionalOptions(testFile.path));
     if (configuration.compiler == Compiler.dart2analyzer) {
-      args.add('--format=machine');
+      args.add('--format=json');
       args.add('--no-hints');
     }
 

pkg/test_runner/tool/update_static_error_tests.dart

@@ -230,7 +230,7 @@ Future<List<StaticError>> runAnalyzer(String path, List<String> options) async {
   // mode.
   var result = await Process.run(_analyzerPath, [
     ...options,
-    "--format=machine",
+    "--format=json",
     path,
   ]);