[cfe] Support error testing in annotated tests

Change-Id: I545ded4d82147f22efc80f4ed8ee833cd75a9be7
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/109963
Reviewed-by: Dan Rubel <danrubel@google.com>
Commit-Queue: Johnni Winther <johnniwinther@google.com>

pkg/front_end/lib/src/testing/features.dart

@@ -7,6 +7,7 @@ import 'id_testing.dart';
 /// Utility class for annotated testing representing a set of features.
 class Features {
   Map<String, Object> _features = {};
+  Set<String> _unsorted = new Set<String>();
 
   /// Mark the feature [key] as existing. If [value] is provided, the feature
   /// [key] is set to have this value.
@@ -22,6 +23,12 @@ class Features {
     }
   }
 
+  /// Marks list values of [key] as unsorted. This prevents the [getText]
+  /// representation from automatically sorting the values.
+  void markAsUnsorted(String key) {
+    _unsorted.add(key);
+  }
+
   /// Returns `true` if feature [key] exists.
   bool containsKey(String key) {
     return _features.containsKey(key);
@@ -63,7 +70,11 @@ class Features {
         }
         sb.write(name);
         if (value is List<String>) {
-          value = '[${(value..sort()).join(',')}]';
+          if (_unsorted.contains(name)) {
+            value = '[${value.join(',')}]';
+          } else {
+            value = '[${(value..sort()).join(',')}]';
+          }
         }
         if (value != '') {
           sb.write('=');

pkg/front_end/lib/src/testing/id_testing_helper.dart

@@ -3,14 +3,19 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:front_end/src/api_prototype/compiler_options.dart'
-    show CompilerOptions;
+    show CompilerOptions, DiagnosticMessage;
 import 'package:front_end/src/api_prototype/experimental_flags.dart'
     show ExperimentalFlag;
-import 'package:front_end/src/testing/id_extractor.dart' show DataExtractor;
+import 'package:front_end/src/api_prototype/terminal_color_support.dart'
+    show printDiagnosticMessage;
+import 'package:front_end/src/fasta/messages.dart' show FormattedMessage;
+import 'package:front_end/src/fasta/severity.dart' show Severity;
 import 'package:kernel/ast.dart';
 import '../kernel_generator_impl.dart' show CompilerResult;
 import 'compiler_common.dart' show compileScript, toTestUri;
-import 'id.dart' show ActualData, ClassId, Id, IdValue, MemberId, NodeId;
+import 'id.dart'
+    show ActualData, ClassId, Id, IdKind, IdValue, MemberId, NodeId;
+import 'id_extractor.dart' show DataExtractor;
 import 'id_testing.dart'
     show
         CompiledData,
@@ -74,6 +79,18 @@ abstract class DataComputer<T> {
       Map<Id, ActualData<T>> actualMap,
       {bool verbose}) {}
 
+  /// Returns `true` if this data computer supports tests with compile-time
+  /// errors.
+  ///
+  /// Unsuccessful compilation might leave the compiler in an inconsistent
+  /// state, so this testing feature is opt-in.
+  bool get supportsErrors => false;
+
+  /// Returns data corresponding to [error].
+  T computeErrorData(
+          CompilerResult compiler, Id id, List<FormattedMessage> errors) =>
+      null;
+
   /// Returns the [DataInterpreter] used to check the actual data with the
   /// expected data.
   DataInterpreter<T> get dataValidator;
@@ -201,16 +218,54 @@ Future<bool> runTestForConfig<T>(
       testData.expectedMaps[config.marker];
   Iterable<Id> globalIds = memberAnnotations.globalData.keys;
   CompilerOptions options = new CompilerOptions();
+  List<FormattedMessage> errors = [];
+  options.onDiagnostic = (DiagnosticMessage message) {
+    if (message is FormattedMessage && message.severity == Severity.error) {
+      errors.add(message);
+    }
+    printDiagnosticMessage(message, print);
+  };
   options.debugDump = printCode;
   options.experimentalFlags.addAll(config.experimentalFlags);
   CompilerResult compilerResult = await compileScript(
       testData.memorySourceFiles,
       options: options,
       retainDataForTesting: true);
+
   Component component = compilerResult.component;
   Map<Uri, Map<Id, ActualData<T>>> actualMaps = <Uri, Map<Id, ActualData<T>>>{};
   Map<Id, ActualData<T>> globalData = <Id, ActualData<T>>{};
 
+  Map<Id, ActualData<T>> actualMapForUri(Uri uri) {
+    return actualMaps.putIfAbsent(uri, () => <Id, ActualData<T>>{});
+  }
+
+  if (errors.isNotEmpty) {
+    if (!dataComputer.supportsErrors) {
+      onFailure("Compilation with compile-time errors not supported for this "
+          "testing setup.");
+    }
+
+    Map<Uri, Map<int, List<FormattedMessage>>> errorMap = {};
+    for (FormattedMessage error in errors) {
+      Map<int, List<FormattedMessage>> map =
+          errorMap.putIfAbsent(error.uri, () => {});
+      List<FormattedMessage> list = map.putIfAbsent(error.charOffset, () => []);
+      list.add(error);
+    }
+
+    errorMap.forEach((Uri uri, Map<int, List<FormattedMessage>> map) {
+      map.forEach((int offset, List<DiagnosticMessage> list) {
+        NodeId id = new NodeId(offset, IdKind.error);
+        T data = dataComputer.computeErrorData(compilerResult, id, list);
+        if (data != null) {
+          Map<Id, ActualData<T>> actualMap = actualMapForUri(uri);
+          actualMap[id] = new ActualData<T>(id, data, uri, offset, list);
+        }
+      });
+    });
+  }
+
   Map<Id, ActualData<T>> actualMapFor(TreeNode node) {
     Uri uri = node is Library ? node.fileUri : node.location.file;
     return actualMaps.putIfAbsent(uri, () => <Id, ActualData<T>>{});

pkg/front_end/test/extensions/extensions_test.dart

@@ -141,12 +141,14 @@ class ExtensionsDataExtractor extends CfeDataExtractor<Features> {
         for (FormalParameterBuilder parameter in memberBuilder.formals) {
           features.addElement(Tags.builderParameters, parameter.name);
         }
+        features.markAsUnsorted(Tags.builderParameters);
       }
       if (memberBuilder.typeVariables != null) {
         for (TypeVariableBuilder typeVariable in memberBuilder.typeVariables) {
           features.addElement(Tags.builderTypeParameters,
               typeVariableBuilderToText(typeVariable));
         }
+        features.markAsUnsorted(Tags.builderTypeParameters);
       }
     }
     features[Tags.memberName] = getMemberName(member);
@@ -155,10 +157,12 @@ class ExtensionsDataExtractor extends CfeDataExtractor<Features> {
           in member.function.positionalParameters) {
         features.addElement(Tags.memberParameters, parameter.name);
       }
+      features.markAsUnsorted(Tags.memberParameters);
       for (TypeParameter typeParameter in member.function.typeParameters) {
         features.addElement(
             Tags.memberTypeParameters, typeParameterToText(typeParameter));
       }
+      features.markAsUnsorted(Tags.memberTypeParameters);
     }
     return features;
   }

pkg/front_end/test/id_testing/data/errors.dart

@@ -0,0 +1,11 @@
+// Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/*cfe.library: file=main.dart*/
+
+/*cfe.member: main:main*/
+main() {
+  // ignore: undefined_function
+  /*error: Method not found: 'foo'.*/ foo();
+}

pkg/front_end/test/id_testing/id_testing_test.dart

@@ -3,6 +3,7 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:io' show Directory, Platform;
+import 'package:front_end/src/fasta/messages.dart' show FormattedMessage;
 import 'package:front_end/src/testing/id.dart' show ActualData, Id;
 import 'package:front_end/src/testing/id_testing.dart'
     show DataInterpreter, StringDataInterpreter, runTests;
@@ -60,6 +61,14 @@ class IdTestingDataComputer extends DataComputer<String> {
         .computeForLibrary(library);
   }
 
+  @override
+  bool get supportsErrors => true;
+
+  String computeErrorData(
+      CompilerResult compiler, Id id, List<FormattedMessage> errors) {
+    return errors.map((m) => m.message).join(',');
+  }
+
   @override
   DataInterpreter<String> get dataValidator => const StringDataInterpreter();
 }

tests/compiler/dart2js/equivalence/id_equivalence_helper.dart

@@ -158,7 +158,7 @@ Future<CompiledData<T>> computeData<T>(Uri entryPoint,
     if (skipFailedCompilations) return null;
     Expect.isTrue(
         dataComputer.supportsErrors,
-        "Compilation with compile-time error not supported for this "
+        "Compilation with compile-time errors not supported for this "
         "testing setup.");
   }
   if (printCode) {
@@ -175,10 +175,6 @@ Future<CompiledData<T>> computeData<T>(Uri entryPoint,
     return actualMaps.putIfAbsent(uri, () => <Id, ActualData<T>>{});
   }
 
-  dynamic closedWorld = testFrontend
-      ? compiler.resolutionWorldBuilder.closedWorldForTesting
-      : compiler.backendClosedWorldForTesting;
-  ElementEnvironment elementEnvironment = closedWorld?.elementEnvironment;
   Map<Uri, Map<int, List<CollectedMessage>>> errors = {};
   for (CollectedMessage error in diagnosticCollector.errors) {
     Map<int, List<CollectedMessage>> map =
@@ -200,9 +196,13 @@ Future<CompiledData<T>> computeData<T>(Uri entryPoint,
 
   if (!result.isSuccess) {
     return new Dart2jsCompiledData<T>(
-        compiler, elementEnvironment, entryPoint, actualMaps, globalData);
+        compiler, null, entryPoint, actualMaps, globalData);
   }
 
+  dynamic closedWorld = testFrontend
+      ? compiler.resolutionWorldBuilder.closedWorldForTesting
+      : compiler.backendClosedWorldForTesting;
+  ElementEnvironment elementEnvironment = closedWorld?.elementEnvironment;
   CommonElements commonElements = closedWorld.commonElements;
 
   Map<Id, ActualData<T>> actualMapFor(Entity entity) {

tests/compiler/dart2js/equivalence/id_testing_test.dart

@@ -64,6 +64,15 @@ class IdTestingDataComputer extends DataComputer<String> {
         .computeForLibrary(node);
   }
 
+  @override
+  bool get supportsErrors => true;
+
+  @override
+  String computeErrorData(
+      Compiler compiler, Id id, List<CollectedMessage> errors) {
+    return errors.map((c) => c.message.message).join(',');
+  }
+
   @override
   bool get testFrontend => true;