revise scanner api so that error tokens are always prepended

This CL revises the scanner API so that tokens are always prepended
to the beginning of the token stream. This allows the parser handling
of error tokens to be simplified. Any clients using the scanner directly
rather than through the scanner API should call scannerRecovery
if the scanner has detected errors in the content.

Change-Id: I32510da10205bd964f80898a238489d1508733e6
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/102680
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Dan Rubel <danrubel@google.com>

pkg/front_end/lib/src/fasta/scanner.dart

@@ -15,7 +15,7 @@ import 'scanner/string_scanner.dart' show StringScanner;
 
 import 'scanner/utf8_bytes_scanner.dart' show Utf8BytesScanner;
 
-import 'scanner/recover.dart' show defaultRecoveryStrategy;
+import 'scanner/recover.dart' show scannerRecovery;
 
 export 'scanner/abstract_scanner.dart'
     show LanguageVersionChanged, ScannerConfiguration;
@@ -68,12 +68,10 @@ class ScannerResult {
 }
 
 /// Scan/tokenize the given UTF8 [bytes].
-/// If [recover] is null, then the [defaultRecoveryStrategy] is used.
 ScannerResult scan(List<int> bytes,
     {ScannerConfiguration configuration,
     bool includeComments: false,
-    LanguageVersionChanged languageVersionChanged,
-    Recover recover}) {
+    LanguageVersionChanged languageVersionChanged}) {
   if (bytes.last != 0) {
     throw new ArgumentError("[bytes]: the last byte must be null.");
   }
@@ -81,31 +79,28 @@ ScannerResult scan(List<int> bytes,
       configuration: configuration,
       includeComments: includeComments,
       languageVersionChanged: languageVersionChanged);
-  return _tokenizeAndRecover(scanner, recover, bytes: bytes);
+  return _tokenizeAndRecover(scanner, bytes: bytes);
 }
 
 /// Scan/tokenize the given [source].
-/// If [recover] is null, then the [defaultRecoveryStrategy] is used.
 ScannerResult scanString(String source,
     {ScannerConfiguration configuration,
     bool includeComments: false,
-    LanguageVersionChanged languageVersionChanged,
-    Recover recover}) {
+    LanguageVersionChanged languageVersionChanged}) {
   assert(source != null, 'source must not be null');
   StringScanner scanner = new StringScanner(source,
       configuration: configuration,
       includeComments: includeComments,
       languageVersionChanged: languageVersionChanged);
-  return _tokenizeAndRecover(scanner, recover, source: source);
+  return _tokenizeAndRecover(scanner, source: source);
 }
 
-ScannerResult _tokenizeAndRecover(Scanner scanner, Recover recover,
+ScannerResult _tokenizeAndRecover(Scanner scanner,
     {List<int> bytes, String source}) {
   Token tokens = scanner.tokenize();
   if (scanner.hasErrors) {
     if (bytes == null) bytes = utf8.encode(source);
-    recover ??= defaultRecoveryStrategy;
-    tokens = recover(bytes, tokens, scanner.lineStarts);
+    tokens = scannerRecovery(bytes, tokens, scanner.lineStarts);
   }
   return new ScannerResult(tokens, scanner.lineStarts, scanner.hasErrors);
 }

pkg/front_end/lib/src/fasta/scanner/recover.dart

@@ -30,8 +30,7 @@ import 'error_token.dart' show NonAsciiIdentifierToken, ErrorToken;
 /// [bytes]. [lineStarts] are the beginning character offsets of lines, and
 /// must be updated if recovery is performed rewriting the original source
 /// code.
-Token defaultRecoveryStrategy(
-    List<int> bytes, Token tokens, List<int> lineStarts) {
+Token scannerRecovery(List<int> bytes, Token tokens, List<int> lineStarts) {
   // See [Parser.reportErrorToken](../parser/src/parser.dart) for how
   // it currently handles lexical errors. In addition, notice how the parser
   // calls [handleInvalidExpression], [handleInvalidFunctionBody], and

pkg/front_end/test/fasta/parser/type_info_test.dart

@@ -30,12 +30,10 @@ main() {
   });
 }
 
-ScannerResult scanString(String source,
-        {bool includeComments: false, Recover recover}) =>
+ScannerResult scanString(String source, {bool includeComments: false}) =>
     scanner.scanString(source,
         configuration: const ScannerConfiguration(enableTripleShift: true),
-        includeComments: includeComments,
-        recover: recover);
+        includeComments: includeComments);
 
 @reflectiveTest
 class NoTypeInfoTest {

pkg/front_end/test/scanner_replacement_test.dart

@@ -2,10 +2,6 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:convert' show utf8;
-
-import 'package:front_end/src/fasta/scanner/recover.dart'
-    show defaultRecoveryStrategy;
 import 'package:front_end/src/fasta/scanner.dart' as fasta;
 import 'package:front_end/src/fasta/scanner/token.dart' as fasta;
 import 'package:front_end/src/fasta/scanner/error_token.dart' as fasta;
@@ -37,21 +33,12 @@ class ScannerTest_Replacement extends ScannerTestBase {
     // pkg/analyzer/lib/src/dart/scanner/scanner.dart
     // to simulate replacing the analyzer scanner
 
-    fasta.ScannerResult result = fasta.scanString(source, includeComments: true,
-        recover: ((List<int> bytes, fasta.Token tokens, List<int> lineStarts) {
-      // perform recovery as a separate step
-      // so that the token stream can be validated before and after recovery
-      return tokens;
-    }));
+    fasta.ScannerResult result =
+        fasta.scanString(source, includeComments: true);
 
     fasta.Token tokens = result.tokens;
-    assertValidTokenStream(tokens);
+    assertValidTokenStream(tokens, errorsFirst: true);
     assertValidBeginTokens(tokens);
-    if (result.hasErrors) {
-      List<int> bytes = utf8.encode(source);
-      tokens = defaultRecoveryStrategy(bytes, tokens, result.lineStarts);
-      assertValidTokenStream(tokens, errorsFirst: true);
-    }
 
     // fasta pretends there is an additional line at EOF
     result.lineStarts.removeLast();
@@ -249,7 +236,7 @@ class ScannerTest_Replacement extends ScannerTestBase {
   /// that is in the stream.
   void assertValidBeginTokens(fasta.Token firstToken) {
     var openerStack = <analyzer.BeginToken>[];
-    analyzer.BeginToken lastClosedGroup;
+    var errorStack = <fasta.ErrorToken>[];
     fasta.Token token = firstToken;
     while (!token.isEof) {
       if (token is analyzer.BeginToken) {
@@ -257,17 +244,17 @@ class ScannerTest_Replacement extends ScannerTestBase {
           expect(token.endGroup, isNotNull, reason: token.lexeme);
         if (token.endGroup != null) openerStack.add(token);
       } else if (openerStack.isNotEmpty && openerStack.last.endGroup == token) {
-        lastClosedGroup = openerStack.removeLast();
-        expect(token.isSynthetic, token.next is fasta.UnmatchedToken,
-            reason: 'Expect synthetic closer then error token, '
-                'but found "$token" followed by "${token.next}"');
+        analyzer.BeginToken beginToken = openerStack.removeLast();
+        if (token.isSynthetic) {
+          fasta.ErrorToken errorToken = errorStack.removeAt(0);
+          expect(errorToken.begin, beginToken);
+        }
       } else if (token is fasta.UnmatchedToken) {
-        expect(lastClosedGroup?.endGroup?.next, same(token),
-            reason: 'Unexpected error token for group: $lastClosedGroup');
-        expect(token.begin, lastClosedGroup);
+        errorStack.add(token);
       }
       token = token.next;
     }
     expect(openerStack, isEmpty, reason: 'Missing closers');
+    expect(errorStack, isEmpty, reason: 'Extra error tokens');
   }
 }