Refactor for statement parsing

This CL introduces 2 new parser listener events
in preparation for parsing for control flow structures
in literal lists, sets, and maps.

Change-Id: I230f36cded714a13e4badb401fe5b5906c93a2da
Reviewed-on: https://dart-review.googlesource.com/c/91144
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Dan Rubel <danrubel@google.com>

pkg/analyzer/lib/src/fasta/ast_builder.dart

@@ -865,14 +865,29 @@ class AstBuilder extends StackListener {
   }
 
   @override
-  void endForStatement(Token forKeyword, Token leftParen, Token leftSeparator,
-      int updateExpressionCount, Token endToken) {
+  void handleForLoopParts(Token forKeyword, Token leftParen,
+      Token leftSeparator, int updateExpressionCount) {
     assert(optional('for', forKeyword));
     assert(optional('(', leftParen));
     assert(optional(';', leftSeparator));
-    debugEvent("ForStatement");
+    assert(updateExpressionCount >= 0);
 
+    push(forKeyword);
+    push(leftParen);
+    push(leftSeparator);
+    push(updateExpressionCount);
+  }
+
+  @override
+  void endForStatement(Token endToken) {
+    debugEvent("ForStatement");
     Statement body = pop();
+
+    int updateExpressionCount = pop();
+    Token leftSeparator = pop();
+    Token leftParen = pop();
+    Token forKeyword = pop();
+
     List<Expression> updates = popTypedList(updateExpressionCount);
     Statement conditionStatement = pop();
     Object initializerPart = pop();
@@ -1276,15 +1291,30 @@ class AstBuilder extends StackListener {
   }
 
   @override
-  void endForIn(Token awaitToken, Token forToken, Token leftParenthesis,
-      Token inKeyword, Token endToken) {
+  void handleForInLoopParts(Token awaitToken, Token forToken,
+      Token leftParenthesis, Token inKeyword) {
     assert(optionalOrNull('await', awaitToken));
     assert(optional('for', forToken));
     assert(optional('(', leftParenthesis));
     assert(optional('in', inKeyword) || optional(':', inKeyword));
+
+    push(awaitToken ?? NullValue.AwaitToken);
+    push(forToken);
+    push(leftParenthesis);
+    push(inKeyword);
+  }
+
+  @override
+  void endForIn(Token endToken) {
     debugEvent("ForInExpression");
 
     Statement body = pop();
+
+    Token inKeyword = pop();
+    Token leftParenthesis = pop();
+    Token forToken = pop();
+    Token awaitToken = pop(NullValue.AwaitToken);
+
     Expression iterator = pop();
     Object variableOrDeclaration = pop();
     if (variableOrDeclaration is VariableDeclarationStatement) {

pkg/analyzer/test/generated/parser_fasta_listener.dart

@@ -704,10 +704,9 @@ class ForwardingTestListener extends ForwardingListener {
   }
 
   @override
-  void endForIn(Token awaitToken, Token forToken, Token leftParen,
-      Token inKeyword, Token endToken) {
+  void endForIn(Token endToken) {
     end('ForStatement');
-    super.endForIn(awaitToken, forToken, leftParen, inKeyword, endToken);
+    super.endForIn(endToken);
   }
 
   @override
@@ -723,11 +722,9 @@ class ForwardingTestListener extends ForwardingListener {
   }
 
   @override
-  void endForStatement(Token forKeyword, Token leftParen, Token leftSeparator,
-      int updateExpressionCount, Token endToken) {
+  void endForStatement(Token endToken) {
     end('ForStatement');
-    super.endForStatement(
-        forKeyword, leftParen, leftSeparator, updateExpressionCount, endToken);
+    super.endForStatement(endToken);
   }
 
   @override

pkg/analyzer/test/src/fasta/recovery/partial_code/forEach_statement_test.dart

@@ -63,6 +63,7 @@ class ForEachStatementTest extends PartialCodeTest {
                 ParserErrorCode.MISSING_IDENTIFIER,
                 ParserErrorCode.MISSING_IDENTIFIER,
                 ParserErrorCode.MISSING_IDENTIFIER,
+                ParserErrorCode.MISSING_IDENTIFIER,
                 ParserErrorCode.EXPECTED_TOKEN,
                 ParserErrorCode.EXPECTED_TOKEN
               ],

pkg/front_end/lib/src/fasta/kernel/body_builder.dart

@@ -2241,10 +2241,24 @@ abstract class BodyBuilder extends ScopeListener<JumpTarget>
   }
 
   @override
-  void endForStatement(Token forKeyword, Token leftParen, Token leftSeparator,
-      int updateExpressionCount, Token endToken) {
+  void handleForLoopParts(Token forKeyword, Token leftParen,
+      Token leftSeparator, int updateExpressionCount) {
+    push(forKeyword);
+    push(leftParen);
+    push(leftSeparator);
+    push(updateExpressionCount);
+  }
+
+  @override
+  void endForStatement(Token endToken) {
     debugEvent("ForStatement");
     Statement body = popStatement();
+
+    int updateExpressionCount = pop();
+    Token leftSeparator = pop();
+    Token leftParen = pop();
+    Token forKeyword = pop();
+
     List<Expression> updates = popListForEffect(updateExpressionCount);
     Statement conditionStatement = popStatement();
     Object variableOrExpression = pop();
@@ -3829,10 +3843,22 @@ abstract class BodyBuilder extends ScopeListener<JumpTarget>
   }
 
   @override
-  void endForIn(Token awaitToken, Token forToken, Token leftParenthesis,
-      Token inKeyword, Token endToken) {
+  void handleForInLoopParts(Token awaitToken, Token forToken,
+      Token leftParenthesis, Token inKeyword) {
+    push(awaitToken ?? NullValue.AwaitToken);
+    push(forToken);
+    push(inKeyword);
+  }
+
+  @override
+  void endForIn(Token endToken) {
     debugEvent("ForIn");
     Statement body = popStatement();
+
+    Token inKeyword = pop();
+    Token forToken = pop();
+    Token awaitToken = pop(NullValue.AwaitToken);
+
     Expression expression = popForValue();
     Object lvalue = pop();
     exitLocalScope();

pkg/front_end/lib/src/fasta/parser/forwarding_listener.dart

@@ -564,9 +564,8 @@ class ForwardingListener implements Listener {
   }
 
   @override
-  void endForIn(Token awaitToken, Token forToken, Token leftParen,
-      Token inKeyword, Token endToken) {
-    listener?.endForIn(awaitToken, forToken, leftParen, inKeyword, endToken);
+  void endForIn(Token endToken) {
+    listener?.endForIn(endToken);
   }
 
   @override
@@ -580,10 +579,8 @@ class ForwardingListener implements Listener {
   }
 
   @override
-  void endForStatement(Token forKeyword, Token leftParen, Token leftSeparator,
-      int updateExpressionCount, Token endToken) {
-    listener?.endForStatement(
-        forKeyword, leftParen, leftSeparator, updateExpressionCount, endToken);
+  void endForStatement(Token endToken) {
+    listener?.endForStatement(endToken);
   }
 
   @override
@@ -1219,6 +1216,20 @@ class ForwardingListener implements Listener {
     listener?.handleForInitializerLocalVariableDeclaration(token);
   }
 
+  @override
+  void handleForInLoopParts(Token awaitToken, Token forToken,
+      Token leftParenthesis, Token inKeyword) {
+    listener?.handleForInLoopParts(
+        awaitToken, forToken, leftParenthesis, inKeyword);
+  }
+
+  @override
+  void handleForLoopParts(Token forKeyword, Token leftParen,
+      Token leftSeparator, int updateExpressionCount) {
+    listener?.handleForLoopParts(
+        forKeyword, leftParen, leftSeparator, updateExpressionCount);
+  }
+
   @override
   void handleNoFieldInitializer(Token token) {
     listener?.handleNoFieldInitializer(token);

pkg/front_end/lib/src/fasta/parser/listener.dart

@@ -323,8 +323,13 @@ class Listener implements UnescapeErrorListener {
   /// [endForStatement] or [endForIn].
   void beginForStatement(Token token) {}
 
-  void endForStatement(Token forKeyword, Token leftParen, Token leftSeparator,
-      int updateExpressionCount, Token endToken) {
+  /// Marks the end of parsing the control structure of a for statement
+  /// or for control flow entry up to and including the closing parenthesis.
+  /// `for` `(` initialization `;` condition `;` updaters `)`
+  void handleForLoopParts(Token forKeyword, Token leftParen,
+      Token leftSeparator, int updateExpressionCount) {}
+
+  void endForStatement(Token endToken) {
     logEvent("ForStatement");
   }
 
@@ -334,9 +339,14 @@ class Listener implements UnescapeErrorListener {
     logEvent("ForStatementBody");
   }
 
+  /// Marks the end of parsing the control structure of a for-in statement
+  /// or for control flow entry up to and including the closing parenthesis.
+  /// `for` `(` (type)? identifier `in` iterator `)`
+  void handleForInLoopParts(Token awaitToken, Token forToken,
+      Token leftParenthesis, Token inKeyword) {}
+
   // One of the two possible corresponding end events for [beginForStatement].
-  void endForIn(Token awaitToken, Token forToken, Token leftParenthesis,
-      Token inKeyword, Token endToken) {
+  void endForIn(Token endToken) {
     logEvent("ForIn");
   }
 

pkg/front_end/lib/src/fasta/parser/parser.dart

@@ -5296,10 +5296,11 @@ class Parser {
   ///   'await'? 'for' '(' forLoopParts ')' statement
   /// ;
   ///
-  /// forLoopParts:
-  ///   forInitializerStatement expression? ';' expressionList? |
-  ///   declaredIdentifier 'in' expression |
-  ///   identifier 'in' expression
+  ///  forLoopParts:
+  ///      localVariableDeclaration ';' expression? ';' expressionList?
+  ///    | expression? ';' expression? ';' expressionList?
+  ///    | localVariableDeclaration 'in' expression
+  ///    | identifier 'in' expression
   /// ;
   ///
   /// forInitializerStatement:
@@ -5308,20 +5309,35 @@ class Parser {
   /// ;
   /// ```
   Token parseForStatement(Token token, Token awaitToken) {
-    Token forKeyword = token = token.next;
+    Token forToken = token = token.next;
     assert(awaitToken == null || optional('await', awaitToken));
     assert(optional('for', token));
-    listener.beginForStatement(forKeyword);
+    listener.beginForStatement(forToken);
 
-    Token leftParenthesis = forKeyword.next;
+    token = parseForLoopPartsStart(awaitToken, forToken);
+    Token identifier = token.next;
+    token = parseForLoopPartsMid(token, forToken);
+    if (looksLikeForInLoopParts(awaitToken, token.next)) {
+      // Process `for ( ... in ... )`
+      return parseForInRest(token, awaitToken, forToken, identifier);
+    } else {
+      // Process `for ( ... ; ... ; ... )`
+      return parseForRest(awaitToken, token, forToken);
+    }
+  }
+
+  /// Parse the start of a for loop control structure
+  /// from the open parenthesis up to but not including the identifier.
+  Token parseForLoopPartsStart(Token awaitToken, Token forToken) {
+    Token leftParenthesis = forToken.next;
     if (!optional('(', leftParenthesis)) {
       // Recovery
       reportRecoverableError(
           leftParenthesis, fasta.templateExpectedButGot.withArguments('('));
       int offset = leftParenthesis.offset;
 
-      BeginToken openParen =
-          token.setNext(new SyntheticBeginToken(TokenType.OPEN_PAREN, offset));
+      BeginToken openParen = forToken
+          .setNext(new SyntheticBeginToken(TokenType.OPEN_PAREN, offset));
 
       Token loopPart;
       if (awaitToken != null) {
@@ -5349,18 +5365,18 @@ class Parser {
 
       leftParenthesis = openParen;
     }
-    token = leftParenthesis;
 
     // Pass `true` so that the [parseExpressionStatementOrDeclaration] only
     // parses the metadata, modifiers, and type of a local variable
     // declaration if it exists. This enables capturing [beforeIdentifier]
     // for later error reporting.
-    token = parseExpressionStatementOrDeclaration(token, true);
-    Token beforeIdentifier = token;
+    return parseExpressionStatementOrDeclaration(forToken.next, true);
+  }
 
-    // Parse the remainder of the local variable declaration
-    // or an expression if no local variable declaration was found.
-    if (token != leftParenthesis) {
+  /// Parse the remainder of the local variable declaration
+  /// or an expression if no local variable declaration was found.
+  Token parseForLoopPartsMid(Token token, Token forToken) {
+    if (token != forToken.next) {
       token = parseVariablesDeclarationRest(token, false);
       listener.handleForInitializerLocalVariableDeclaration(token);
     } else if (optional(';', token.next)) {
@@ -5369,47 +5385,17 @@ class Parser {
       token = parseExpression(token);
       listener.handleForInitializerExpressionStatement(token);
     }
-
-    Token next = token.next;
-    if (!optional('in', next)) {
-      if (optional(':', next)) {
-        // Recovery
-        reportRecoverableError(next, fasta.messageColonInPlaceOfIn);
-        // Fall through to process `for ( ... in ... )`
-      } else if (awaitToken == null || optional(';', next)) {
-        // Process `for ( ... ; ... ; ... )`
-        if (awaitToken != null) {
-          reportRecoverableError(awaitToken, fasta.messageInvalidAwaitFor);
-        }
-        return parseForRest(token, forKeyword, leftParenthesis);
-      } else {
-        // Recovery
-        reportRecoverableError(
-            next, fasta.templateExpectedButGot.withArguments('in'));
-        next = token.setNext(
-            new SyntheticKeywordToken(Keyword.IN, next.offset)..setNext(next));
-      }
-    }
-
-    // Process `for ( ... in ... )`
-    Token identifier = beforeIdentifier.next;
-    if (!identifier.isIdentifier) {
-      reportRecoverableErrorWithToken(
-          identifier, fasta.templateExpectedIdentifier);
-    } else if (identifier != token) {
-      if (optional('=', identifier.next)) {
-        reportRecoverableError(
-            identifier.next, fasta.messageInitializedVariableInForEach);
-      } else {
-        reportRecoverableErrorWithToken(
-            identifier.next, fasta.templateUnexpectedToken);
-      }
-    } else if (awaitToken != null && !inAsync) {
-      reportRecoverableError(next, fasta.messageAwaitForNotAsync);
-    }
-    return parseForInRest(token, awaitToken, forKeyword, leftParenthesis);
+    return token;
   }
 
+  /// Return true if the combination of the [awaitToken] and [inKeyword]
+  /// indicate that a for loop is being parsed of the form:
+  /// (`await`)? `for` `(` type? identifier `in`
+  bool looksLikeForInLoopParts(Token awaitToken, Token inKeyword) =>
+      optional('in', inKeyword) ||
+      optional(':', inKeyword) ||
+      (!optional(';', inKeyword) && awaitToken != null);
+
   /// This method parses the portion of the forLoopParts that starts with the
   /// first semicolon (the one that terminates the forInitializerStatement).
   ///
@@ -5420,7 +5406,26 @@ class Parser {
   ///   identifier 'in' expression
   /// ;
   /// ```
-  Token parseForRest(Token token, Token forToken, Token leftParenthesis) {
+  Token parseForRest(Token awaitToken, Token token, Token forToken) {
+    token = parseForLoopPartsRest(forToken, awaitToken, token);
+    listener.beginForStatementBody(token.next);
+    LoopState savedLoopState = loopState;
+    loopState = LoopState.InsideLoop;
+    token = parseStatement(token);
+    loopState = savedLoopState;
+    listener.endForStatementBody(token.next);
+    listener.endForStatement(token.next);
+    return token;
+  }
+
+  Token parseForLoopPartsRest(Token forToken, Token awaitToken, Token token) {
+    Token leftParenthesis = forToken.next;
+    assert(optional('for', forToken));
+    assert(optional('(', leftParenthesis));
+
+    if (awaitToken != null) {
+      reportRecoverableError(awaitToken, fasta.messageInvalidAwaitFor);
+    }
     Token leftSeparator = ensureSemicolon(token);
     if (optional(';', leftSeparator.next)) {
       token = parseEmptyStatement(leftSeparator);
@@ -5444,14 +5449,8 @@ class Parser {
       reportRecoverableErrorWithToken(token, fasta.templateUnexpectedToken);
       token = leftParenthesis.endGroup;
     }
-    listener.beginForStatementBody(token.next);
-    LoopState savedLoopState = loopState;
-    loopState = LoopState.InsideLoop;
-    token = parseStatement(token);
-    loopState = savedLoopState;
-    listener.endForStatementBody(token.next);
-    listener.endForStatement(
-        forToken, leftParenthesis, leftSeparator, expressionCount, token.next);
+    listener.handleForLoopParts(
+        forToken, leftParenthesis, leftSeparator, expressionCount);
     return token;
   }
 
@@ -5467,21 +5466,59 @@ class Parser {
   /// ;
   /// ```
   Token parseForInRest(
-      Token token, Token awaitToken, Token forKeyword, Token leftParenthesis) {
-    Token inKeyword = token.next;
-    assert(optional('in', inKeyword) || optional(':', inKeyword));
-    listener.beginForInExpression(inKeyword.next);
-    token = parseExpression(inKeyword);
-    token = ensureCloseParen(token, leftParenthesis);
-    listener.endForInExpression(token);
+      Token token, Token awaitToken, Token forToken, Token identifier) {
+    token = parseForInLoopPartsRest(forToken, token, identifier, awaitToken);
     listener.beginForInBody(token.next);
     LoopState savedLoopState = loopState;
     loopState = LoopState.InsideLoop;
     token = parseStatement(token);
     loopState = savedLoopState;
     listener.endForInBody(token.next);
-    listener.endForIn(
-        awaitToken, forKeyword, leftParenthesis, inKeyword, token.next);
+    listener.endForIn(token.next);
+    return token;
+  }
+
+  Token parseForInLoopPartsRest(
+      Token forToken, Token token, Token identifier, Token awaitToken) {
+    assert(optional('for', forToken));
+    assert(optional('(', forToken.next));
+
+    Token inKeyword = token.next;
+    if (!optional('in', inKeyword)) {
+      // Recovery
+      if (optional(':', inKeyword)) {
+        reportRecoverableError(inKeyword, fasta.messageColonInPlaceOfIn);
+      } else {
+        reportRecoverableError(
+            inKeyword, fasta.templateExpectedButGot.withArguments('in'));
+        inKeyword = token.setNext(
+            new SyntheticKeywordToken(Keyword.IN, inKeyword.offset)
+              ..setNext(inKeyword));
+      }
+    }
+
+    if (!identifier.isIdentifier) {
+      reportRecoverableErrorWithToken(
+          identifier, fasta.templateExpectedIdentifier);
+    } else if (identifier != token) {
+      if (optional('=', identifier.next)) {
+        reportRecoverableError(
+            identifier.next, fasta.messageInitializedVariableInForEach);
+      } else {
+        reportRecoverableErrorWithToken(
+            identifier.next, fasta.templateUnexpectedToken);
+      }
+    } else if (awaitToken != null && !inAsync) {
+      // TODO(danrubel): consider reporting the error on awaitToken
+      reportRecoverableError(inKeyword, fasta.messageAwaitForNotAsync);
+    }
+
+    listener.beginForInExpression(inKeyword.next);
+    token = parseExpression(inKeyword);
+    token = ensureCloseParen(token, forToken.next);
+    listener.endForInExpression(token);
+    listener.handleForInLoopParts(
+        awaitToken, forToken, forToken.next, inKeyword);
     return token;
   }
 

pkg/front_end/lib/src/fasta/source/stack_listener.dart

@@ -31,6 +31,7 @@ import '../scanner.dart' show Token;
 enum NullValue {
   Arguments,
   As,
+  AwaitToken,
   Block,
   BreakTarget,
   CascadeReceiver,

pkg/front_end/lib/src/fasta/source/type_promotion_look_ahead_listener.dart

@@ -541,9 +541,8 @@ class TypePromotionLookAheadListener extends Listener {
   }
 
   @override
-  void endForIn(Token awaitToken, Token forToken, Token leftParenthesis,
-      Token inKeyword, Token endToken) {
-    debugEvent("ForIn", awaitToken);
+  void endForIn(Token endToken) {
+    debugEvent("ForIn", endToken);
   }
 
   @override
@@ -574,12 +573,17 @@ class TypePromotionLookAheadListener extends Listener {
   }
 
   @override
-  void endForStatement(Token forKeyword, Token leftParen, Token leftSeparator,
-      int updateExpressionCount, Token endToken) {
-    debugEvent("ForStatement", forKeyword);
+  void handleForLoopParts(Token forKeyword, Token leftParen,
+      Token leftSeparator, int updateExpressionCount) {
+    debugEvent("handleForLoopParts", forKeyword);
     state.discard(updateExpressionCount);
   }
 
+  @override
+  void endForStatement(Token endToken) {
+    debugEvent("ForStatement", endToken);
+  }
+
   @override
   void endForStatementBody(Token token) {
     debugEvent("ForStatementBody", token);