Auto merge of #32479 - eddyb:eof-not-even-twice, r=nikomatsakis

Prevent bumping the parser past the EOF.

Makes `Parser::bump` after EOF into an ICE, forcing callers to avoid repeated EOF bumps.
This ICE is intended to break infinite loops where EOF wasn't stopping the loop.

For example, the handling of EOF in `parse_trait_items`' recovery loop fixes #32446.
But even without this specific fix, the ICE is triggered, which helps diagnosis and UX.

This is a `[breaking-change]` for plugins authors who eagerly eat multiple EOFs.
See https://github.com/docopt/docopt.rs/pull/171 for such an example and the necessary fix.

src/libsyntax/parse/parser.rs

@@ -254,6 +254,7 @@ pub struct Parser<'a> {
     /// the previous token or None (only stashed sometimes).
     pub last_token: Option<Box<token::Token>>,
     last_token_interpolated: bool,
+    last_token_eof: bool,
     pub buffer: [TokenAndSpan; 4],
     pub buffer_start: isize,
     pub buffer_end: isize,
@@ -366,6 +367,7 @@ pub fn new(sess: &'a ParseSess,
             last_span: span,
             last_token: None,
             last_token_interpolated: false,
+            last_token_eof: false,
             buffer: [
                 placeholder.clone(),
                 placeholder.clone(),
@@ -955,7 +957,9 @@ pub fn parse_unspanned_seq<T, F>(&mut self,
     {
         self.expect(bra)?;
         let result = self.parse_seq_to_before_end(ket, sep, f);
-        self.bump();
+        if self.token == *ket {
+            self.bump();
+        }
         Ok(result)
     }
 
@@ -998,6 +1002,15 @@ pub fn parse_seq<T, F>(&mut self,
 
     /// Advance the parser by one token
     pub fn bump(&mut self) {
+        if self.last_token_eof {
+            // Bumping after EOF is a bad sign, usually an infinite loop.
+            self.bug("attempted to bump the parser past EOF (may be stuck in a loop)");
+        }
+
+        if self.token == token::Eof {
+            self.last_token_eof = true;
+        }
+
         self.last_span = self.span;
         // Stash token for error recovery (sometimes; clone is not necessarily cheap).
         self.last_token = if self.token.is_ident() ||
@@ -1281,15 +1294,21 @@ pub fn parse_trait_items(&mut self) -> PResult<'a,  Vec<TraitItem>> {
                     Ok(cua) => cua,
                     Err(e) => {
                         loop {
-                            p.bump();
-                            if p.token == token::Semi {
-                                p.bump();
-                                break;
-                            }
+                            match p.token {
+                                token::Eof => break,
 
-                            if p.token == token::OpenDelim(token::DelimToken::Brace) {
-                                p.parse_token_tree()?;
-                                break;
+                                token::CloseDelim(token::Brace) |
+                                token::Semi => {
+                                    p.bump();
+                                    break;
+                                }
+
+                                token::OpenDelim(token::Brace) => {
+                                    p.parse_token_tree()?;
+                                    break;
+                                }
+
+                                _ => p.bump()
                             }
                         }
 

src/test/compile-fail/issue-10636-2.rs

@@ -14,6 +14,5 @@
 pub fn trace_option(option: Option<isize>) {
     option.map(|some| 42; //~ NOTE: unclosed delimiter
                           //~^ ERROR: expected one of
-                          //~^^ ERROR: mismatched types
 } //~ ERROR: incorrect close delimiter
-//~^ ERROR: expected one of
+//~^ ERROR: unexpected token

src/test/compile-fail/token-error-correct-3.rs

@@ -21,8 +21,9 @@ pub fn ensure_dir_exists<P: AsRef<Path>, F: FnOnce(&Path)>(path: P,
         if !is_directory(path.as_ref()) { //~ ERROR: unresolved name `is_directory`
             callback(path.as_ref();  //~ NOTE: unclosed delimiter
                      //~^ ERROR: expected one of
-            fs::create_dir_all(path.as_ref()).map(|()| true) //~ ERROR: expected one of
+            fs::create_dir_all(path.as_ref()).map(|()| true) //~ ERROR: mismatched types
         } else { //~ ERROR: incorrect close delimiter: `}`
+            //~^ ERROR: expected one of
             Ok(false);
         }
 

src/test/parse-fail/issue-32446.rs

@@ -0,0 +1,16 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// compile-flags: -Z parse-only
+
+fn main() {}
+
+// This used to end up in an infite loop trying to bump past EOF.
+trait T { ... } //~ ERROR

src/test/parse-fail/pat-lt-bracket-6.rs

@@ -10,5 +10,5 @@
 
 fn main() {
     let Test(&desc[..]) = x; //~ error: expected one of `,` or `@`, found `[`
-    //~^ ERROR expected one of `:`, `;`, or `=`, found `..`
+    //~^ ERROR expected one of `:`, `;`, `=`, or `@`, found `[`
 }

src/test/parse-fail/pat-lt-bracket-7.rs

@@ -10,5 +10,5 @@
 
 fn main() {
     for thing(x[]) in foo {} //~ error: expected one of `,` or `@`, found `[`
-    //~^ ERROR: expected `in`, found `]`
+    //~^ ERROR expected one of `@` or `in`, found `[`
 }