Merge #11141

11141: internal: add tests for extra parser entry points r=matklad a=matklad

bors r+
🤖

Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

crates/hir_def/src/macro_expansion_tests/mbe/matching.rs

@@ -50,3 +50,56 @@ fn test_empty_comments() {
 "#]],
     );
 }
+
+#[test]
+fn asi() {
+    // Thanks, Christopher!
+    //
+    // https://internals.rust-lang.org/t/understanding-decisions-behind-semicolons/15181/29
+    check(
+        r#"
+macro_rules! asi { ($($stmt:stmt)*) => ($($stmt)*); }
+
+fn main() {
+    asi! {
+        let a = 2
+        let b = 5
+        drop(b-a)
+        println!("{}", a+b)
+    }
+}
+"#,
+        expect![[r#"
+macro_rules! asi { ($($stmt:stmt)*) => ($($stmt)*); }
+
+fn main() {
+    let a = 2let b = 5drop(b-a)println!("{}", a+b)
+}
+"#]],
+    )
+}
+
+#[test]
+fn stmt_boundaries() {
+    // FIXME: this actually works OK under rustc.
+    check(
+        r#"
+macro_rules! m {
+    ($($s:stmt)*) => (stringify!($($s |)*);)
+}
+m!(;;92;let x = 92; loop {};);
+"#,
+        expect![[r#"
+macro_rules! m {
+    ($($s:stmt)*) => (stringify!($($s |)*);)
+}
+stringify!(;
+|;
+|92|;
+|let x = 92|;
+|loop {}
+|;
+|);
+"#]],
+    );
+}

crates/parser/src/grammar.rs

@@ -59,7 +59,7 @@ pub(crate) fn block(p: &mut Parser) {
         }
 
         pub(crate) fn stmt(p: &mut Parser) {
-            expressions::stmt(p, expressions::StmtWithSemi::No, true);
+            expressions::stmt(p, expressions::Semicolon::Forbidden);
         }
 
         pub(crate) fn pat(p: &mut Parser) {
@@ -98,12 +98,7 @@ pub(crate) fn macro_stmts(p: &mut Parser) {
             let m = p.start();
 
             while !p.at(EOF) {
-                if p.at(T![;]) {
-                    p.bump(T![;]);
-                    continue;
-                }
-
-                expressions::stmt(p, expressions::StmtWithSemi::Optional, true);
+                expressions::stmt(p, expressions::Semicolon::Optional);
             }
 
             m.complete(p, MACRO_STMTS);

crates/parser/src/grammar/expressions.rs

@@ -5,10 +5,11 @@
 pub(crate) use self::atom::{block_expr, match_arm_list};
 pub(super) use self::atom::{literal, LITERAL_FIRST};
 
-pub(super) enum StmtWithSemi {
-    Yes,
-    No,
+#[derive(PartialEq, Eq)]
+pub(super) enum Semicolon {
+    Required,
     Optional,
+    Forbidden,
 }
 
 const EXPR_FIRST: TokenSet = LHS_FIRST;
@@ -28,7 +29,11 @@ fn expr_no_struct(p: &mut Parser) {
     expr_bp(p, None, r, 1);
 }
 
-pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi, prefer_expr: bool) {
+pub(super) fn stmt(p: &mut Parser, semicolon: Semicolon) {
+    if p.eat(T![;]) {
+        return;
+    }
+
     let m = p.start();
     // test attr_on_expr_stmt
     // fn foo() {
@@ -40,7 +45,7 @@ pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi, prefer_expr: bool) {
     attributes::outer_attrs(p);
 
     if p.at(T![let]) {
-        let_stmt(p, m, with_semi);
+        let_stmt(p, m, semicolon);
         return;
     }
 
@@ -52,7 +57,7 @@ pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi, prefer_expr: bool) {
     };
 
     if let Some((cm, blocklike)) = expr_stmt(p, Some(m)) {
-        if !(p.at(T!['}']) || (prefer_expr && p.at(EOF))) {
+        if !(p.at(T!['}']) || (semicolon != Semicolon::Required && p.at(EOF))) {
             // test no_semi_after_block
             // fn foo() {
             //     if true {}
@@ -68,27 +73,26 @@ pub(super) fn stmt(p: &mut Parser, with_semi: StmtWithSemi, prefer_expr: bool) {
             //     test!{}
             // }
             let m = cm.precede(p);
-            match with_semi {
-                StmtWithSemi::No => (),
-                StmtWithSemi::Optional => {
-                    p.eat(T![;]);
-                }
-                StmtWithSemi::Yes => {
+            match semicolon {
+                Semicolon::Required => {
                     if blocklike.is_block() {
                         p.eat(T![;]);
                     } else {
                         p.expect(T![;]);
                     }
                 }
+                Semicolon::Optional => {
+                    p.eat(T![;]);
+                }
+                Semicolon::Forbidden => (),
             }
-
             m.complete(p, EXPR_STMT);
         }
     }
 
     // test let_stmt
     // fn f() { let x: i32 = 92; }
-    fn let_stmt(p: &mut Parser, m: Marker, with_semi: StmtWithSemi) {
+    fn let_stmt(p: &mut Parser, m: Marker, with_semi: Semicolon) {
         p.bump(T![let]);
         patterns::pattern(p);
         if p.at(T![:]) {
@@ -113,11 +117,11 @@ fn let_stmt(p: &mut Parser, m: Marker, with_semi: StmtWithSemi) {
         }
 
         match with_semi {
-            StmtWithSemi::No => (),
-            StmtWithSemi::Optional => {
+            Semicolon::Forbidden => (),
+            Semicolon::Optional => {
                 p.eat(T![;]);
             }
-            StmtWithSemi::Yes => {
+            Semicolon::Required => {
                 p.expect(T![;]);
             }
         }
@@ -143,13 +147,7 @@ pub(super) fn expr_block_contents(p: &mut Parser) {
         //     fn f() {};
         //     struct S {};
         // }
-
-        if p.at(T![;]) {
-            p.bump(T![;]);
-            continue;
-        }
-
-        stmt(p, StmtWithSemi::Yes, false);
+        stmt(p, Semicolon::Required);
     }
 }
 

crates/parser/src/output.rs

@@ -22,6 +22,7 @@ pub struct Output {
     error: Vec<String>,
 }
 
+#[derive(Debug)]
 pub enum Step<'a> {
     Token { kind: SyntaxKind, n_input_tokens: u8 },
     Enter { kind: SyntaxKind },

crates/parser/src/shortcuts.rs

@@ -16,6 +16,7 @@
     SyntaxKind::{self, *},
 };
 
+#[derive(Debug)]
 pub enum StrStep<'a> {
     Token { kind: SyntaxKind, text: &'a str },
     Enter { kind: SyntaxKind },
@@ -49,6 +50,7 @@ pub fn to_input(&self) -> crate::Input {
         res
     }
 
+    /// NB: only valid to call with Output from Reparser/TopLevelEntry.
     pub fn intersperse_trivia(
         &self,
         output: &crate::Output,

crates/parser/src/tests.rs

@@ -1,4 +1,5 @@
 mod sourcegen_inline_tests;
+mod entries;
 
 use std::{
     fmt::Write,

crates/parser/src/tests/entries.rs

@@ -0,0 +1,54 @@
+use crate::{LexedStr, PrefixEntryPoint, Step};
+
+#[test]
+fn vis() {
+    check_prefix(PrefixEntryPoint::Vis, "pub(crate) fn foo() {}", "pub(crate)");
+    check_prefix(PrefixEntryPoint::Vis, "fn foo() {}", "");
+    check_prefix(PrefixEntryPoint::Vis, "pub(fn foo() {}", "pub");
+    check_prefix(PrefixEntryPoint::Vis, "pub(crate fn foo() {}", "pub(crate");
+    check_prefix(PrefixEntryPoint::Vis, "crate fn foo() {}", "crate");
+}
+
+#[test]
+fn block() {
+    check_prefix(PrefixEntryPoint::Block, "{}, 92", "{}");
+    check_prefix(PrefixEntryPoint::Block, "{, 92)", "{, 92)");
+    check_prefix(PrefixEntryPoint::Block, "()", "");
+}
+
+#[test]
+fn stmt() {
+    check_prefix(PrefixEntryPoint::Stmt, "92; fn", "92");
+    check_prefix(PrefixEntryPoint::Stmt, "let _ = 92; 1", "let _ = 92");
+    check_prefix(PrefixEntryPoint::Stmt, "pub fn f() {} = 92", "pub fn f() {}");
+    check_prefix(PrefixEntryPoint::Stmt, ";;;", ";");
+    check_prefix(PrefixEntryPoint::Stmt, "+", "+");
+    check_prefix(PrefixEntryPoint::Stmt, "@", "@");
+    check_prefix(PrefixEntryPoint::Stmt, "loop {} - 1", "loop {}");
+}
+
+fn check_prefix(entry: PrefixEntryPoint, input: &str, prefix: &str) {
+    let lexed = LexedStr::new(input);
+    let input = lexed.to_input();
+
+    let mut n_tokens = 0;
+    for step in entry.parse(&input).iter() {
+        match step {
+            Step::Token { n_input_tokens, .. } => n_tokens += n_input_tokens as usize,
+            Step::Enter { .. } | Step::Exit | Step::Error { .. } => (),
+        }
+    }
+
+    let mut i = 0;
+    loop {
+        if n_tokens == 0 {
+            break;
+        }
+        if !lexed.kind(i).is_trivia() {
+            n_tokens -= 1;
+        }
+        i += 1;
+    }
+    let buf = &lexed.as_str()[..lexed.text_start(i)];
+    assert_eq!(buf, prefix);
+}