languages/rust
languages/rust
1
0
Fork 0
mirror of https://github.com/rust-lang/rust synced 2024-11-05 20:45:15 +00:00
Code Issues Projects Releases Packages Wiki Activity

Handle None-delimited groups when parsing macro_rules! macro

Browse source 
When a `macro_rules!` macro expands to another `macro_rules!` macro, we
may see `None`-delimited groups in odd places when another crate
deserializes the 'inner' macro. This commit 'unwraps' an outer
`None`-delimited group to avoid breaking existing code.

See https://github.com/rust-lang/rust/pull/73569#issuecomment-650860457
for more details.

The proper fix is to handle `None`-delimited groups systematically
throughout the parser, but that will require significant work. In the
meantime, this hack lets us fix important hygiene bugs in macros
This commit is contained in:
Aaron Hill 2020-06-29 09:55:28 -04:00
parent c84402872e
commit 1ded7a5815
No known key found for this signature in database
GPG key ID: B4087E510E98B164
3 changed files with 87 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

110
src/librustc_expand/mbe/quoted.rs
View file

@ -90,7 +90,7 @@ pub(super) fn parse(
/// # Parameters
///
/// - `tree`: the tree we wish to convert.
/// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish
/// - `outer_trees`: an iterator over trees. We may need to read more tokens from it in order to finish
/// converting `tree`
/// - `expect_matchers`: same as for `parse` (see above).
/// - `sess`: the parsing session. Any errors will be emitted to this session.
@ -98,7 +98,7 @@ pub(super) fn parse(
/// unstable features or not.
fn parse_tree(
tree: tokenstream::TokenTree,
trees: &mut impl Iterator<Item = tokenstream::TokenTree>,
outer_trees: &mut impl Iterator<Item = tokenstream::TokenTree>,
expect_matchers: bool,
sess: &ParseSess,
node_id: NodeId,
@ -106,56 +106,72 @@ fn parse_tree(
// Depending on what `tree` is, we could be parsing different parts of a macro
match tree {
// `tree` is a `$` token. Look at the next token in `trees`
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() {
// `tree` is followed by a delimited set of token trees. This indicates the beginning
// of a repetition sequence in the macro (e.g. `$(pat)*`).
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
// Must have `(` not `{` or `[`
if delim != token::Paren {
let tok = pprust::token_kind_to_string(&token::OpenDelim(delim));
let msg = format!("expected `(`, found `{}`", tok);
sess.span_diagnostic.span_err(span.entire(), &msg);
tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => {
// FIXME: Handle `None`-delimited groups in a more systematic way
// during parsing.
let mut next = outer_trees.next();
let mut trees: Box<dyn Iterator<Item = tokenstream::TokenTree>>;
if let Some(tokenstream::TokenTree::Delimited(_, token::NoDelim, tts)) = next {
trees = Box::new(tts.into_trees());
next = trees.next();
} else {
trees = Box::new(outer_trees);
}
match next {
// `tree` is followed by a delimited set of token trees. This indicates the beginning
// of a repetition sequence in the macro (e.g. `$(pat)*`).
Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => {
// Must have `(` not `{` or `[`
if delim != token::Paren {
let tok = pprust::token_kind_to_string(&token::OpenDelim(delim));
let msg = format!("expected `(`, found `{}`", tok);
sess.span_diagnostic.span_err(span.entire(), &msg);
}
// Parse the contents of the sequence itself
let sequence = parse(tts, expect_matchers, sess, node_id);
// Get the Kleene operator and optional separator
let (separator, kleene) =
parse_sep_and_kleene_op(&mut trees, span.entire(), sess);
// Count the number of captured "names" (i.e., named metavars)
let name_captures = macro_parser::count_names(&sequence);
TokenTree::Sequence(
span,
Lrc::new(SequenceRepetition {
tts: sequence,
separator,
kleene,
num_captures: name_captures,
}),
)
}
// Parse the contents of the sequence itself
let sequence = parse(tts, expect_matchers, sess, node_id);
// Get the Kleene operator and optional separator
let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess);
// Count the number of captured "names" (i.e., named metavars)
let name_captures = macro_parser::count_names(&sequence);
TokenTree::Sequence(
span,
Lrc::new(SequenceRepetition {
tts: sequence,
separator,
kleene,
num_captures: name_captures,
}),
)
}
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special
// metavariable that names the crate of the invocation.
Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => {
let (ident, is_raw) = token.ident().unwrap();
let span = ident.span.with_lo(span.lo());
if ident.name == kw::Crate && !is_raw {
TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span)
} else {
TokenTree::MetaVar(span, ident)
// `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special
// metavariable that names the crate of the invocation.
Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => {
let (ident, is_raw) = token.ident().unwrap();
let span = ident.span.with_lo(span.lo());
if ident.name == kw::Crate && !is_raw {
TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span)
} else {
TokenTree::MetaVar(span, ident)
}
}
}
// `tree` is followed by a random token. This is an error.
Some(tokenstream::TokenTree::Token(token)) => {
let msg =
format!("expected identifier, found `{}`", pprust::token_to_string(&token),);
sess.span_diagnostic.span_err(token.span, &msg);
TokenTree::MetaVar(token.span, Ident::invalid())
}
// `tree` is followed by a random token. This is an error.
Some(tokenstream::TokenTree::Token(token)) => {
let msg = format!(
"expected identifier, found `{}`",
pprust::token_to_string(&token),
);
sess.span_diagnostic.span_err(token.span, &msg);
TokenTree::MetaVar(token.span, Ident::invalid())
}
// There are no more tokens. Just return the `$` we already have.
None => TokenTree::token(token::Dollar, span),
},
// There are no more tokens. Just return the `$` we already have.
None => TokenTree::token(token::Dollar, span),
}
}
// `tree` is an arbitrary token. Keep it.
tokenstream::TokenTree::Token(token) => TokenTree::Token(token),

12
src/test/ui/proc-macro/auxiliary/meta-delim.rs Normal file
View file

@ -0,0 +1,12 @@
macro_rules! produce_it {
($dollar_one:tt $foo:ident $my_name:ident) => {
#[macro_export]
macro_rules! meta_delim {
($dollar_one ($dollar_one $my_name:ident)*) => {
stringify!($dollar_one ($dollar_one $my_name)*)
}
}
}
}
produce_it!($my_name name);

12
src/test/ui/proc-macro/meta-delim.rs Normal file
View file

@ -0,0 +1,12 @@
// aux-build:meta-delim.rs
// edition:2018
// run-pass
// Tests that we can properly deserialize a macro with strange delimiters
// See https://github.com/rust-lang/rust/pull/73569#issuecomment-650860457
extern crate meta_delim;
fn main() {
assert_eq!("a bunch of idents", meta_delim::meta_delim!(a bunch of idents));
}