mirror of
https://github.com/rust-lang/rust
synced 2024-10-14 20:46:49 +00:00
Rollup merge of #96421 - nnethercote:less-NoDelim, r=petrochenkov
Less `NoDelim` Currently there are several places where `NoDelim` (which really means "implicit delimiter" or "invisible delimiter") is used to mean "no delimiter". The name `NoDelim` is a bit misleading, and may be a cause. This PR changes these places, e.g. by changing a `DelimToken` to `Option<DelimToken>` and then using `None` to mean "no delimiter". As a result, the *only* place where `NoDelim` values are now produced is within: - `Delimiter::to_internal()`, when converting from `Delimiter::None`. - `FlattenNonterminals::process_token()`, when converting `TokenKind::Interpolated`. r? ````@petrochenkov````
This commit is contained in:
commit
80045d65e1
|
@ -1542,10 +1542,10 @@ pub enum MacArgs {
|
|||
}
|
||||
|
||||
impl MacArgs {
|
||||
pub fn delim(&self) -> DelimToken {
|
||||
pub fn delim(&self) -> Option<DelimToken> {
|
||||
match self {
|
||||
MacArgs::Delimited(_, delim, _) => delim.to_token(),
|
||||
MacArgs::Empty | MacArgs::Eq(..) => token::NoDelim,
|
||||
MacArgs::Delimited(_, delim, _) => Some(delim.to_token()),
|
||||
MacArgs::Empty | MacArgs::Eq(..) => None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -464,7 +464,7 @@ fn print_attr_item(&mut self, item: &ast::AttrItem, span: Span) {
|
|||
Some(MacHeader::Path(&item.path)),
|
||||
false,
|
||||
None,
|
||||
delim.to_token(),
|
||||
Some(delim.to_token()),
|
||||
tokens,
|
||||
true,
|
||||
span,
|
||||
|
@ -530,7 +530,7 @@ fn print_tt(&mut self, tt: &TokenTree, convert_dollar_crate: bool) {
|
|||
None,
|
||||
false,
|
||||
None,
|
||||
*delim,
|
||||
Some(*delim),
|
||||
tts,
|
||||
convert_dollar_crate,
|
||||
dspan.entire(),
|
||||
|
@ -556,12 +556,12 @@ fn print_mac_common(
|
|||
header: Option<MacHeader<'_>>,
|
||||
has_bang: bool,
|
||||
ident: Option<Ident>,
|
||||
delim: DelimToken,
|
||||
delim: Option<DelimToken>,
|
||||
tts: &TokenStream,
|
||||
convert_dollar_crate: bool,
|
||||
span: Span,
|
||||
) {
|
||||
if delim == DelimToken::Brace {
|
||||
if delim == Some(DelimToken::Brace) {
|
||||
self.cbox(INDENT_UNIT);
|
||||
}
|
||||
match header {
|
||||
|
@ -577,7 +577,7 @@ fn print_mac_common(
|
|||
self.print_ident(ident);
|
||||
}
|
||||
match delim {
|
||||
DelimToken::Brace => {
|
||||
Some(DelimToken::Brace) => {
|
||||
if header.is_some() || has_bang || ident.is_some() {
|
||||
self.nbsp();
|
||||
}
|
||||
|
@ -585,23 +585,25 @@ fn print_mac_common(
|
|||
if !tts.is_empty() {
|
||||
self.space();
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
let token_str = self.token_kind_to_string(&token::OpenDelim(delim));
|
||||
self.word(token_str)
|
||||
}
|
||||
}
|
||||
self.ibox(0);
|
||||
self.print_tts(tts, convert_dollar_crate);
|
||||
self.end();
|
||||
match delim {
|
||||
DelimToken::Brace => {
|
||||
self.ibox(0);
|
||||
self.print_tts(tts, convert_dollar_crate);
|
||||
self.end();
|
||||
let empty = tts.is_empty();
|
||||
self.bclose(span, empty);
|
||||
}
|
||||
_ => {
|
||||
Some(delim) => {
|
||||
let token_str = self.token_kind_to_string(&token::OpenDelim(delim));
|
||||
self.word(token_str);
|
||||
self.ibox(0);
|
||||
self.print_tts(tts, convert_dollar_crate);
|
||||
self.end();
|
||||
let token_str = self.token_kind_to_string(&token::CloseDelim(delim));
|
||||
self.word(token_str)
|
||||
self.word(token_str);
|
||||
}
|
||||
None => {
|
||||
self.ibox(0);
|
||||
self.print_tts(tts, convert_dollar_crate);
|
||||
self.end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -260,16 +260,15 @@ fn generic_extension<'cx, 'tt>(
|
|||
// Merge the gated spans from parsing the matcher with the pre-existing ones.
|
||||
sess.gated_spans.merge(gated_spans_snapshot);
|
||||
|
||||
// Ignore the delimiters on the RHS.
|
||||
let rhs = match &rhses[i] {
|
||||
mbe::TokenTree::Delimited(_, delimited) => &delimited.tts,
|
||||
let (rhs, rhs_span): (&mbe::Delimited, DelimSpan) = match &rhses[i] {
|
||||
mbe::TokenTree::Delimited(span, delimited) => (&delimited, *span),
|
||||
_ => cx.span_bug(sp, "malformed macro rhs"),
|
||||
};
|
||||
let arm_span = rhses[i].span();
|
||||
|
||||
let rhs_spans = rhs.iter().map(|t| t.span()).collect::<Vec<_>>();
|
||||
let rhs_spans = rhs.tts.iter().map(|t| t.span()).collect::<Vec<_>>();
|
||||
// rhs has holes ( `$id` and `$(...)` that need filled)
|
||||
let mut tts = match transcribe(cx, &named_matches, &rhs, transparency) {
|
||||
let mut tts = match transcribe(cx, &named_matches, &rhs, rhs_span, transparency) {
|
||||
Ok(tts) => tts,
|
||||
Err(mut err) => {
|
||||
err.emit();
|
||||
|
|
|
@ -29,8 +29,8 @@ fn visit_span(&mut self, span: &mut Span) {
|
|||
enum Frame<'a> {
|
||||
Delimited {
|
||||
tts: &'a [mbe::TokenTree],
|
||||
delim_token: token::DelimToken,
|
||||
idx: usize,
|
||||
delim_token: token::DelimToken,
|
||||
span: DelimSpan,
|
||||
},
|
||||
Sequence {
|
||||
|
@ -42,8 +42,8 @@ enum Frame<'a> {
|
|||
|
||||
impl<'a> Frame<'a> {
|
||||
/// Construct a new frame around the delimited set of tokens.
|
||||
fn new(tts: &'a [mbe::TokenTree]) -> Frame<'a> {
|
||||
Frame::Delimited { tts, delim_token: token::NoDelim, idx: 0, span: DelimSpan::dummy() }
|
||||
fn new(src: &'a mbe::Delimited, span: DelimSpan) -> Frame<'a> {
|
||||
Frame::Delimited { tts: &src.tts, idx: 0, delim_token: src.delim, span }
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -85,17 +85,18 @@ fn next(&mut self) -> Option<&'a mbe::TokenTree> {
|
|||
pub(super) fn transcribe<'a>(
|
||||
cx: &ExtCtxt<'a>,
|
||||
interp: &FxHashMap<MacroRulesNormalizedIdent, NamedMatch>,
|
||||
src: &[mbe::TokenTree],
|
||||
src: &mbe::Delimited,
|
||||
src_span: DelimSpan,
|
||||
transparency: Transparency,
|
||||
) -> PResult<'a, TokenStream> {
|
||||
// Nothing for us to transcribe...
|
||||
if src.is_empty() {
|
||||
if src.tts.is_empty() {
|
||||
return Ok(TokenStream::default());
|
||||
}
|
||||
|
||||
// We descend into the RHS (`src`), expanding things as we go. This stack contains the things
|
||||
// we have yet to expand/are still expanding. We start the stack off with the whole RHS.
|
||||
let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(&src)];
|
||||
let mut stack: SmallVec<[Frame<'_>; 1]> = smallvec![Frame::new(&src, src_span)];
|
||||
|
||||
// As we descend in the RHS, we will need to be able to match nested sequences of matchers.
|
||||
// `repeats` keeps track of where we are in matching at each level, with the last element being
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
use rustc_ast::{self as ast};
|
||||
use rustc_ast::{AstLike, AttrVec, Attribute};
|
||||
use rustc_errors::PResult;
|
||||
use rustc_span::{sym, Span, DUMMY_SP};
|
||||
use rustc_span::{sym, Span};
|
||||
|
||||
use std::convert::TryInto;
|
||||
use std::ops::Range;
|
||||
|
@ -400,24 +400,26 @@ fn make_token_stream(
|
|||
) -> AttrAnnotatedTokenStream {
|
||||
#[derive(Debug)]
|
||||
struct FrameData {
|
||||
open: Span,
|
||||
open_delim: DelimToken,
|
||||
// This is `None` for the first frame, `Some` for all others.
|
||||
open_delim_sp: Option<(DelimToken, Span)>,
|
||||
inner: Vec<(AttrAnnotatedTokenTree, Spacing)>,
|
||||
}
|
||||
let mut stack =
|
||||
vec![FrameData { open: DUMMY_SP, open_delim: DelimToken::NoDelim, inner: vec![] }];
|
||||
let mut stack = vec![FrameData { open_delim_sp: None, inner: vec![] }];
|
||||
let mut token_and_spacing = iter.next();
|
||||
while let Some((token, spacing)) = token_and_spacing {
|
||||
match token {
|
||||
FlatToken::Token(Token { kind: TokenKind::OpenDelim(delim), span }) => {
|
||||
stack.push(FrameData { open: span, open_delim: delim, inner: vec![] });
|
||||
stack.push(FrameData { open_delim_sp: Some((delim, span)), inner: vec![] });
|
||||
}
|
||||
FlatToken::Token(Token { kind: TokenKind::CloseDelim(delim), span }) => {
|
||||
// HACK: If we encounter a mismatched `None` delimiter at the top
|
||||
// level, just ignore it.
|
||||
if matches!(delim, DelimToken::NoDelim)
|
||||
&& (stack.len() == 1
|
||||
|| !matches!(stack.last_mut().unwrap().open_delim, DelimToken::NoDelim))
|
||||
|| !matches!(
|
||||
stack.last_mut().unwrap().open_delim_sp.unwrap().0,
|
||||
DelimToken::NoDelim
|
||||
))
|
||||
{
|
||||
token_and_spacing = iter.next();
|
||||
continue;
|
||||
|
@ -430,7 +432,7 @@ struct FrameData {
|
|||
// merge our current frame with the one above it. That is, transform
|
||||
// `[ { < first second } third ]` into `[ { first second } third ]`
|
||||
if !matches!(delim, DelimToken::NoDelim)
|
||||
&& matches!(frame_data.open_delim, DelimToken::NoDelim)
|
||||
&& matches!(frame_data.open_delim_sp.unwrap().0, DelimToken::NoDelim)
|
||||
{
|
||||
stack.last_mut().unwrap().inner.extend(frame_data.inner);
|
||||
// Process our closing delimiter again, this time at the previous
|
||||
|
@ -439,12 +441,13 @@ struct FrameData {
|
|||
continue;
|
||||
}
|
||||
|
||||
let (open_delim, open_sp) = frame_data.open_delim_sp.unwrap();
|
||||
assert_eq!(
|
||||
frame_data.open_delim, delim,
|
||||
open_delim, delim,
|
||||
"Mismatched open/close delims: open={:?} close={:?}",
|
||||
frame_data.open, span
|
||||
open_delim, span
|
||||
);
|
||||
let dspan = DelimSpan::from_pair(frame_data.open, span);
|
||||
let dspan = DelimSpan::from_pair(open_sp, span);
|
||||
let stream = AttrAnnotatedTokenStream::new(frame_data.inner);
|
||||
let delimited = AttrAnnotatedTokenTree::Delimited(dspan, delim, stream);
|
||||
stack
|
||||
|
@ -472,7 +475,7 @@ struct FrameData {
|
|||
// HACK: If we don't have a closing `None` delimiter for our last
|
||||
// frame, merge the frame with the top-level frame. That is,
|
||||
// turn `< first second` into `first second`
|
||||
if stack.len() == 2 && stack[1].open_delim == DelimToken::NoDelim {
|
||||
if stack.len() == 2 && stack[1].open_delim_sp.unwrap().0 == DelimToken::NoDelim {
|
||||
let temp_buf = stack.pop().unwrap();
|
||||
stack.last_mut().unwrap().inner.extend(temp_buf.inner);
|
||||
}
|
||||
|
|
|
@ -2043,7 +2043,8 @@ fn parse_closure_expr(&mut self, attrs: AttrVec) -> PResult<'a, P<Expr>> {
|
|||
self.sess.gated_spans.gate(sym::async_closure, span);
|
||||
}
|
||||
|
||||
if self.token.kind == TokenKind::Semi && self.token_cursor.frame.delim == DelimToken::Paren
|
||||
if self.token.kind == TokenKind::Semi
|
||||
&& matches!(self.token_cursor.frame.delim_sp, Some((DelimToken::Paren, _)))
|
||||
{
|
||||
// It is likely that the closure body is a block but where the
|
||||
// braces have been removed. We will recover and eat the next
|
||||
|
|
|
@ -244,14 +244,13 @@ struct TokenCursor {
|
|||
|
||||
#[derive(Clone)]
|
||||
struct TokenCursorFrame {
|
||||
delim: token::DelimToken,
|
||||
span: DelimSpan,
|
||||
delim_sp: Option<(DelimToken, DelimSpan)>,
|
||||
tree_cursor: tokenstream::Cursor,
|
||||
}
|
||||
|
||||
impl TokenCursorFrame {
|
||||
fn new(span: DelimSpan, delim: DelimToken, tts: TokenStream) -> Self {
|
||||
TokenCursorFrame { delim, span, tree_cursor: tts.into_trees() }
|
||||
fn new(delim_sp: Option<(DelimToken, DelimSpan)>, tts: TokenStream) -> Self {
|
||||
TokenCursorFrame { delim_sp, tree_cursor: tts.into_trees() }
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -266,7 +265,7 @@ fn inlined_next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
|
|||
loop {
|
||||
// FIXME: we currently don't return `NoDelim` open/close delims. To fix #67062 we will
|
||||
// need to, whereupon the `delim != DelimToken::NoDelim` conditions below can be
|
||||
// removed, as well as the loop.
|
||||
// removed.
|
||||
if let Some((tree, spacing)) = self.frame.tree_cursor.next_with_spacing_ref() {
|
||||
match tree {
|
||||
&TokenTree::Token(ref token) => match (desugar_doc_comments, token) {
|
||||
|
@ -277,7 +276,7 @@ fn inlined_next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
|
|||
},
|
||||
&TokenTree::Delimited(sp, delim, ref tts) => {
|
||||
// Set `open_delim` to true here because we deal with it immediately.
|
||||
let frame = TokenCursorFrame::new(sp, delim, tts.clone());
|
||||
let frame = TokenCursorFrame::new(Some((delim, sp)), tts.clone());
|
||||
self.stack.push(mem::replace(&mut self.frame, frame));
|
||||
if delim != DelimToken::NoDelim {
|
||||
return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
|
||||
|
@ -286,12 +285,11 @@ fn inlined_next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
|
|||
}
|
||||
};
|
||||
} else if let Some(frame) = self.stack.pop() {
|
||||
let delim = self.frame.delim;
|
||||
let span = self.frame.span;
|
||||
self.frame = frame;
|
||||
if delim != DelimToken::NoDelim {
|
||||
if let Some((delim, span)) = self.frame.delim_sp && delim != DelimToken::NoDelim {
|
||||
self.frame = frame;
|
||||
return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
|
||||
}
|
||||
self.frame = frame;
|
||||
// No close delimiter to return; continue on to the next iteration.
|
||||
} else {
|
||||
return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone);
|
||||
|
@ -330,8 +328,7 @@ fn desugar(&mut self, attr_style: AttrStyle, data: Symbol, span: Span) -> (Token
|
|||
self.stack.push(mem::replace(
|
||||
&mut self.frame,
|
||||
TokenCursorFrame::new(
|
||||
delim_span,
|
||||
token::NoDelim,
|
||||
None,
|
||||
if attr_style == AttrStyle::Inner {
|
||||
[TokenTree::token(token::Pound, span), TokenTree::token(token::Not, span), body]
|
||||
.iter()
|
||||
|
@ -431,10 +428,6 @@ pub fn new(
|
|||
desugar_doc_comments: bool,
|
||||
subparser_name: Option<&'static str>,
|
||||
) -> Self {
|
||||
// Note: because of the way `TokenCursor::inlined_next` is structured, the `span` and
|
||||
// `delim` arguments here are never used.
|
||||
let start_frame = TokenCursorFrame::new(DelimSpan::dummy(), token::NoDelim, tokens);
|
||||
|
||||
let mut parser = Parser {
|
||||
sess,
|
||||
token: Token::dummy(),
|
||||
|
@ -444,7 +437,7 @@ pub fn new(
|
|||
restrictions: Restrictions::empty(),
|
||||
expected_tokens: Vec::new(),
|
||||
token_cursor: TokenCursor {
|
||||
frame: start_frame,
|
||||
frame: TokenCursorFrame::new(None, tokens),
|
||||
stack: Vec::new(),
|
||||
num_next_calls: 0,
|
||||
desugar_doc_comments,
|
||||
|
@ -1025,7 +1018,7 @@ pub fn look_ahead<R>(&self, dist: usize, looker: impl FnOnce(&Token) -> R) -> R
|
|||
}
|
||||
|
||||
let frame = &self.token_cursor.frame;
|
||||
if frame.delim != DelimToken::NoDelim {
|
||||
if let Some((delim, span)) = frame.delim_sp && delim != DelimToken::NoDelim {
|
||||
let all_normal = (0..dist).all(|i| {
|
||||
let token = frame.tree_cursor.look_ahead(i);
|
||||
!matches!(token, Some(TokenTree::Delimited(_, DelimToken::NoDelim, _)))
|
||||
|
@ -1038,7 +1031,7 @@ pub fn look_ahead<R>(&self, dist: usize, looker: impl FnOnce(&Token) -> R) -> R
|
|||
looker(&Token::new(token::OpenDelim(*delim), dspan.open))
|
||||
}
|
||||
},
|
||||
None => looker(&Token::new(token::CloseDelim(frame.delim), frame.span.close)),
|
||||
None => looker(&Token::new(token::CloseDelim(delim), span.close)),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
@ -1198,8 +1191,7 @@ pub(crate) fn parse_token_tree(&mut self) -> TokenTree {
|
|||
// Grab the tokens from this frame.
|
||||
let frame = &self.token_cursor.frame;
|
||||
let stream = frame.tree_cursor.stream.clone();
|
||||
let span = frame.span;
|
||||
let delim = frame.delim;
|
||||
let (delim, span) = frame.delim_sp.unwrap();
|
||||
|
||||
// Advance the token cursor through the entire delimited
|
||||
// sequence. After getting the `OpenDelim` we are *within* the
|
||||
|
|
|
@ -164,25 +164,29 @@ fn parse_stmt_mac(&mut self, lo: Span, attrs: AttrVec, path: ast::Path) -> PResu
|
|||
let delim = args.delim();
|
||||
let hi = self.prev_token.span;
|
||||
|
||||
let style =
|
||||
if delim == token::Brace { MacStmtStyle::Braces } else { MacStmtStyle::NoBraces };
|
||||
let style = match delim {
|
||||
Some(token::Brace) => MacStmtStyle::Braces,
|
||||
Some(_) => MacStmtStyle::NoBraces,
|
||||
None => unreachable!(),
|
||||
};
|
||||
|
||||
let mac = MacCall { path, args, prior_type_ascription: self.last_type_ascription };
|
||||
|
||||
let kind =
|
||||
if (delim == token::Brace && self.token != token::Dot && self.token != token::Question)
|
||||
|| self.token == token::Semi
|
||||
|| self.token == token::Eof
|
||||
{
|
||||
StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None }))
|
||||
} else {
|
||||
// Since none of the above applied, this is an expression statement macro.
|
||||
let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac), AttrVec::new());
|
||||
let e = self.maybe_recover_from_bad_qpath(e, true)?;
|
||||
let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?;
|
||||
let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
|
||||
StmtKind::Expr(e)
|
||||
};
|
||||
let kind = if (style == MacStmtStyle::Braces
|
||||
&& self.token != token::Dot
|
||||
&& self.token != token::Question)
|
||||
|| self.token == token::Semi
|
||||
|| self.token == token::Eof
|
||||
{
|
||||
StmtKind::MacCall(P(MacCallStmt { mac, style, attrs, tokens: None }))
|
||||
} else {
|
||||
// Since none of the above applied, this is an expression statement macro.
|
||||
let e = self.mk_expr(lo.to(hi), ExprKind::MacCall(mac), AttrVec::new());
|
||||
let e = self.maybe_recover_from_bad_qpath(e, true)?;
|
||||
let e = self.parse_dot_or_call_expr_with(e, lo, attrs.into())?;
|
||||
let e = self.parse_assoc_expr_with(0, LhsExpr::AlreadyParsed(e))?;
|
||||
StmtKind::Expr(e)
|
||||
};
|
||||
Ok(self.mk_stmt(lo.to(hi), kind))
|
||||
}
|
||||
|
||||
|
|
|
@ -1325,7 +1325,7 @@ pub(crate) fn can_be_overflowed_expr(
|
|||
}
|
||||
ast::ExprKind::MacCall(ref mac) => {
|
||||
match (
|
||||
rustc_ast::ast::MacDelimiter::from_token(mac.args.delim()),
|
||||
rustc_ast::ast::MacDelimiter::from_token(mac.args.delim().unwrap()),
|
||||
context.config.overflow_delimited_expr(),
|
||||
) {
|
||||
(Some(ast::MacDelimiter::Bracket), true)
|
||||
|
|
|
@ -562,7 +562,7 @@ fn delim_token_to_str(
|
|||
("{ ", " }")
|
||||
}
|
||||
}
|
||||
DelimToken::NoDelim => ("", ""),
|
||||
DelimToken::NoDelim => unreachable!(),
|
||||
};
|
||||
if use_multiple_lines {
|
||||
let indent_str = shape.indent.to_string_with_newline(context.config);
|
||||
|
|
Loading…
Reference in a new issue