mirror of
https://github.com/rust-lang/rust
synced 2024-10-22 16:35:28 +00:00
Remove trivia tokens
This commit is contained in:
parent
8f24c2ec9d
commit
5326361fc0
|
@ -251,17 +251,6 @@ pub enum TokenKind {
|
|||
/// similarly to symbols in string literal tokens.
|
||||
DocComment(CommentKind, ast::AttrStyle, Symbol),
|
||||
|
||||
// Junk. These carry no data because we don't really care about the data
|
||||
// they *would* carry, and don't really want to allocate a new ident for
|
||||
// them. Instead, users could extract that from the associated span.
|
||||
/// Whitespace.
|
||||
Whitespace,
|
||||
/// A comment.
|
||||
Comment,
|
||||
Shebang(Symbol),
|
||||
/// A completely invalid token which should be skipped.
|
||||
Unknown(Symbol),
|
||||
|
||||
Eof,
|
||||
}
|
||||
|
||||
|
@ -331,7 +320,7 @@ pub fn new(kind: TokenKind, span: Span) -> Self {
|
|||
|
||||
/// Some token that will be thrown away later.
|
||||
pub fn dummy() -> Self {
|
||||
Token::new(TokenKind::Whitespace, DUMMY_SP)
|
||||
Token::new(TokenKind::Question, DUMMY_SP)
|
||||
}
|
||||
|
||||
/// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary.
|
||||
|
@ -360,7 +349,7 @@ pub fn uninterpolated_span(&self) -> Span {
|
|||
pub fn is_op(&self) -> bool {
|
||||
match self.kind {
|
||||
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
|
||||
| Lifetime(..) | Interpolated(..) | Whitespace | Comment | Shebang(..) | Eof => false,
|
||||
| Lifetime(..) | Interpolated(..) | Eof => false,
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
@ -676,8 +665,7 @@ pub fn glue(&self, joint: &Token) -> Option<Token> {
|
|||
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
|
||||
| DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar
|
||||
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..)
|
||||
| Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment
|
||||
| Shebang(..) | Unknown(..) | Eof => return None,
|
||||
| Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None,
|
||||
};
|
||||
|
||||
Some(Token::new(kind, self.span.to(joint.span)))
|
||||
|
|
|
@ -289,10 +289,6 @@ fn token_kind_to_string_ext(tok: &TokenKind, convert_dollar_crate: Option<Span>)
|
|||
doc_comment_to_string(comment_kind, attr_style, data)
|
||||
}
|
||||
token::Eof => "<eof>".to_string(),
|
||||
token::Whitespace => " ".to_string(),
|
||||
token::Comment => "/* */".to_string(),
|
||||
token::Shebang(s) => format!("/* shebang: {}*/", s),
|
||||
token::Unknown(s) => s.to_string(),
|
||||
|
||||
token::Interpolated(ref nt) => nonterminal_to_string(nt),
|
||||
}
|
||||
|
|
|
@ -189,7 +189,7 @@ macro_rules! op {
|
|||
}
|
||||
|
||||
OpenDelim(..) | CloseDelim(..) => unreachable!(),
|
||||
Whitespace | Comment | Shebang(..) | Unknown(..) | Eof => unreachable!(),
|
||||
Eof => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -303,7 +303,7 @@
|
|||
// However, we should first remove compound tokens like `<<` from `rustc_lexer`, and then add
|
||||
// fancier error recovery to it, as there will be less overall work to do this way.
|
||||
const ASCII_ARRAY: &[(char, &str, Option<token::TokenKind>)] = &[
|
||||
(' ', "Space", Some(token::Whitespace)),
|
||||
(' ', "Space", None),
|
||||
('_', "Underscore", Some(token::Ident(kw::Underscore, false))),
|
||||
('-', "Minus/Hyphen", Some(token::BinOp(token::Minus))),
|
||||
(',', "Comma", Some(token::Comma)),
|
||||
|
|
|
@ -348,9 +348,6 @@ fn semantic_tree(tree: &TokenTree) -> bool {
|
|||
| token::CloseDelim(DelimToken::NoDelim)
|
||||
// The pretty printer collapses many semicolons into one.
|
||||
| token::Semi
|
||||
// The pretty printer collapses whitespace arbitrarily and can
|
||||
// introduce whitespace from `NoDelim`.
|
||||
| token::Whitespace
|
||||
// The pretty printer can turn `$crate` into `::crate_name`
|
||||
| token::ModSep = token.kind {
|
||||
return false;
|
||||
|
@ -506,8 +503,6 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool {
|
|||
| (&Pound, &Pound)
|
||||
| (&Dollar, &Dollar)
|
||||
| (&Question, &Question)
|
||||
| (&Whitespace, &Whitespace)
|
||||
| (&Comment, &Comment)
|
||||
| (&Eof, &Eof) => true,
|
||||
|
||||
(&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b,
|
||||
|
@ -516,8 +511,6 @@ fn token_probably_equal_for_proc_macro(first: &Token, other: &Token) -> bool {
|
|||
|
||||
(&DocComment(a1, a2, a3), &DocComment(b1, b2, b3)) => a1 == b1 && a2 == b2 && a3 == b3,
|
||||
|
||||
(&Shebang(a), &Shebang(b)) => a == b,
|
||||
|
||||
(&Literal(a), &Literal(b)) => a == b,
|
||||
|
||||
(&Lifetime(a), &Lifetime(b)) => a == b,
|
||||
|
|
Loading…
Reference in a new issue