pub use BinOpToken::*;
-pub use DelimToken::*;
pub use LitKind::*;
pub use Nonterminal::*;
pub use TokenKind::*;
Shr,
}
-/// A delimiter token.
-#[derive(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Debug, Copy)]
-#[derive(HashStable_Generic)]
-pub enum DelimToken {
- /// A round parenthesis (i.e., `(` or `)`).
- Paren,
- /// A square bracket (i.e., `[` or `]`).
- Bracket,
- /// A curly brace (i.e., `{` or `}`).
+/// Describes how a sequence of token trees is delimited.
+/// Cannot use `proc_macro::Delimiter` directly because this
+/// structure should implement some additional traits.
+/// The `None` variant is also renamed to `Invisible` to be
+/// less confusing and better convey the semantics.
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+#[derive(Encodable, Decodable, Hash, HashStable_Generic)]
+pub enum Delimiter {
+ /// `( ... )`
+ Parenthesis,
+ /// `{ ... }`
Brace,
- /// An empty delimiter.
- NoDelim,
+ /// `[ ... ]`
+ Bracket,
+ /// `Ø ... Ø`
+ /// An invisible delimiter, that may, for example, appear around tokens coming from a
+ /// "macro variable" `$var`. It is important to preserve operator priorities in cases like
+ /// `$var * 3` where `$var` is `1 + 2`.
+ /// Invisible delimiters might not survive roundtrip of a token stream through a string.
+ Invisible,
}
#[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
/// Used by proc macros for representing lifetimes, not generated by lexer right now.
SingleQuote,
/// An opening delimiter (e.g., `{`).
- OpenDelim(DelimToken),
+ OpenDelim(Delimiter),
/// A closing delimiter (e.g., `}`).
- CloseDelim(DelimToken),
+ CloseDelim(Delimiter),
/* Literals */
Literal(Lit),
/// treat regular and interpolated lifetime identifiers in the same way.
Lifetime(Symbol),
+ /// An embedded AST node, as produced by a macro. This only exists for
+ /// historical reasons. We'd like to get rid of it, for multiple reasons.
+ /// - It's conceptually very strange. Saying a token can contain an AST
+ /// node is like saying, in natural language, that a word can contain a
+ /// sentence.
+ /// - It requires special handling in a bunch of places in the parser.
+ /// - It prevents `Token` from implementing `Copy`.
+ /// It adds complexity and likely slows things down. Please don't add new
+ /// occurrences of this token kind!
Interpolated(Lrc<Nonterminal>),
/// A doc comment token.
match self.uninterpolate().kind {
Ident(name, is_raw) =>
ident_can_begin_type(name, self.span, is_raw), // type name or keyword
- OpenDelim(Paren) | // tuple
- OpenDelim(Bracket) | // array
+ OpenDelim(Delimiter::Parenthesis) | // tuple
+ OpenDelim(Delimiter::Bracket) | // array
Not | // never
BinOp(Star) | // raw pointer
BinOp(And) | // reference
/// Returns `true` if the token can appear at the start of a const param.
pub fn can_begin_const_arg(&self) -> bool {
match self.kind {
- OpenDelim(Brace) => true,
+ OpenDelim(Delimiter::Brace) => true,
Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)),
_ => self.can_begin_literal_maybe_minus(),
}
|| self.is_lifetime()
|| self.is_keyword(kw::For)
|| self == &Question
- || self == &OpenDelim(Paren)
+ || self == &OpenDelim(Delimiter::Parenthesis)
}
/// Returns `true` if the token is any literal.
}
/// Returns an identifier if this token is an identifier.
+ #[inline]
pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> {
- let token = self.uninterpolate();
- match token.kind {
- Ident(name, is_raw) => Some((Ident::new(name, token.span), is_raw)),
+ // We avoid using `Token::uninterpolate` here because it's slow.
+ match &self.kind {
+ &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
+ Interpolated(nt) => match **nt {
+ NtIdent(ident, is_raw) => Some((ident, is_raw)),
+ _ => None,
+ },
_ => None,
}
}
/// Returns a lifetime identifier if this token is a lifetime.
+ #[inline]
pub fn lifetime(&self) -> Option<Ident> {
- let token = self.uninterpolate();
- match token.kind {
- Lifetime(name) => Some(Ident::new(name, token.span)),
+ // We avoid using `Token::uninterpolate` here because it's slow.
+ match &self.kind {
+ &Lifetime(name) => Some(Ident::new(name, self.span)),
+ Interpolated(nt) => match **nt {
+ NtLifetime(ident) => Some(ident),
+ _ => None,
+ },
_ => None,
}
}
/// (which happens while parsing the result of macro expansion)?
pub fn is_whole_expr(&self) -> bool {
if let Interpolated(ref nt) = self.kind
- && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt
+ && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt
{
return true;
}
pub enum Nonterminal {
NtItem(P<ast::Item>),
NtBlock(P<ast::Block>),
- NtStmt(ast::Stmt),
+ NtStmt(P<ast::Stmt>),
NtPat(P<ast::Pat>),
NtExpr(P<ast::Expr>),
NtTy(P<ast::Ty>),
NtLiteral(P<ast::Expr>),
/// Stuff inside brackets for attributes
NtMeta(P<ast::AttrItem>),
- NtPath(ast::Path),
- NtVis(ast::Visibility),
+ NtPath(P<ast::Path>),
+ NtVis(P<ast::Visibility>),
}
// `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(Nonterminal, 48);
+rustc_data_structures::static_assert_size!(Nonterminal, 16);
#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
pub enum NonterminalKind {
Edition::Edition2015 | Edition::Edition2018 => {
NonterminalKind::PatParam { inferred: true }
}
- Edition::Edition2021 => NonterminalKind::PatWithOr,
+ Edition::Edition2021 | Edition::Edition2024 => NonterminalKind::PatWithOr,
},
sym::pat_param => NonterminalKind::PatParam { inferred: false },
sym::expr => NonterminalKind::Expr,
projects / rustc.git / blobdiff