]>
Commit | Line | Data |
---|---|---|
9fa01778 | 1 | pub use BinOpToken::*; |
dc9dc135 | 2 | pub use LitKind::*; |
dfeec247 | 3 | pub use Nonterminal::*; |
dc9dc135 | 4 | pub use TokenKind::*; |
9fa01778 | 5 | |
e74abb32 | 6 | use crate::ast; |
9fa01778 | 7 | use crate::ptr::P; |
9fa01778 | 8 | |
60c5eb7d | 9 | use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; |
9fa01778 | 10 | use rustc_data_structures::sync::Lrc; |
60c5eb7d | 11 | use rustc_macros::HashStable_Generic; |
f035d41b | 12 | use rustc_span::symbol::{kw, sym}; |
f9f354fc | 13 | use rustc_span::symbol::{Ident, Symbol}; |
6a06907d | 14 | use rustc_span::{self, edition::Edition, Span, DUMMY_SP}; |
74b04a01 XL |
15 | use std::borrow::Cow; |
16 | use std::{fmt, mem}; | |
1a4d82fc | 17 | |
3dfed10e XL |
18 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
19 | pub enum CommentKind { | |
20 | Line, | |
21 | Block, | |
22 | } | |
23 | ||
24 | #[derive(Clone, PartialEq, Encodable, Decodable, Hash, Debug, Copy)] | |
60c5eb7d | 25 | #[derive(HashStable_Generic)] |
1a4d82fc JJ |
26 | pub enum BinOpToken { |
27 | Plus, | |
28 | Minus, | |
29 | Star, | |
30 | Slash, | |
31 | Percent, | |
32 | Caret, | |
33 | And, | |
34 | Or, | |
35 | Shl, | |
36 | Shr, | |
37 | } | |
38 | ||
04454e1e FG |
39 | /// Describes how a sequence of token trees is delimited. |
40 | /// Cannot use `proc_macro::Delimiter` directly because this | |
41 | /// structure should implement some additional traits. | |
42 | /// The `None` variant is also renamed to `Invisible` to be | |
43 | /// less confusing and better convey the semantics. | |
44 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] | |
45 | #[derive(Encodable, Decodable, Hash, HashStable_Generic)] | |
46 | pub enum Delimiter { | |
47 | /// `( ... )` | |
48 | Parenthesis, | |
49 | /// `{ ... }` | |
1a4d82fc | 50 | Brace, |
04454e1e FG |
51 | /// `[ ... ]` |
52 | Bracket, | |
53 | /// `Ø ... Ø` | |
54 | /// An invisible delimiter, that may, for example, appear around tokens coming from a | |
55 | /// "macro variable" `$var`. It is important to preserve operator priorities in cases like | |
56 | /// `$var * 3` where `$var` is `1 + 2`. | |
57 | /// Invisible delimiters might not survive roundtrip of a token stream through a string. | |
58 | Invisible, | |
1a4d82fc JJ |
59 | } |
60 | ||
3dfed10e | 61 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
62 | pub enum LitKind { |
63 | Bool, // AST only, must never appear in a `Token` | |
64 | Byte, | |
65 | Char, | |
66 | Integer, | |
67 | Float, | |
68 | Str, | |
5e7ed085 | 69 | StrRaw(u8), // raw string delimited by `n` hash symbols |
dc9dc135 | 70 | ByteStr, |
5e7ed085 | 71 | ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols |
dc9dc135 | 72 | Err, |
1a4d82fc JJ |
73 | } |
74 | ||
dc9dc135 | 75 | /// A literal token. |
3dfed10e | 76 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
77 | pub struct Lit { |
78 | pub kind: LitKind, | |
79 | pub symbol: Symbol, | |
80 | pub suffix: Option<Symbol>, | |
81 | } | |
48663c56 | 82 | |
416331ca XL |
83 | impl fmt::Display for Lit { |
84 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
85 | let Lit { kind, symbol, suffix } = *self; | |
86 | match kind { | |
dfeec247 XL |
87 | Byte => write!(f, "b'{}'", symbol)?, |
88 | Char => write!(f, "'{}'", symbol)?, | |
89 | Str => write!(f, "\"{}\"", symbol)?, | |
90 | StrRaw(n) => write!( | |
91 | f, | |
92 | "r{delim}\"{string}\"{delim}", | |
93 | delim = "#".repeat(n as usize), | |
94 | string = symbol | |
95 | )?, | |
96 | ByteStr => write!(f, "b\"{}\"", symbol)?, | |
97 | ByteStrRaw(n) => write!( | |
98 | f, | |
99 | "br{delim}\"{string}\"{delim}", | |
100 | delim = "#".repeat(n as usize), | |
101 | string = symbol | |
102 | )?, | |
103 | Integer | Float | Bool | Err => write!(f, "{}", symbol)?, | |
416331ca XL |
104 | } |
105 | ||
106 | if let Some(suffix) = suffix { | |
107 | write!(f, "{}", suffix)?; | |
108 | } | |
109 | ||
110 | Ok(()) | |
111 | } | |
112 | } | |
113 | ||
dc9dc135 XL |
114 | impl LitKind { |
115 | /// An English article for the literal token kind. | |
60c5eb7d | 116 | pub fn article(self) -> &'static str { |
dc9dc135 XL |
117 | match self { |
118 | Integer | Err => "an", | |
119 | _ => "a", | |
1a4d82fc JJ |
120 | } |
121 | } | |
83c7162d | 122 | |
60c5eb7d | 123 | pub fn descr(self) -> &'static str { |
dc9dc135 XL |
124 | match self { |
125 | Bool => panic!("literal token contains `Lit::Bool`"), | |
126 | Byte => "byte", | |
127 | Char => "char", | |
128 | Integer => "integer", | |
129 | Float => "float", | |
130 | Str | StrRaw(..) => "string", | |
131 | ByteStr | ByteStrRaw(..) => "byte string", | |
132 | Err => "error", | |
133 | } | |
134 | } | |
135 | ||
923072b8 | 136 | pub(crate) fn may_have_suffix(self) -> bool { |
5869c6ff | 137 | matches!(self, Integer | Float | Err) |
48663c56 | 138 | } |
dc9dc135 | 139 | } |
48663c56 | 140 | |
dc9dc135 XL |
141 | impl Lit { |
142 | pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit { | |
143 | Lit { kind, symbol, suffix } | |
83c7162d | 144 | } |
1a4d82fc JJ |
145 | } |
146 | ||
f9f354fc | 147 | pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool { |
dc9dc135 | 148 | let ident_token = Token::new(Ident(name, is_raw), span); |
32a655c1 | 149 | |
dfeec247 XL |
150 | !ident_token.is_reserved_ident() |
151 | || ident_token.is_path_segment_keyword() | |
74b04a01 XL |
152 | || [ |
153 | kw::Async, | |
154 | kw::Do, | |
155 | kw::Box, | |
156 | kw::Break, | |
29967ef6 | 157 | kw::Const, |
74b04a01 XL |
158 | kw::Continue, |
159 | kw::False, | |
160 | kw::For, | |
161 | kw::If, | |
162 | kw::Let, | |
163 | kw::Loop, | |
164 | kw::Match, | |
165 | kw::Move, | |
166 | kw::Return, | |
167 | kw::True, | |
1b1a35ee | 168 | kw::Try, |
74b04a01 XL |
169 | kw::Unsafe, |
170 | kw::While, | |
171 | kw::Yield, | |
172 | kw::Static, | |
173 | ] | |
174 | .contains(&name) | |
32a655c1 SL |
175 | } |
176 | ||
f9f354fc | 177 | fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool { |
dc9dc135 | 178 | let ident_token = Token::new(Ident(name, is_raw), span); |
cc61c64b | 179 | |
dfeec247 XL |
180 | !ident_token.is_reserved_ident() |
181 | || ident_token.is_path_segment_keyword() | |
182 | || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn] | |
183 | .contains(&name) | |
cc61c64b XL |
184 | } |
185 | ||
3dfed10e | 186 | #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 | 187 | pub enum TokenKind { |
223e47cc | 188 | /* Expression-operator symbols. */ |
1a4d82fc JJ |
189 | Eq, |
190 | Lt, | |
191 | Le, | |
192 | EqEq, | |
193 | Ne, | |
194 | Ge, | |
195 | Gt, | |
196 | AndAnd, | |
197 | OrOr, | |
198 | Not, | |
199 | Tilde, | |
200 | BinOp(BinOpToken), | |
201 | BinOpEq(BinOpToken), | |
223e47cc LB |
202 | |
203 | /* Structural symbols */ | |
1a4d82fc JJ |
204 | At, |
205 | Dot, | |
206 | DotDot, | |
207 | DotDotDot, | |
ea8adc8c | 208 | DotDotEq, |
1a4d82fc JJ |
209 | Comma, |
210 | Semi, | |
211 | Colon, | |
212 | ModSep, | |
213 | RArrow, | |
214 | LArrow, | |
215 | FatArrow, | |
216 | Pound, | |
217 | Dollar, | |
218 | Question, | |
94b46f34 XL |
219 | /// Used by proc macros for representing lifetimes, not generated by lexer right now. |
220 | SingleQuote, | |
9fa01778 | 221 | /// An opening delimiter (e.g., `{`). |
04454e1e | 222 | OpenDelim(Delimiter), |
9fa01778 | 223 | /// A closing delimiter (e.g., `}`). |
04454e1e | 224 | CloseDelim(Delimiter), |
223e47cc LB |
225 | |
226 | /* Literals */ | |
dc9dc135 | 227 | Literal(Lit), |
223e47cc | 228 | |
74b04a01 XL |
229 | /// Identifier token. |
230 | /// Do not forget about `NtIdent` when you want to match on identifiers. | |
231 | /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to | |
232 | /// treat regular and interpolated identifiers in the same way. | |
f9f354fc | 233 | Ident(Symbol, /* is_raw */ bool), |
74b04a01 XL |
234 | /// Lifetime identifier token. |
235 | /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. | |
236 | /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to | |
237 | /// treat regular and interpolated lifetime identifiers in the same way. | |
f9f354fc | 238 | Lifetime(Symbol), |
223e47cc | 239 | |
04454e1e FG |
240 | /// An embedded AST node, as produced by a macro. This only exists for |
241 | /// historical reasons. We'd like to get rid of it, for multiple reasons. | |
242 | /// - It's conceptually very strange. Saying a token can contain an AST | |
243 | /// node is like saying, in natural language, that a word can contain a | |
244 | /// sentence. | |
245 | /// - It requires special handling in a bunch of places in the parser. | |
246 | /// - It prevents `Token` from implementing `Copy`. | |
247 | /// It adds complexity and likely slows things down. Please don't add new | |
248 | /// occurrences of this token kind! | |
9fa01778 XL |
249 | Interpolated(Lrc<Nonterminal>), |
250 | ||
3dfed10e XL |
251 | /// A doc comment token. |
252 | /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc) | |
253 | /// similarly to symbols in string literal tokens. | |
254 | DocComment(CommentKind, ast::AttrStyle, Symbol), | |
1a4d82fc | 255 | |
1a4d82fc JJ |
256 | Eof, |
257 | } | |
258 | ||
dc9dc135 | 259 | // `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. |
6a06907d | 260 | #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] |
60c5eb7d | 261 | rustc_data_structures::static_assert_size!(TokenKind, 16); |
dc9dc135 | 262 | |
3dfed10e | 263 | #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
264 | pub struct Token { |
265 | pub kind: TokenKind, | |
266 | pub span: Span, | |
267 | } | |
268 | ||
269 | impl TokenKind { | |
270 | pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind { | |
271 | Literal(Lit::new(kind, symbol, suffix)) | |
272 | } | |
273 | ||
74b04a01 XL |
274 | // An approximation to proc-macro-style single-character operators used by rustc parser. |
275 | // If the operator token can be broken into two tokens, the first of which is single-character, | |
276 | // then this function performs that operation, otherwise it returns `None`. | |
277 | pub fn break_two_token_op(&self) -> Option<(TokenKind, TokenKind)> { | |
278 | Some(match *self { | |
279 | Le => (Lt, Eq), | |
280 | EqEq => (Eq, Eq), | |
281 | Ne => (Not, Eq), | |
282 | Ge => (Gt, Eq), | |
283 | AndAnd => (BinOp(And), BinOp(And)), | |
284 | OrOr => (BinOp(Or), BinOp(Or)), | |
285 | BinOp(Shl) => (Lt, Lt), | |
286 | BinOp(Shr) => (Gt, Gt), | |
287 | BinOpEq(Plus) => (BinOp(Plus), Eq), | |
288 | BinOpEq(Minus) => (BinOp(Minus), Eq), | |
289 | BinOpEq(Star) => (BinOp(Star), Eq), | |
290 | BinOpEq(Slash) => (BinOp(Slash), Eq), | |
291 | BinOpEq(Percent) => (BinOp(Percent), Eq), | |
292 | BinOpEq(Caret) => (BinOp(Caret), Eq), | |
293 | BinOpEq(And) => (BinOp(And), Eq), | |
294 | BinOpEq(Or) => (BinOp(Or), Eq), | |
295 | BinOpEq(Shl) => (Lt, Le), | |
296 | BinOpEq(Shr) => (Gt, Ge), | |
297 | DotDot => (Dot, Dot), | |
298 | DotDotDot => (Dot, DotDot), | |
299 | ModSep => (Colon, Colon), | |
300 | RArrow => (BinOp(Minus), Gt), | |
301 | LArrow => (Lt, BinOp(Minus)), | |
302 | FatArrow => (Eq, Gt), | |
303 | _ => return None, | |
304 | }) | |
305 | } | |
306 | ||
dc9dc135 XL |
307 | /// Returns tokens that are likely to be typed accidentally instead of the current token. |
308 | /// Enables better error recovery when the wrong token is found. | |
60c5eb7d | 309 | pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> { |
dc9dc135 XL |
310 | match *self { |
311 | Comma => Some(vec![Dot, Lt, Semi]), | |
312 | Semi => Some(vec![Colon, Comma]), | |
c295e0f8 | 313 | FatArrow => Some(vec![Eq, RArrow]), |
dfeec247 | 314 | _ => None, |
dc9dc135 XL |
315 | } |
316 | } | |
29967ef6 XL |
317 | |
318 | pub fn should_end_const_arg(&self) -> bool { | |
5869c6ff | 319 | matches!(self, Gt | Ge | BinOp(Shr) | BinOpEq(Shr)) |
29967ef6 | 320 | } |
dc9dc135 | 321 | } |
0731742a | 322 | |
1a4d82fc | 323 | impl Token { |
e74abb32 | 324 | pub fn new(kind: TokenKind, span: Span) -> Self { |
dc9dc135 XL |
325 | Token { kind, span } |
326 | } | |
327 | ||
328 | /// Some token that will be thrown away later. | |
60c5eb7d | 329 | pub fn dummy() -> Self { |
1b1a35ee | 330 | Token::new(TokenKind::Question, DUMMY_SP) |
dc9dc135 XL |
331 | } |
332 | ||
f9f354fc XL |
333 | /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary. |
334 | pub fn from_ast_ident(ident: Ident) -> Self { | |
dc9dc135 XL |
335 | Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) |
336 | } | |
337 | ||
338 | /// Return this token by value and leave a dummy token in its place. | |
e74abb32 | 339 | pub fn take(&mut self) -> Self { |
dc9dc135 XL |
340 | mem::replace(self, Token::dummy()) |
341 | } | |
342 | ||
74b04a01 XL |
343 | /// For interpolated tokens, returns a span of the fragment to which the interpolated |
344 | /// token refers. For all other tokens this is just a regular span. | |
345 | /// It is particularly important to use this for identifiers and lifetimes | |
346 | /// for which spans affect name resolution and edition checks. | |
347 | /// Note that keywords are also identifiers, so they should use this | |
348 | /// if they keep spans or perform edition checks. | |
349 | pub fn uninterpolated_span(&self) -> Span { | |
350 | match &self.kind { | |
351 | Interpolated(nt) => nt.span(), | |
352 | _ => self.span, | |
353 | } | |
354 | } | |
355 | ||
60c5eb7d | 356 | pub fn is_op(&self) -> bool { |
5869c6ff XL |
357 | !matches!( |
358 | self.kind, | |
359 | OpenDelim(..) | |
360 | | CloseDelim(..) | |
361 | | Literal(..) | |
362 | | DocComment(..) | |
363 | | Ident(..) | |
364 | | Lifetime(..) | |
365 | | Interpolated(..) | |
366 | | Eof | |
367 | ) | |
041b39d2 XL |
368 | } |
369 | ||
60c5eb7d | 370 | pub fn is_like_plus(&self) -> bool { |
5869c6ff | 371 | matches!(self.kind, BinOp(Plus) | BinOpEq(Plus)) |
bd371182 AL |
372 | } |
373 | ||
1a4d82fc | 374 | /// Returns `true` if the token can appear at the start of an expression. |
e74abb32 | 375 | pub fn can_begin_expr(&self) -> bool { |
74b04a01 | 376 | match self.uninterpolate().kind { |
dc9dc135 XL |
377 | Ident(name, is_raw) => |
378 | ident_can_begin_expr(name, self.span, is_raw), // value name or keyword | |
ea8adc8c XL |
379 | OpenDelim(..) | // tuple, array or block |
380 | Literal(..) | // literal | |
381 | Not | // operator not | |
382 | BinOp(Minus) | // unary minus | |
383 | BinOp(Star) | // dereference | |
384 | BinOp(Or) | OrOr | // closure | |
385 | BinOp(And) | // reference | |
386 | AndAnd | // double reference | |
abe05a73 | 387 | // DotDotDot is no longer supported, but we need some way to display the error |
ea8adc8c | 388 | DotDot | DotDotDot | DotDotEq | // range notation |
ea8adc8c XL |
389 | Lt | BinOp(Shl) | // associated path |
390 | ModSep | // global path | |
83c7162d | 391 | Lifetime(..) | // labeled loop |
ea8adc8c | 392 | Pound => true, // expression attributes |
5869c6ff | 393 | Interpolated(ref nt) => matches!(**nt, NtLiteral(..) | |
94b46f34 XL |
394 | NtExpr(..) | |
395 | NtBlock(..) | | |
5869c6ff | 396 | NtPath(..)), |
c30ab7b3 | 397 | _ => false, |
1a4d82fc JJ |
398 | } |
399 | } | |
223e47cc | 400 | |
f2b60f7d FG |
401 | /// Returns `true` if the token can appear at the start of an pattern. |
402 | /// | |
403 | /// Shamelessly borrowed from `can_begin_expr`, only used for diagnostics right now. | |
404 | pub fn can_begin_pattern(&self) -> bool { | |
405 | match self.uninterpolate().kind { | |
406 | Ident(name, is_raw) => | |
407 | ident_can_begin_expr(name, self.span, is_raw), // value name or keyword | |
408 | | OpenDelim(Delimiter::Bracket | Delimiter::Parenthesis) // tuple or array | |
409 | | Literal(..) // literal | |
410 | | BinOp(Minus) // unary minus | |
411 | | BinOp(And) // reference | |
412 | | AndAnd // double reference | |
413 | // DotDotDot is no longer supported | |
414 | | DotDot | DotDotDot | DotDotEq // ranges | |
415 | | Lt | BinOp(Shl) // associated path | |
416 | | ModSep => true, // global path | |
417 | Interpolated(ref nt) => matches!(**nt, NtLiteral(..) | | |
418 | NtPat(..) | | |
419 | NtBlock(..) | | |
420 | NtPath(..)), | |
421 | _ => false, | |
422 | } | |
423 | } | |
424 | ||
32a655c1 | 425 | /// Returns `true` if the token can appear at the start of a type. |
e74abb32 | 426 | pub fn can_begin_type(&self) -> bool { |
74b04a01 | 427 | match self.uninterpolate().kind { |
dc9dc135 XL |
428 | Ident(name, is_raw) => |
429 | ident_can_begin_type(name, self.span, is_raw), // type name or keyword | |
04454e1e FG |
430 | OpenDelim(Delimiter::Parenthesis) | // tuple |
431 | OpenDelim(Delimiter::Bracket) | // array | |
7cac9316 XL |
432 | Not | // never |
433 | BinOp(Star) | // raw pointer | |
434 | BinOp(And) | // reference | |
435 | AndAnd | // double reference | |
436 | Question | // maybe bound in trait object | |
437 | Lifetime(..) | // lifetime bound in trait object | |
438 | Lt | BinOp(Shl) | // associated path | |
32a655c1 | 439 | ModSep => true, // global path |
5869c6ff | 440 | Interpolated(ref nt) => matches!(**nt, NtTy(..) | NtPath(..)), |
32a655c1 SL |
441 | _ => false, |
442 | } | |
443 | } | |
444 | ||
9fa01778 | 445 | /// Returns `true` if the token can appear at the start of a const param. |
60c5eb7d | 446 | pub fn can_begin_const_arg(&self) -> bool { |
dc9dc135 | 447 | match self.kind { |
04454e1e | 448 | OpenDelim(Delimiter::Brace) => true, |
5869c6ff | 449 | Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)), |
74b04a01 | 450 | _ => self.can_begin_literal_maybe_minus(), |
9fa01778 XL |
451 | } |
452 | } | |
453 | ||
abe05a73 | 454 | /// Returns `true` if the token can appear at the start of a generic bound. |
60c5eb7d | 455 | pub fn can_begin_bound(&self) -> bool { |
dfeec247 XL |
456 | self.is_path_start() |
457 | || self.is_lifetime() | |
458 | || self.is_keyword(kw::For) | |
459 | || self == &Question | |
04454e1e | 460 | || self == &OpenDelim(Delimiter::Parenthesis) |
abe05a73 XL |
461 | } |
462 | ||
f2b60f7d FG |
463 | /// Returns `true` if the token can appear at the start of an item. |
464 | pub fn can_begin_item(&self) -> bool { | |
465 | match self.kind { | |
466 | Ident(name, _) => [ | |
467 | kw::Fn, | |
468 | kw::Use, | |
469 | kw::Struct, | |
470 | kw::Enum, | |
471 | kw::Pub, | |
472 | kw::Trait, | |
473 | kw::Extern, | |
474 | kw::Impl, | |
475 | kw::Unsafe, | |
476 | kw::Const, | |
477 | kw::Static, | |
478 | kw::Union, | |
479 | kw::Macro, | |
480 | kw::Mod, | |
481 | kw::Type, | |
482 | ] | |
483 | .contains(&name), | |
484 | _ => false, | |
485 | } | |
486 | } | |
487 | ||
fc512014 | 488 | /// Returns `true` if the token is any literal. |
e74abb32 | 489 | pub fn is_lit(&self) -> bool { |
5869c6ff | 490 | matches!(self.kind, Literal(..)) |
1a4d82fc | 491 | } |
223e47cc | 492 | |
9fa01778 | 493 | /// Returns `true` if the token is any literal, a minus (which can prefix a literal, |
94b46f34 | 494 | /// for example a '-42', or one of the boolean idents). |
74b04a01 XL |
495 | /// |
496 | /// In other words, would this token be a valid start of `parse_literal_maybe_minus`? | |
497 | /// | |
498 | /// Keep this in sync with and `Lit::from_token`, excluding unary negation. | |
499 | pub fn can_begin_literal_maybe_minus(&self) -> bool { | |
500 | match self.uninterpolate().kind { | |
e1599b0c XL |
501 | Literal(..) | BinOp(Minus) => true, |
502 | Ident(name, false) if name.is_bool_lit() => true, | |
74b04a01 XL |
503 | Interpolated(ref nt) => match &**nt { |
504 | NtLiteral(_) => true, | |
505 | NtExpr(e) => match &e.kind { | |
506 | ast::ExprKind::Lit(_) => true, | |
507 | ast::ExprKind::Unary(ast::UnOp::Neg, e) => { | |
508 | matches!(&e.kind, ast::ExprKind::Lit(_)) | |
509 | } | |
510 | _ => false, | |
511 | }, | |
dfeec247 | 512 | _ => false, |
b7449926 | 513 | }, |
dfeec247 | 514 | _ => false, |
94b46f34 XL |
515 | } |
516 | } | |
517 | ||
74b04a01 XL |
518 | // A convenience function for matching on identifiers during parsing. |
519 | // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token | |
520 | // into the regular identifier or lifetime token it refers to, | |
521 | // otherwise returns the original token. | |
522 | pub fn uninterpolate(&self) -> Cow<'_, Token> { | |
523 | match &self.kind { | |
524 | Interpolated(nt) => match **nt { | |
525 | NtIdent(ident, is_raw) => { | |
526 | Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)) | |
527 | } | |
528 | NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), | |
529 | _ => Cow::Borrowed(self), | |
530 | }, | |
531 | _ => Cow::Borrowed(self), | |
532 | } | |
533 | } | |
534 | ||
83c7162d | 535 | /// Returns an identifier if this token is an identifier. |
04454e1e | 536 | #[inline] |
f9f354fc | 537 | pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> { |
04454e1e FG |
538 | // We avoid using `Token::uninterpolate` here because it's slow. |
539 | match &self.kind { | |
540 | &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), | |
541 | Interpolated(nt) => match **nt { | |
542 | NtIdent(ident, is_raw) => Some((ident, is_raw)), | |
543 | _ => None, | |
544 | }, | |
83c7162d XL |
545 | _ => None, |
546 | } | |
547 | } | |
dc9dc135 | 548 | |
83c7162d | 549 | /// Returns a lifetime identifier if this token is a lifetime. |
04454e1e | 550 | #[inline] |
f9f354fc | 551 | pub fn lifetime(&self) -> Option<Ident> { |
04454e1e FG |
552 | // We avoid using `Token::uninterpolate` here because it's slow. |
553 | match &self.kind { | |
554 | &Lifetime(name) => Some(Ident::new(name, self.span)), | |
555 | Interpolated(nt) => match **nt { | |
556 | NtLifetime(ident) => Some(ident), | |
557 | _ => None, | |
558 | }, | |
cc61c64b | 559 | _ => None, |
223e47cc | 560 | } |
1a4d82fc | 561 | } |
dc9dc135 | 562 | |
cc61c64b XL |
563 | /// Returns `true` if the token is an identifier. |
564 | pub fn is_ident(&self) -> bool { | |
565 | self.ident().is_some() | |
566 | } | |
e1599b0c | 567 | |
83c7162d | 568 | /// Returns `true` if the token is a lifetime. |
60c5eb7d | 569 | pub fn is_lifetime(&self) -> bool { |
83c7162d XL |
570 | self.lifetime().is_some() |
571 | } | |
572 | ||
94222f64 | 573 | /// Returns `true` if the token is an identifier whose name is the given |
83c7162d | 574 | /// string slice. |
60c5eb7d | 575 | pub fn is_ident_named(&self, name: Symbol) -> bool { |
dc9dc135 | 576 | self.ident().map_or(false, |(ident, _)| ident.name == name) |
83c7162d | 577 | } |
cc61c64b | 578 | |
1a4d82fc | 579 | /// Returns `true` if the token is an interpolated path. |
94b46f34 | 580 | fn is_path(&self) -> bool { |
5e7ed085 FG |
581 | if let Interpolated(ref nt) = self.kind && let NtPath(..) = **nt { |
582 | return true; | |
1a4d82fc | 583 | } |
c30ab7b3 | 584 | false |
1a4d82fc JJ |
585 | } |
586 | ||
416331ca XL |
587 | /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? |
588 | /// That is, is this a pre-parsed expression dropped into the token stream | |
589 | /// (which happens while parsing the result of macro expansion)? | |
60c5eb7d | 590 | pub fn is_whole_expr(&self) -> bool { |
5e7ed085 | 591 | if let Interpolated(ref nt) = self.kind |
04454e1e | 592 | && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt |
5e7ed085 FG |
593 | { |
594 | return true; | |
416331ca XL |
595 | } |
596 | ||
597 | false | |
598 | } | |
599 | ||
ba9703b0 XL |
600 | // Is the token an interpolated block (`$b:block`)? |
601 | pub fn is_whole_block(&self) -> bool { | |
5e7ed085 FG |
602 | if let Interpolated(ref nt) = self.kind && let NtBlock(..) = **nt { |
603 | return true; | |
ba9703b0 XL |
604 | } |
605 | false | |
606 | } | |
607 | ||
1a4d82fc | 608 | /// Returns `true` if the token is either the `mut` or `const` keyword. |
60c5eb7d | 609 | pub fn is_mutability(&self) -> bool { |
dfeec247 | 610 | self.is_keyword(kw::Mut) || self.is_keyword(kw::Const) |
1a4d82fc JJ |
611 | } |
612 | ||
60c5eb7d | 613 | pub fn is_qpath_start(&self) -> bool { |
c30ab7b3 SL |
614 | self == &Lt || self == &BinOp(Shl) |
615 | } | |
616 | ||
60c5eb7d | 617 | pub fn is_path_start(&self) -> bool { |
dfeec247 XL |
618 | self == &ModSep |
619 | || self.is_qpath_start() | |
620 | || self.is_path() | |
621 | || self.is_path_segment_keyword() | |
622 | || self.is_ident() && !self.is_reserved_ident() | |
a7813a04 XL |
623 | } |
624 | ||
1a4d82fc | 625 | /// Returns `true` if the token is a given keyword, `kw`. |
dc9dc135 | 626 | pub fn is_keyword(&self, kw: Symbol) -> bool { |
e1599b0c | 627 | self.is_non_raw_ident_where(|id| id.name == kw) |
1a4d82fc JJ |
628 | } |
629 | ||
60c5eb7d | 630 | pub fn is_path_segment_keyword(&self) -> bool { |
f9f354fc | 631 | self.is_non_raw_ident_where(Ident::is_path_segment_keyword) |
1a4d82fc JJ |
632 | } |
633 | ||
041b39d2 XL |
634 | // Returns true for reserved identifiers used internally for elided lifetimes, |
635 | // unnamed method parameters, crate root module, error recovery etc. | |
60c5eb7d | 636 | pub fn is_special_ident(&self) -> bool { |
f9f354fc | 637 | self.is_non_raw_ident_where(Ident::is_special) |
1a4d82fc JJ |
638 | } |
639 | ||
041b39d2 | 640 | /// Returns `true` if the token is a keyword used in the language. |
60c5eb7d | 641 | pub fn is_used_keyword(&self) -> bool { |
f9f354fc | 642 | self.is_non_raw_ident_where(Ident::is_used_keyword) |
1a4d82fc JJ |
643 | } |
644 | ||
a7813a04 | 645 | /// Returns `true` if the token is a keyword reserved for possible future use. |
60c5eb7d | 646 | pub fn is_unused_keyword(&self) -> bool { |
f9f354fc | 647 | self.is_non_raw_ident_where(Ident::is_unused_keyword) |
1a4d82fc | 648 | } |
041b39d2 | 649 | |
83c7162d XL |
650 | /// Returns `true` if the token is either a special identifier or a keyword. |
651 | pub fn is_reserved_ident(&self) -> bool { | |
f9f354fc | 652 | self.is_non_raw_ident_where(Ident::is_reserved) |
e1599b0c XL |
653 | } |
654 | ||
655 | /// Returns `true` if the token is the identifier `true` or `false`. | |
60c5eb7d | 656 | pub fn is_bool_lit(&self) -> bool { |
e1599b0c XL |
657 | self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) |
658 | } | |
659 | ||
c295e0f8 XL |
660 | pub fn is_numeric_lit(&self) -> bool { |
661 | matches!( | |
662 | self.kind, | |
663 | Literal(Lit { kind: LitKind::Integer, .. }) | Literal(Lit { kind: LitKind::Float, .. }) | |
664 | ) | |
665 | } | |
666 | ||
e1599b0c | 667 | /// Returns `true` if the token is a non-raw identifier for which `pred` holds. |
f9f354fc | 668 | pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool { |
83c7162d | 669 | match self.ident() { |
e1599b0c | 670 | Some((id, false)) => pred(id), |
83c7162d XL |
671 | _ => false, |
672 | } | |
673 | } | |
674 | ||
60c5eb7d | 675 | pub fn glue(&self, joint: &Token) -> Option<Token> { |
dc9dc135 XL |
676 | let kind = match self.kind { |
677 | Eq => match joint.kind { | |
041b39d2 XL |
678 | Eq => EqEq, |
679 | Gt => FatArrow, | |
680 | _ => return None, | |
681 | }, | |
dc9dc135 | 682 | Lt => match joint.kind { |
041b39d2 XL |
683 | Eq => Le, |
684 | Lt => BinOp(Shl), | |
685 | Le => BinOpEq(Shl), | |
686 | BinOp(Minus) => LArrow, | |
687 | _ => return None, | |
688 | }, | |
dc9dc135 | 689 | Gt => match joint.kind { |
041b39d2 XL |
690 | Eq => Ge, |
691 | Gt => BinOp(Shr), | |
692 | Ge => BinOpEq(Shr), | |
693 | _ => return None, | |
694 | }, | |
dc9dc135 | 695 | Not => match joint.kind { |
041b39d2 XL |
696 | Eq => Ne, |
697 | _ => return None, | |
698 | }, | |
dc9dc135 | 699 | BinOp(op) => match joint.kind { |
041b39d2 XL |
700 | Eq => BinOpEq(op), |
701 | BinOp(And) if op == And => AndAnd, | |
702 | BinOp(Or) if op == Or => OrOr, | |
703 | Gt if op == Minus => RArrow, | |
704 | _ => return None, | |
705 | }, | |
dc9dc135 | 706 | Dot => match joint.kind { |
041b39d2 XL |
707 | Dot => DotDot, |
708 | DotDot => DotDotDot, | |
709 | _ => return None, | |
710 | }, | |
dc9dc135 | 711 | DotDot => match joint.kind { |
041b39d2 | 712 | Dot => DotDotDot, |
ea8adc8c | 713 | Eq => DotDotEq, |
041b39d2 XL |
714 | _ => return None, |
715 | }, | |
dc9dc135 | 716 | Colon => match joint.kind { |
041b39d2 XL |
717 | Colon => ModSep, |
718 | _ => return None, | |
719 | }, | |
dc9dc135 XL |
720 | SingleQuote => match joint.kind { |
721 | Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), | |
94b46f34 XL |
722 | _ => return None, |
723 | }, | |
041b39d2 | 724 | |
dfeec247 XL |
725 | Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot |
726 | | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | |
727 | | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | |
1b1a35ee | 728 | | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None, |
dc9dc135 | 729 | }; |
041b39d2 | 730 | |
dc9dc135 | 731 | Some(Token::new(kind, self.span.to(joint.span))) |
abe05a73 | 732 | } |
223e47cc LB |
733 | } |
734 | ||
dc9dc135 XL |
735 | impl PartialEq<TokenKind> for Token { |
736 | fn eq(&self, rhs: &TokenKind) -> bool { | |
737 | self.kind == *rhs | |
738 | } | |
739 | } | |
740 | ||
3dfed10e | 741 | #[derive(Clone, Encodable, Decodable)] |
1a4d82fc JJ |
742 | /// For interpolation during macro expansion. |
743 | pub enum Nonterminal { | |
744 | NtItem(P<ast::Item>), | |
745 | NtBlock(P<ast::Block>), | |
04454e1e | 746 | NtStmt(P<ast::Stmt>), |
1a4d82fc JJ |
747 | NtPat(P<ast::Pat>), |
748 | NtExpr(P<ast::Expr>), | |
749 | NtTy(P<ast::Ty>), | |
f9f354fc XL |
750 | NtIdent(Ident, /* is_raw */ bool), |
751 | NtLifetime(Ident), | |
94b46f34 | 752 | NtLiteral(P<ast::Expr>), |
1a4d82fc | 753 | /// Stuff inside brackets for attributes |
74b04a01 | 754 | NtMeta(P<ast::AttrItem>), |
04454e1e FG |
755 | NtPath(P<ast::Path>), |
756 | NtVis(P<ast::Visibility>), | |
74b04a01 XL |
757 | } |
758 | ||
759 | // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger. | |
6a06907d | 760 | #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] |
04454e1e | 761 | rustc_data_structures::static_assert_size!(Nonterminal, 16); |
74b04a01 | 762 | |
3dfed10e XL |
763 | #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)] |
764 | pub enum NonterminalKind { | |
765 | Item, | |
766 | Block, | |
767 | Stmt, | |
cdc7bbd5 XL |
768 | PatParam { |
769 | /// Keep track of whether the user used `:pat_param` or `:pat` and we inferred it from the | |
5869c6ff XL |
770 | /// edition of the span. This is used for diagnostics. |
771 | inferred: bool, | |
772 | }, | |
cdc7bbd5 | 773 | PatWithOr, |
3dfed10e XL |
774 | Expr, |
775 | Ty, | |
776 | Ident, | |
777 | Lifetime, | |
778 | Literal, | |
779 | Meta, | |
780 | Path, | |
781 | Vis, | |
782 | TT, | |
783 | } | |
784 | ||
785 | impl NonterminalKind { | |
5869c6ff XL |
786 | /// The `edition` closure is used to get the edition for the given symbol. Doing |
787 | /// `span.edition()` is expensive, so we do it lazily. | |
788 | pub fn from_symbol( | |
789 | symbol: Symbol, | |
790 | edition: impl FnOnce() -> Edition, | |
791 | ) -> Option<NonterminalKind> { | |
3dfed10e XL |
792 | Some(match symbol { |
793 | sym::item => NonterminalKind::Item, | |
794 | sym::block => NonterminalKind::Block, | |
795 | sym::stmt => NonterminalKind::Stmt, | |
5869c6ff XL |
796 | sym::pat => match edition() { |
797 | Edition::Edition2015 | Edition::Edition2018 => { | |
cdc7bbd5 | 798 | NonterminalKind::PatParam { inferred: true } |
5869c6ff | 799 | } |
04454e1e | 800 | Edition::Edition2021 | Edition::Edition2024 => NonterminalKind::PatWithOr, |
5869c6ff | 801 | }, |
cdc7bbd5 | 802 | sym::pat_param => NonterminalKind::PatParam { inferred: false }, |
3dfed10e XL |
803 | sym::expr => NonterminalKind::Expr, |
804 | sym::ty => NonterminalKind::Ty, | |
805 | sym::ident => NonterminalKind::Ident, | |
806 | sym::lifetime => NonterminalKind::Lifetime, | |
807 | sym::literal => NonterminalKind::Literal, | |
808 | sym::meta => NonterminalKind::Meta, | |
809 | sym::path => NonterminalKind::Path, | |
810 | sym::vis => NonterminalKind::Vis, | |
811 | sym::tt => NonterminalKind::TT, | |
812 | _ => return None, | |
813 | }) | |
814 | } | |
815 | fn symbol(self) -> Symbol { | |
816 | match self { | |
817 | NonterminalKind::Item => sym::item, | |
818 | NonterminalKind::Block => sym::block, | |
819 | NonterminalKind::Stmt => sym::stmt, | |
cdc7bbd5 XL |
820 | NonterminalKind::PatParam { inferred: false } => sym::pat_param, |
821 | NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat, | |
3dfed10e XL |
822 | NonterminalKind::Expr => sym::expr, |
823 | NonterminalKind::Ty => sym::ty, | |
824 | NonterminalKind::Ident => sym::ident, | |
825 | NonterminalKind::Lifetime => sym::lifetime, | |
826 | NonterminalKind::Literal => sym::literal, | |
827 | NonterminalKind::Meta => sym::meta, | |
828 | NonterminalKind::Path => sym::path, | |
829 | NonterminalKind::Vis => sym::vis, | |
830 | NonterminalKind::TT => sym::tt, | |
831 | } | |
832 | } | |
833 | } | |
834 | ||
835 | impl fmt::Display for NonterminalKind { | |
836 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
837 | write!(f, "{}", self.symbol()) | |
838 | } | |
839 | } | |
840 | ||
74b04a01 | 841 | impl Nonterminal { |
5869c6ff | 842 | pub fn span(&self) -> Span { |
74b04a01 XL |
843 | match self { |
844 | NtItem(item) => item.span, | |
845 | NtBlock(block) => block.span, | |
846 | NtStmt(stmt) => stmt.span, | |
847 | NtPat(pat) => pat.span, | |
848 | NtExpr(expr) | NtLiteral(expr) => expr.span, | |
849 | NtTy(ty) => ty.span, | |
850 | NtIdent(ident, _) | NtLifetime(ident) => ident.span, | |
851 | NtMeta(attr_item) => attr_item.span(), | |
852 | NtPath(path) => path.span, | |
853 | NtVis(vis) => vis.span, | |
74b04a01 XL |
854 | } |
855 | } | |
83c7162d XL |
856 | } |
857 | ||
858 | impl PartialEq for Nonterminal { | |
859 | fn eq(&self, rhs: &Self) -> bool { | |
860 | match (self, rhs) { | |
dfeec247 XL |
861 | (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { |
862 | ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs | |
863 | } | |
83c7162d | 864 | (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, |
83c7162d XL |
865 | // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them |
866 | // correctly based on data from AST. This will prevent them from matching each other | |
867 | // in macros. The comparison will become possible only when each nonterminal has an | |
868 | // attached token stream from which it was parsed. | |
869 | _ => false, | |
870 | } | |
871 | } | |
1a4d82fc JJ |
872 | } |
873 | ||
85aaf69f | 874 | impl fmt::Debug for Nonterminal { |
9fa01778 | 875 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
1a4d82fc JJ |
876 | match *self { |
877 | NtItem(..) => f.pad("NtItem(..)"), | |
878 | NtBlock(..) => f.pad("NtBlock(..)"), | |
879 | NtStmt(..) => f.pad("NtStmt(..)"), | |
880 | NtPat(..) => f.pad("NtPat(..)"), | |
881 | NtExpr(..) => f.pad("NtExpr(..)"), | |
882 | NtTy(..) => f.pad("NtTy(..)"), | |
883 | NtIdent(..) => f.pad("NtIdent(..)"), | |
94b46f34 | 884 | NtLiteral(..) => f.pad("NtLiteral(..)"), |
1a4d82fc JJ |
885 | NtMeta(..) => f.pad("NtMeta(..)"), |
886 | NtPath(..) => f.pad("NtPath(..)"), | |
cc61c64b | 887 | NtVis(..) => f.pad("NtVis(..)"), |
ff7c6d11 | 888 | NtLifetime(..) => f.pad("NtLifetime(..)"), |
1a4d82fc | 889 | } |
970d7e83 LB |
890 | } |
891 | } | |
60c5eb7d XL |
892 | |
893 | impl<CTX> HashStable<CTX> for Nonterminal | |
dfeec247 XL |
894 | where |
895 | CTX: crate::HashStableContext, | |
60c5eb7d XL |
896 | { |
897 | fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { | |
898 | panic!("interpolated tokens should not be present in the HIR") | |
899 | } | |
900 | } |