]>
Commit | Line | Data |
---|---|---|
f20569fa XL |
1 | pub use BinOpToken::*; |
2 | pub use DelimToken::*; | |
3 | pub use LitKind::*; | |
4 | pub use Nonterminal::*; | |
5 | pub use TokenKind::*; | |
6 | ||
7 | use crate::ast; | |
8 | use crate::ptr::P; | |
9 | use crate::tokenstream::TokenTree; | |
10 | ||
11 | use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; | |
12 | use rustc_data_structures::sync::Lrc; | |
13 | use rustc_macros::HashStable_Generic; | |
14 | use rustc_span::hygiene::ExpnKind; | |
15 | use rustc_span::source_map::SourceMap; | |
16 | use rustc_span::symbol::{kw, sym}; | |
17 | use rustc_span::symbol::{Ident, Symbol}; | |
18 | use rustc_span::{self, edition::Edition, FileName, RealFileName, Span, DUMMY_SP}; | |
19 | use std::borrow::Cow; | |
20 | use std::{fmt, mem}; | |
21 | ||
22 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] | |
23 | pub enum CommentKind { | |
24 | Line, | |
25 | Block, | |
26 | } | |
27 | ||
28 | #[derive(Clone, PartialEq, Encodable, Decodable, Hash, Debug, Copy)] | |
29 | #[derive(HashStable_Generic)] | |
30 | pub enum BinOpToken { | |
31 | Plus, | |
32 | Minus, | |
33 | Star, | |
34 | Slash, | |
35 | Percent, | |
36 | Caret, | |
37 | And, | |
38 | Or, | |
39 | Shl, | |
40 | Shr, | |
41 | } | |
42 | ||
43 | /// A delimiter token. | |
44 | #[derive(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Debug, Copy)] | |
45 | #[derive(HashStable_Generic)] | |
46 | pub enum DelimToken { | |
47 | /// A round parenthesis (i.e., `(` or `)`). | |
48 | Paren, | |
49 | /// A square bracket (i.e., `[` or `]`). | |
50 | Bracket, | |
51 | /// A curly brace (i.e., `{` or `}`). | |
52 | Brace, | |
53 | /// An empty delimiter. | |
54 | NoDelim, | |
55 | } | |
56 | ||
57 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] | |
58 | pub enum LitKind { | |
59 | Bool, // AST only, must never appear in a `Token` | |
60 | Byte, | |
61 | Char, | |
62 | Integer, | |
63 | Float, | |
64 | Str, | |
65 | StrRaw(u16), // raw string delimited by `n` hash symbols | |
66 | ByteStr, | |
67 | ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols | |
68 | Err, | |
69 | } | |
70 | ||
71 | /// A literal token. | |
72 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] | |
73 | pub struct Lit { | |
74 | pub kind: LitKind, | |
75 | pub symbol: Symbol, | |
76 | pub suffix: Option<Symbol>, | |
77 | } | |
78 | ||
79 | impl fmt::Display for Lit { | |
80 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
81 | let Lit { kind, symbol, suffix } = *self; | |
82 | match kind { | |
83 | Byte => write!(f, "b'{}'", symbol)?, | |
84 | Char => write!(f, "'{}'", symbol)?, | |
85 | Str => write!(f, "\"{}\"", symbol)?, | |
86 | StrRaw(n) => write!( | |
87 | f, | |
88 | "r{delim}\"{string}\"{delim}", | |
89 | delim = "#".repeat(n as usize), | |
90 | string = symbol | |
91 | )?, | |
92 | ByteStr => write!(f, "b\"{}\"", symbol)?, | |
93 | ByteStrRaw(n) => write!( | |
94 | f, | |
95 | "br{delim}\"{string}\"{delim}", | |
96 | delim = "#".repeat(n as usize), | |
97 | string = symbol | |
98 | )?, | |
99 | Integer | Float | Bool | Err => write!(f, "{}", symbol)?, | |
100 | } | |
101 | ||
102 | if let Some(suffix) = suffix { | |
103 | write!(f, "{}", suffix)?; | |
104 | } | |
105 | ||
106 | Ok(()) | |
107 | } | |
108 | } | |
109 | ||
110 | impl LitKind { | |
111 | /// An English article for the literal token kind. | |
112 | pub fn article(self) -> &'static str { | |
113 | match self { | |
114 | Integer | Err => "an", | |
115 | _ => "a", | |
116 | } | |
117 | } | |
118 | ||
119 | pub fn descr(self) -> &'static str { | |
120 | match self { | |
121 | Bool => panic!("literal token contains `Lit::Bool`"), | |
122 | Byte => "byte", | |
123 | Char => "char", | |
124 | Integer => "integer", | |
125 | Float => "float", | |
126 | Str | StrRaw(..) => "string", | |
127 | ByteStr | ByteStrRaw(..) => "byte string", | |
128 | Err => "error", | |
129 | } | |
130 | } | |
131 | ||
132 | crate fn may_have_suffix(self) -> bool { | |
133 | matches!(self, Integer | Float | Err) | |
134 | } | |
135 | } | |
136 | ||
137 | impl Lit { | |
138 | pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit { | |
139 | Lit { kind, symbol, suffix } | |
140 | } | |
141 | } | |
142 | ||
143 | pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool { | |
144 | let ident_token = Token::new(Ident(name, is_raw), span); | |
145 | ||
146 | !ident_token.is_reserved_ident() | |
147 | || ident_token.is_path_segment_keyword() | |
148 | || [ | |
149 | kw::Async, | |
150 | kw::Do, | |
151 | kw::Box, | |
152 | kw::Break, | |
153 | kw::Const, | |
154 | kw::Continue, | |
155 | kw::False, | |
156 | kw::For, | |
157 | kw::If, | |
158 | kw::Let, | |
159 | kw::Loop, | |
160 | kw::Match, | |
161 | kw::Move, | |
162 | kw::Return, | |
163 | kw::True, | |
164 | kw::Try, | |
165 | kw::Unsafe, | |
166 | kw::While, | |
167 | kw::Yield, | |
168 | kw::Static, | |
169 | ] | |
170 | .contains(&name) | |
171 | } | |
172 | ||
173 | fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool { | |
174 | let ident_token = Token::new(Ident(name, is_raw), span); | |
175 | ||
176 | !ident_token.is_reserved_ident() | |
177 | || ident_token.is_path_segment_keyword() | |
178 | || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn] | |
179 | .contains(&name) | |
180 | } | |
181 | ||
182 | #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] | |
183 | pub enum TokenKind { | |
184 | /* Expression-operator symbols. */ | |
185 | Eq, | |
186 | Lt, | |
187 | Le, | |
188 | EqEq, | |
189 | Ne, | |
190 | Ge, | |
191 | Gt, | |
192 | AndAnd, | |
193 | OrOr, | |
194 | Not, | |
195 | Tilde, | |
196 | BinOp(BinOpToken), | |
197 | BinOpEq(BinOpToken), | |
198 | ||
199 | /* Structural symbols */ | |
200 | At, | |
201 | Dot, | |
202 | DotDot, | |
203 | DotDotDot, | |
204 | DotDotEq, | |
205 | Comma, | |
206 | Semi, | |
207 | Colon, | |
208 | ModSep, | |
209 | RArrow, | |
210 | LArrow, | |
211 | FatArrow, | |
212 | Pound, | |
213 | Dollar, | |
214 | Question, | |
215 | /// Used by proc macros for representing lifetimes, not generated by lexer right now. | |
216 | SingleQuote, | |
217 | /// An opening delimiter (e.g., `{`). | |
218 | OpenDelim(DelimToken), | |
219 | /// A closing delimiter (e.g., `}`). | |
220 | CloseDelim(DelimToken), | |
221 | ||
222 | /* Literals */ | |
223 | Literal(Lit), | |
224 | ||
225 | /// Identifier token. | |
226 | /// Do not forget about `NtIdent` when you want to match on identifiers. | |
227 | /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to | |
228 | /// treat regular and interpolated identifiers in the same way. | |
229 | Ident(Symbol, /* is_raw */ bool), | |
230 | /// Lifetime identifier token. | |
231 | /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. | |
232 | /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to | |
233 | /// treat regular and interpolated lifetime identifiers in the same way. | |
234 | Lifetime(Symbol), | |
235 | ||
236 | Interpolated(Lrc<Nonterminal>), | |
237 | ||
238 | /// A doc comment token. | |
239 | /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc) | |
240 | /// similarly to symbols in string literal tokens. | |
241 | DocComment(CommentKind, ast::AttrStyle, Symbol), | |
242 | ||
243 | Eof, | |
244 | } | |
245 | ||
246 | // `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. | |
247 | #[cfg(target_arch = "x86_64")] | |
248 | rustc_data_structures::static_assert_size!(TokenKind, 16); | |
249 | ||
250 | #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] | |
251 | pub struct Token { | |
252 | pub kind: TokenKind, | |
253 | pub span: Span, | |
254 | } | |
255 | ||
256 | impl TokenKind { | |
257 | pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind { | |
258 | Literal(Lit::new(kind, symbol, suffix)) | |
259 | } | |
260 | ||
261 | // An approximation to proc-macro-style single-character operators used by rustc parser. | |
262 | // If the operator token can be broken into two tokens, the first of which is single-character, | |
263 | // then this function performs that operation, otherwise it returns `None`. | |
264 | pub fn break_two_token_op(&self) -> Option<(TokenKind, TokenKind)> { | |
265 | Some(match *self { | |
266 | Le => (Lt, Eq), | |
267 | EqEq => (Eq, Eq), | |
268 | Ne => (Not, Eq), | |
269 | Ge => (Gt, Eq), | |
270 | AndAnd => (BinOp(And), BinOp(And)), | |
271 | OrOr => (BinOp(Or), BinOp(Or)), | |
272 | BinOp(Shl) => (Lt, Lt), | |
273 | BinOp(Shr) => (Gt, Gt), | |
274 | BinOpEq(Plus) => (BinOp(Plus), Eq), | |
275 | BinOpEq(Minus) => (BinOp(Minus), Eq), | |
276 | BinOpEq(Star) => (BinOp(Star), Eq), | |
277 | BinOpEq(Slash) => (BinOp(Slash), Eq), | |
278 | BinOpEq(Percent) => (BinOp(Percent), Eq), | |
279 | BinOpEq(Caret) => (BinOp(Caret), Eq), | |
280 | BinOpEq(And) => (BinOp(And), Eq), | |
281 | BinOpEq(Or) => (BinOp(Or), Eq), | |
282 | BinOpEq(Shl) => (Lt, Le), | |
283 | BinOpEq(Shr) => (Gt, Ge), | |
284 | DotDot => (Dot, Dot), | |
285 | DotDotDot => (Dot, DotDot), | |
286 | ModSep => (Colon, Colon), | |
287 | RArrow => (BinOp(Minus), Gt), | |
288 | LArrow => (Lt, BinOp(Minus)), | |
289 | FatArrow => (Eq, Gt), | |
290 | _ => return None, | |
291 | }) | |
292 | } | |
293 | ||
294 | /// Returns tokens that are likely to be typed accidentally instead of the current token. | |
295 | /// Enables better error recovery when the wrong token is found. | |
296 | pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> { | |
297 | match *self { | |
298 | Comma => Some(vec![Dot, Lt, Semi]), | |
299 | Semi => Some(vec![Colon, Comma]), | |
300 | _ => None, | |
301 | } | |
302 | } | |
303 | ||
304 | pub fn should_end_const_arg(&self) -> bool { | |
305 | matches!(self, Gt | Ge | BinOp(Shr) | BinOpEq(Shr)) | |
306 | } | |
307 | } | |
308 | ||
309 | impl Token { | |
310 | pub fn new(kind: TokenKind, span: Span) -> Self { | |
311 | Token { kind, span } | |
312 | } | |
313 | ||
314 | /// Some token that will be thrown away later. | |
315 | pub fn dummy() -> Self { | |
316 | Token::new(TokenKind::Question, DUMMY_SP) | |
317 | } | |
318 | ||
319 | /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary. | |
320 | pub fn from_ast_ident(ident: Ident) -> Self { | |
321 | Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) | |
322 | } | |
323 | ||
324 | /// Return this token by value and leave a dummy token in its place. | |
325 | pub fn take(&mut self) -> Self { | |
326 | mem::replace(self, Token::dummy()) | |
327 | } | |
328 | ||
329 | /// For interpolated tokens, returns a span of the fragment to which the interpolated | |
330 | /// token refers. For all other tokens this is just a regular span. | |
331 | /// It is particularly important to use this for identifiers and lifetimes | |
332 | /// for which spans affect name resolution and edition checks. | |
333 | /// Note that keywords are also identifiers, so they should use this | |
334 | /// if they keep spans or perform edition checks. | |
335 | pub fn uninterpolated_span(&self) -> Span { | |
336 | match &self.kind { | |
337 | Interpolated(nt) => nt.span(), | |
338 | _ => self.span, | |
339 | } | |
340 | } | |
341 | ||
342 | pub fn is_op(&self) -> bool { | |
343 | !matches!( | |
344 | self.kind, | |
345 | OpenDelim(..) | |
346 | | CloseDelim(..) | |
347 | | Literal(..) | |
348 | | DocComment(..) | |
349 | | Ident(..) | |
350 | | Lifetime(..) | |
351 | | Interpolated(..) | |
352 | | Eof | |
353 | ) | |
354 | } | |
355 | ||
356 | pub fn is_like_plus(&self) -> bool { | |
357 | matches!(self.kind, BinOp(Plus) | BinOpEq(Plus)) | |
358 | } | |
359 | ||
360 | /// Returns `true` if the token can appear at the start of an expression. | |
361 | pub fn can_begin_expr(&self) -> bool { | |
362 | match self.uninterpolate().kind { | |
363 | Ident(name, is_raw) => | |
364 | ident_can_begin_expr(name, self.span, is_raw), // value name or keyword | |
365 | OpenDelim(..) | // tuple, array or block | |
366 | Literal(..) | // literal | |
367 | Not | // operator not | |
368 | BinOp(Minus) | // unary minus | |
369 | BinOp(Star) | // dereference | |
370 | BinOp(Or) | OrOr | // closure | |
371 | BinOp(And) | // reference | |
372 | AndAnd | // double reference | |
373 | // DotDotDot is no longer supported, but we need some way to display the error | |
374 | DotDot | DotDotDot | DotDotEq | // range notation | |
375 | Lt | BinOp(Shl) | // associated path | |
376 | ModSep | // global path | |
377 | Lifetime(..) | // labeled loop | |
378 | Pound => true, // expression attributes | |
379 | Interpolated(ref nt) => matches!(**nt, NtLiteral(..) | | |
380 | NtExpr(..) | | |
381 | NtBlock(..) | | |
382 | NtPath(..)), | |
383 | _ => false, | |
384 | } | |
385 | } | |
386 | ||
387 | /// Returns `true` if the token can appear at the start of a type. | |
388 | pub fn can_begin_type(&self) -> bool { | |
389 | match self.uninterpolate().kind { | |
390 | Ident(name, is_raw) => | |
391 | ident_can_begin_type(name, self.span, is_raw), // type name or keyword | |
392 | OpenDelim(Paren) | // tuple | |
393 | OpenDelim(Bracket) | // array | |
394 | Not | // never | |
395 | BinOp(Star) | // raw pointer | |
396 | BinOp(And) | // reference | |
397 | AndAnd | // double reference | |
398 | Question | // maybe bound in trait object | |
399 | Lifetime(..) | // lifetime bound in trait object | |
400 | Lt | BinOp(Shl) | // associated path | |
401 | ModSep => true, // global path | |
402 | Interpolated(ref nt) => matches!(**nt, NtTy(..) | NtPath(..)), | |
403 | _ => false, | |
404 | } | |
405 | } | |
406 | ||
407 | /// Returns `true` if the token can appear at the start of a const param. | |
408 | pub fn can_begin_const_arg(&self) -> bool { | |
409 | match self.kind { | |
410 | OpenDelim(Brace) => true, | |
411 | Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)), | |
412 | _ => self.can_begin_literal_maybe_minus(), | |
413 | } | |
414 | } | |
415 | ||
416 | /// Returns `true` if the token can appear at the start of a generic bound. | |
417 | pub fn can_begin_bound(&self) -> bool { | |
418 | self.is_path_start() | |
419 | || self.is_lifetime() | |
420 | || self.is_keyword(kw::For) | |
421 | || self == &Question | |
422 | || self == &OpenDelim(Paren) | |
423 | } | |
424 | ||
425 | /// Returns `true` if the token is any literal. | |
426 | pub fn is_lit(&self) -> bool { | |
427 | matches!(self.kind, Literal(..)) | |
428 | } | |
429 | ||
430 | /// Returns `true` if the token is any literal, a minus (which can prefix a literal, | |
431 | /// for example a '-42', or one of the boolean idents). | |
432 | /// | |
433 | /// In other words, would this token be a valid start of `parse_literal_maybe_minus`? | |
434 | /// | |
435 | /// Keep this in sync with and `Lit::from_token`, excluding unary negation. | |
436 | pub fn can_begin_literal_maybe_minus(&self) -> bool { | |
437 | match self.uninterpolate().kind { | |
438 | Literal(..) | BinOp(Minus) => true, | |
439 | Ident(name, false) if name.is_bool_lit() => true, | |
440 | Interpolated(ref nt) => match &**nt { | |
441 | NtLiteral(_) => true, | |
442 | NtExpr(e) => match &e.kind { | |
443 | ast::ExprKind::Lit(_) => true, | |
444 | ast::ExprKind::Unary(ast::UnOp::Neg, e) => { | |
445 | matches!(&e.kind, ast::ExprKind::Lit(_)) | |
446 | } | |
447 | _ => false, | |
448 | }, | |
449 | _ => false, | |
450 | }, | |
451 | _ => false, | |
452 | } | |
453 | } | |
454 | ||
455 | // A convenience function for matching on identifiers during parsing. | |
456 | // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token | |
457 | // into the regular identifier or lifetime token it refers to, | |
458 | // otherwise returns the original token. | |
459 | pub fn uninterpolate(&self) -> Cow<'_, Token> { | |
460 | match &self.kind { | |
461 | Interpolated(nt) => match **nt { | |
462 | NtIdent(ident, is_raw) => { | |
463 | Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)) | |
464 | } | |
465 | NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), | |
466 | _ => Cow::Borrowed(self), | |
467 | }, | |
468 | _ => Cow::Borrowed(self), | |
469 | } | |
470 | } | |
471 | ||
472 | /// Returns an identifier if this token is an identifier. | |
473 | pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> { | |
474 | let token = self.uninterpolate(); | |
475 | match token.kind { | |
476 | Ident(name, is_raw) => Some((Ident::new(name, token.span), is_raw)), | |
477 | _ => None, | |
478 | } | |
479 | } | |
480 | ||
481 | /// Returns a lifetime identifier if this token is a lifetime. | |
482 | pub fn lifetime(&self) -> Option<Ident> { | |
483 | let token = self.uninterpolate(); | |
484 | match token.kind { | |
485 | Lifetime(name) => Some(Ident::new(name, token.span)), | |
486 | _ => None, | |
487 | } | |
488 | } | |
489 | ||
490 | /// Returns `true` if the token is an identifier. | |
491 | pub fn is_ident(&self) -> bool { | |
492 | self.ident().is_some() | |
493 | } | |
494 | ||
495 | /// Returns `true` if the token is a lifetime. | |
496 | pub fn is_lifetime(&self) -> bool { | |
497 | self.lifetime().is_some() | |
498 | } | |
499 | ||
500 | /// Returns `true` if the token is a identifier whose name is the given | |
501 | /// string slice. | |
502 | pub fn is_ident_named(&self, name: Symbol) -> bool { | |
503 | self.ident().map_or(false, |(ident, _)| ident.name == name) | |
504 | } | |
505 | ||
506 | /// Returns `true` if the token is an interpolated path. | |
507 | fn is_path(&self) -> bool { | |
508 | if let Interpolated(ref nt) = self.kind { | |
509 | if let NtPath(..) = **nt { | |
510 | return true; | |
511 | } | |
512 | } | |
513 | false | |
514 | } | |
515 | ||
516 | /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? | |
517 | /// That is, is this a pre-parsed expression dropped into the token stream | |
518 | /// (which happens while parsing the result of macro expansion)? | |
519 | pub fn is_whole_expr(&self) -> bool { | |
520 | if let Interpolated(ref nt) = self.kind { | |
521 | if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { | |
522 | return true; | |
523 | } | |
524 | } | |
525 | ||
526 | false | |
527 | } | |
528 | ||
529 | // Is the token an interpolated block (`$b:block`)? | |
530 | pub fn is_whole_block(&self) -> bool { | |
531 | if let Interpolated(ref nt) = self.kind { | |
532 | if let NtBlock(..) = **nt { | |
533 | return true; | |
534 | } | |
535 | } | |
536 | false | |
537 | } | |
538 | ||
539 | /// Returns `true` if the token is either the `mut` or `const` keyword. | |
540 | pub fn is_mutability(&self) -> bool { | |
541 | self.is_keyword(kw::Mut) || self.is_keyword(kw::Const) | |
542 | } | |
543 | ||
544 | pub fn is_qpath_start(&self) -> bool { | |
545 | self == &Lt || self == &BinOp(Shl) | |
546 | } | |
547 | ||
548 | pub fn is_path_start(&self) -> bool { | |
549 | self == &ModSep | |
550 | || self.is_qpath_start() | |
551 | || self.is_path() | |
552 | || self.is_path_segment_keyword() | |
553 | || self.is_ident() && !self.is_reserved_ident() | |
554 | } | |
555 | ||
556 | /// Returns `true` if the token is a given keyword, `kw`. | |
557 | pub fn is_keyword(&self, kw: Symbol) -> bool { | |
558 | self.is_non_raw_ident_where(|id| id.name == kw) | |
559 | } | |
560 | ||
561 | pub fn is_path_segment_keyword(&self) -> bool { | |
562 | self.is_non_raw_ident_where(Ident::is_path_segment_keyword) | |
563 | } | |
564 | ||
565 | // Returns true for reserved identifiers used internally for elided lifetimes, | |
566 | // unnamed method parameters, crate root module, error recovery etc. | |
567 | pub fn is_special_ident(&self) -> bool { | |
568 | self.is_non_raw_ident_where(Ident::is_special) | |
569 | } | |
570 | ||
571 | /// Returns `true` if the token is a keyword used in the language. | |
572 | pub fn is_used_keyword(&self) -> bool { | |
573 | self.is_non_raw_ident_where(Ident::is_used_keyword) | |
574 | } | |
575 | ||
576 | /// Returns `true` if the token is a keyword reserved for possible future use. | |
577 | pub fn is_unused_keyword(&self) -> bool { | |
578 | self.is_non_raw_ident_where(Ident::is_unused_keyword) | |
579 | } | |
580 | ||
581 | /// Returns `true` if the token is either a special identifier or a keyword. | |
582 | pub fn is_reserved_ident(&self) -> bool { | |
583 | self.is_non_raw_ident_where(Ident::is_reserved) | |
584 | } | |
585 | ||
586 | /// Returns `true` if the token is the identifier `true` or `false`. | |
587 | pub fn is_bool_lit(&self) -> bool { | |
588 | self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) | |
589 | } | |
590 | ||
591 | /// Returns `true` if the token is a non-raw identifier for which `pred` holds. | |
592 | pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool { | |
593 | match self.ident() { | |
594 | Some((id, false)) => pred(id), | |
595 | _ => false, | |
596 | } | |
597 | } | |
598 | ||
599 | pub fn glue(&self, joint: &Token) -> Option<Token> { | |
600 | let kind = match self.kind { | |
601 | Eq => match joint.kind { | |
602 | Eq => EqEq, | |
603 | Gt => FatArrow, | |
604 | _ => return None, | |
605 | }, | |
606 | Lt => match joint.kind { | |
607 | Eq => Le, | |
608 | Lt => BinOp(Shl), | |
609 | Le => BinOpEq(Shl), | |
610 | BinOp(Minus) => LArrow, | |
611 | _ => return None, | |
612 | }, | |
613 | Gt => match joint.kind { | |
614 | Eq => Ge, | |
615 | Gt => BinOp(Shr), | |
616 | Ge => BinOpEq(Shr), | |
617 | _ => return None, | |
618 | }, | |
619 | Not => match joint.kind { | |
620 | Eq => Ne, | |
621 | _ => return None, | |
622 | }, | |
623 | BinOp(op) => match joint.kind { | |
624 | Eq => BinOpEq(op), | |
625 | BinOp(And) if op == And => AndAnd, | |
626 | BinOp(Or) if op == Or => OrOr, | |
627 | Gt if op == Minus => RArrow, | |
628 | _ => return None, | |
629 | }, | |
630 | Dot => match joint.kind { | |
631 | Dot => DotDot, | |
632 | DotDot => DotDotDot, | |
633 | _ => return None, | |
634 | }, | |
635 | DotDot => match joint.kind { | |
636 | Dot => DotDotDot, | |
637 | Eq => DotDotEq, | |
638 | _ => return None, | |
639 | }, | |
640 | Colon => match joint.kind { | |
641 | Colon => ModSep, | |
642 | _ => return None, | |
643 | }, | |
644 | SingleQuote => match joint.kind { | |
645 | Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), | |
646 | _ => return None, | |
647 | }, | |
648 | ||
649 | Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot | |
650 | | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | |
651 | | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | |
652 | | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None, | |
653 | }; | |
654 | ||
655 | Some(Token::new(kind, self.span.to(joint.span))) | |
656 | } | |
657 | } | |
658 | ||
659 | impl PartialEq<TokenKind> for Token { | |
660 | fn eq(&self, rhs: &TokenKind) -> bool { | |
661 | self.kind == *rhs | |
662 | } | |
663 | } | |
664 | ||
665 | #[derive(Clone, Encodable, Decodable)] | |
666 | /// For interpolation during macro expansion. | |
667 | pub enum Nonterminal { | |
668 | NtItem(P<ast::Item>), | |
669 | NtBlock(P<ast::Block>), | |
670 | NtStmt(ast::Stmt), | |
671 | NtPat(P<ast::Pat>), | |
672 | NtExpr(P<ast::Expr>), | |
673 | NtTy(P<ast::Ty>), | |
674 | NtIdent(Ident, /* is_raw */ bool), | |
675 | NtLifetime(Ident), | |
676 | NtLiteral(P<ast::Expr>), | |
677 | /// Stuff inside brackets for attributes | |
678 | NtMeta(P<ast::AttrItem>), | |
679 | NtPath(ast::Path), | |
680 | NtVis(ast::Visibility), | |
681 | NtTT(TokenTree), | |
682 | } | |
683 | ||
684 | // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger. | |
685 | #[cfg(target_arch = "x86_64")] | |
686 | rustc_data_structures::static_assert_size!(Nonterminal, 48); | |
687 | ||
688 | #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)] | |
689 | pub enum NonterminalKind { | |
690 | Item, | |
691 | Block, | |
692 | Stmt, | |
693 | Pat2018 { | |
694 | /// Keep track of whether the user used `:pat2018` or `:pat` and we inferred it from the | |
695 | /// edition of the span. This is used for diagnostics. | |
696 | inferred: bool, | |
697 | }, | |
698 | Pat2021 { | |
699 | /// Keep track of whether the user used `:pat2018` or `:pat` and we inferred it from the | |
700 | /// edition of the span. This is used for diagnostics. | |
701 | inferred: bool, | |
702 | }, | |
703 | Expr, | |
704 | Ty, | |
705 | Ident, | |
706 | Lifetime, | |
707 | Literal, | |
708 | Meta, | |
709 | Path, | |
710 | Vis, | |
711 | TT, | |
712 | } | |
713 | ||
714 | impl NonterminalKind { | |
715 | /// The `edition` closure is used to get the edition for the given symbol. Doing | |
716 | /// `span.edition()` is expensive, so we do it lazily. | |
717 | pub fn from_symbol( | |
718 | symbol: Symbol, | |
719 | edition: impl FnOnce() -> Edition, | |
720 | ) -> Option<NonterminalKind> { | |
721 | Some(match symbol { | |
722 | sym::item => NonterminalKind::Item, | |
723 | sym::block => NonterminalKind::Block, | |
724 | sym::stmt => NonterminalKind::Stmt, | |
725 | sym::pat => match edition() { | |
726 | Edition::Edition2015 | Edition::Edition2018 => { | |
727 | NonterminalKind::Pat2018 { inferred: true } | |
728 | } | |
729 | Edition::Edition2021 => NonterminalKind::Pat2021 { inferred: true }, | |
730 | }, | |
731 | sym::pat2018 => NonterminalKind::Pat2018 { inferred: false }, | |
732 | sym::pat2021 => NonterminalKind::Pat2021 { inferred: false }, | |
733 | sym::expr => NonterminalKind::Expr, | |
734 | sym::ty => NonterminalKind::Ty, | |
735 | sym::ident => NonterminalKind::Ident, | |
736 | sym::lifetime => NonterminalKind::Lifetime, | |
737 | sym::literal => NonterminalKind::Literal, | |
738 | sym::meta => NonterminalKind::Meta, | |
739 | sym::path => NonterminalKind::Path, | |
740 | sym::vis => NonterminalKind::Vis, | |
741 | sym::tt => NonterminalKind::TT, | |
742 | _ => return None, | |
743 | }) | |
744 | } | |
745 | fn symbol(self) -> Symbol { | |
746 | match self { | |
747 | NonterminalKind::Item => sym::item, | |
748 | NonterminalKind::Block => sym::block, | |
749 | NonterminalKind::Stmt => sym::stmt, | |
750 | NonterminalKind::Pat2018 { inferred: false } => sym::pat2018, | |
751 | NonterminalKind::Pat2021 { inferred: false } => sym::pat2021, | |
752 | NonterminalKind::Pat2018 { inferred: true } | |
753 | | NonterminalKind::Pat2021 { inferred: true } => sym::pat, | |
754 | NonterminalKind::Expr => sym::expr, | |
755 | NonterminalKind::Ty => sym::ty, | |
756 | NonterminalKind::Ident => sym::ident, | |
757 | NonterminalKind::Lifetime => sym::lifetime, | |
758 | NonterminalKind::Literal => sym::literal, | |
759 | NonterminalKind::Meta => sym::meta, | |
760 | NonterminalKind::Path => sym::path, | |
761 | NonterminalKind::Vis => sym::vis, | |
762 | NonterminalKind::TT => sym::tt, | |
763 | } | |
764 | } | |
765 | } | |
766 | ||
767 | impl fmt::Display for NonterminalKind { | |
768 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
769 | write!(f, "{}", self.symbol()) | |
770 | } | |
771 | } | |
772 | ||
773 | impl Nonterminal { | |
774 | pub fn span(&self) -> Span { | |
775 | match self { | |
776 | NtItem(item) => item.span, | |
777 | NtBlock(block) => block.span, | |
778 | NtStmt(stmt) => stmt.span, | |
779 | NtPat(pat) => pat.span, | |
780 | NtExpr(expr) | NtLiteral(expr) => expr.span, | |
781 | NtTy(ty) => ty.span, | |
782 | NtIdent(ident, _) | NtLifetime(ident) => ident.span, | |
783 | NtMeta(attr_item) => attr_item.span(), | |
784 | NtPath(path) => path.span, | |
785 | NtVis(vis) => vis.span, | |
786 | NtTT(tt) => tt.span(), | |
787 | } | |
788 | } | |
789 | ||
790 | /// This nonterminal looks like some specific enums from | |
791 | /// `proc-macro-hack` and `procedural-masquerade` crates. | |
792 | /// We need to maintain some special pretty-printing behavior for them due to incorrect | |
793 | /// asserts in old versions of those crates and their wide use in the ecosystem. | |
794 | /// See issue #73345 for more details. | |
795 | /// FIXME(#73933): Remove this eventually. | |
796 | pub fn pretty_printing_compatibility_hack(&self) -> bool { | |
797 | let item = match self { | |
798 | NtItem(item) => item, | |
799 | NtStmt(stmt) => match &stmt.kind { | |
800 | ast::StmtKind::Item(item) => item, | |
801 | _ => return false, | |
802 | }, | |
803 | _ => return false, | |
804 | }; | |
805 | ||
806 | let name = item.ident.name; | |
807 | if name == sym::ProceduralMasqueradeDummyType || name == sym::ProcMacroHack { | |
808 | if let ast::ItemKind::Enum(enum_def, _) = &item.kind { | |
809 | if let [variant] = &*enum_def.variants { | |
810 | return variant.ident.name == sym::Input; | |
811 | } | |
812 | } | |
813 | } | |
814 | false | |
815 | } | |
816 | ||
817 | // See issue #74616 for details | |
818 | pub fn ident_name_compatibility_hack( | |
819 | &self, | |
820 | orig_span: Span, | |
821 | source_map: &SourceMap, | |
822 | ) -> Option<(Ident, bool)> { | |
823 | if let NtIdent(ident, is_raw) = self { | |
824 | if let ExpnKind::Macro(_, macro_name) = orig_span.ctxt().outer_expn_data().kind { | |
825 | let filename = source_map.span_to_filename(orig_span); | |
826 | if let FileName::Real(RealFileName::Named(path)) = filename { | |
827 | let matches_prefix = |prefix, filename| { | |
828 | // Check for a path that ends with 'prefix*/src/<filename>' | |
829 | let mut iter = path.components().rev(); | |
830 | iter.next().and_then(|p| p.as_os_str().to_str()) == Some(filename) | |
831 | && iter.next().and_then(|p| p.as_os_str().to_str()) == Some("src") | |
832 | && iter | |
833 | .next() | |
834 | .and_then(|p| p.as_os_str().to_str()) | |
835 | .map_or(false, |p| p.starts_with(prefix)) | |
836 | }; | |
837 | ||
838 | if (macro_name == sym::impl_macros | |
839 | && matches_prefix("time-macros-impl", "lib.rs")) | |
840 | || (macro_name == sym::arrays && matches_prefix("js-sys", "lib.rs")) | |
841 | { | |
842 | let snippet = source_map.span_to_snippet(orig_span); | |
843 | if snippet.as_deref() == Ok("$name") { | |
844 | return Some((*ident, *is_raw)); | |
845 | } | |
846 | } | |
847 | ||
848 | if macro_name == sym::tuple_from_req | |
849 | && (matches_prefix("actix-web", "extract.rs") | |
850 | || matches_prefix("actori-web", "extract.rs")) | |
851 | { | |
852 | let snippet = source_map.span_to_snippet(orig_span); | |
853 | if snippet.as_deref() == Ok("$T") { | |
854 | return Some((*ident, *is_raw)); | |
855 | } | |
856 | } | |
857 | } | |
858 | } | |
859 | } | |
860 | None | |
861 | } | |
862 | } | |
863 | ||
864 | impl PartialEq for Nonterminal { | |
865 | fn eq(&self, rhs: &Self) -> bool { | |
866 | match (self, rhs) { | |
867 | (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { | |
868 | ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs | |
869 | } | |
870 | (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, | |
871 | (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, | |
872 | // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them | |
873 | // correctly based on data from AST. This will prevent them from matching each other | |
874 | // in macros. The comparison will become possible only when each nonterminal has an | |
875 | // attached token stream from which it was parsed. | |
876 | _ => false, | |
877 | } | |
878 | } | |
879 | } | |
880 | ||
881 | impl fmt::Debug for Nonterminal { | |
882 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
883 | match *self { | |
884 | NtItem(..) => f.pad("NtItem(..)"), | |
885 | NtBlock(..) => f.pad("NtBlock(..)"), | |
886 | NtStmt(..) => f.pad("NtStmt(..)"), | |
887 | NtPat(..) => f.pad("NtPat(..)"), | |
888 | NtExpr(..) => f.pad("NtExpr(..)"), | |
889 | NtTy(..) => f.pad("NtTy(..)"), | |
890 | NtIdent(..) => f.pad("NtIdent(..)"), | |
891 | NtLiteral(..) => f.pad("NtLiteral(..)"), | |
892 | NtMeta(..) => f.pad("NtMeta(..)"), | |
893 | NtPath(..) => f.pad("NtPath(..)"), | |
894 | NtTT(..) => f.pad("NtTT(..)"), | |
895 | NtVis(..) => f.pad("NtVis(..)"), | |
896 | NtLifetime(..) => f.pad("NtLifetime(..)"), | |
897 | } | |
898 | } | |
899 | } | |
900 | ||
901 | impl<CTX> HashStable<CTX> for Nonterminal | |
902 | where | |
903 | CTX: crate::HashStableContext, | |
904 | { | |
905 | fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { | |
906 | panic!("interpolated tokens should not be present in the HIR") | |
907 | } | |
908 | } |