]>
Commit | Line | Data |
---|---|---|
9fa01778 | 1 | pub use BinOpToken::*; |
9fa01778 | 2 | pub use DelimToken::*; |
dc9dc135 | 3 | pub use LitKind::*; |
dfeec247 | 4 | pub use Nonterminal::*; |
dc9dc135 | 5 | pub use TokenKind::*; |
9fa01778 | 6 | |
e74abb32 | 7 | use crate::ast; |
9fa01778 | 8 | use crate::ptr::P; |
e74abb32 | 9 | use crate::tokenstream::TokenTree; |
9fa01778 | 10 | |
60c5eb7d | 11 | use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; |
9fa01778 | 12 | use rustc_data_structures::sync::Lrc; |
60c5eb7d | 13 | use rustc_macros::HashStable_Generic; |
f035d41b | 14 | use rustc_span::symbol::{kw, sym}; |
f9f354fc | 15 | use rustc_span::symbol::{Ident, Symbol}; |
6a06907d | 16 | use rustc_span::{self, edition::Edition, Span, DUMMY_SP}; |
74b04a01 XL |
17 | use std::borrow::Cow; |
18 | use std::{fmt, mem}; | |
1a4d82fc | 19 | |
3dfed10e XL |
20 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
21 | pub enum CommentKind { | |
22 | Line, | |
23 | Block, | |
24 | } | |
25 | ||
26 | #[derive(Clone, PartialEq, Encodable, Decodable, Hash, Debug, Copy)] | |
60c5eb7d | 27 | #[derive(HashStable_Generic)] |
1a4d82fc JJ |
28 | pub enum BinOpToken { |
29 | Plus, | |
30 | Minus, | |
31 | Star, | |
32 | Slash, | |
33 | Percent, | |
34 | Caret, | |
35 | And, | |
36 | Or, | |
37 | Shl, | |
38 | Shr, | |
39 | } | |
40 | ||
9fa01778 | 41 | /// A delimiter token. |
3dfed10e | 42 | #[derive(Clone, PartialEq, Eq, Encodable, Decodable, Hash, Debug, Copy)] |
60c5eb7d | 43 | #[derive(HashStable_Generic)] |
1a4d82fc | 44 | pub enum DelimToken { |
9fa01778 | 45 | /// A round parenthesis (i.e., `(` or `)`). |
1a4d82fc | 46 | Paren, |
9fa01778 | 47 | /// A square bracket (i.e., `[` or `]`). |
1a4d82fc | 48 | Bracket, |
9fa01778 | 49 | /// A curly brace (i.e., `{` or `}`). |
1a4d82fc | 50 | Brace, |
9fa01778 | 51 | /// An empty delimiter. |
5bcae85e | 52 | NoDelim, |
1a4d82fc JJ |
53 | } |
54 | ||
3dfed10e | 55 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
56 | pub enum LitKind { |
57 | Bool, // AST only, must never appear in a `Token` | |
58 | Byte, | |
59 | Char, | |
60 | Integer, | |
61 | Float, | |
62 | Str, | |
63 | StrRaw(u16), // raw string delimited by `n` hash symbols | |
64 | ByteStr, | |
65 | ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols | |
66 | Err, | |
1a4d82fc JJ |
67 | } |
68 | ||
dc9dc135 | 69 | /// A literal token. |
3dfed10e | 70 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
71 | pub struct Lit { |
72 | pub kind: LitKind, | |
73 | pub symbol: Symbol, | |
74 | pub suffix: Option<Symbol>, | |
75 | } | |
48663c56 | 76 | |
416331ca XL |
77 | impl fmt::Display for Lit { |
78 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
79 | let Lit { kind, symbol, suffix } = *self; | |
80 | match kind { | |
dfeec247 XL |
81 | Byte => write!(f, "b'{}'", symbol)?, |
82 | Char => write!(f, "'{}'", symbol)?, | |
83 | Str => write!(f, "\"{}\"", symbol)?, | |
84 | StrRaw(n) => write!( | |
85 | f, | |
86 | "r{delim}\"{string}\"{delim}", | |
87 | delim = "#".repeat(n as usize), | |
88 | string = symbol | |
89 | )?, | |
90 | ByteStr => write!(f, "b\"{}\"", symbol)?, | |
91 | ByteStrRaw(n) => write!( | |
92 | f, | |
93 | "br{delim}\"{string}\"{delim}", | |
94 | delim = "#".repeat(n as usize), | |
95 | string = symbol | |
96 | )?, | |
97 | Integer | Float | Bool | Err => write!(f, "{}", symbol)?, | |
416331ca XL |
98 | } |
99 | ||
100 | if let Some(suffix) = suffix { | |
101 | write!(f, "{}", suffix)?; | |
102 | } | |
103 | ||
104 | Ok(()) | |
105 | } | |
106 | } | |
107 | ||
dc9dc135 XL |
108 | impl LitKind { |
109 | /// An English article for the literal token kind. | |
60c5eb7d | 110 | pub fn article(self) -> &'static str { |
dc9dc135 XL |
111 | match self { |
112 | Integer | Err => "an", | |
113 | _ => "a", | |
1a4d82fc JJ |
114 | } |
115 | } | |
83c7162d | 116 | |
60c5eb7d | 117 | pub fn descr(self) -> &'static str { |
dc9dc135 XL |
118 | match self { |
119 | Bool => panic!("literal token contains `Lit::Bool`"), | |
120 | Byte => "byte", | |
121 | Char => "char", | |
122 | Integer => "integer", | |
123 | Float => "float", | |
124 | Str | StrRaw(..) => "string", | |
125 | ByteStr | ByteStrRaw(..) => "byte string", | |
126 | Err => "error", | |
127 | } | |
128 | } | |
129 | ||
130 | crate fn may_have_suffix(self) -> bool { | |
5869c6ff | 131 | matches!(self, Integer | Float | Err) |
48663c56 | 132 | } |
dc9dc135 | 133 | } |
48663c56 | 134 | |
dc9dc135 XL |
135 | impl Lit { |
136 | pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit { | |
137 | Lit { kind, symbol, suffix } | |
83c7162d | 138 | } |
1a4d82fc JJ |
139 | } |
140 | ||
f9f354fc | 141 | pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool { |
dc9dc135 | 142 | let ident_token = Token::new(Ident(name, is_raw), span); |
32a655c1 | 143 | |
dfeec247 XL |
144 | !ident_token.is_reserved_ident() |
145 | || ident_token.is_path_segment_keyword() | |
74b04a01 XL |
146 | || [ |
147 | kw::Async, | |
148 | kw::Do, | |
149 | kw::Box, | |
150 | kw::Break, | |
29967ef6 | 151 | kw::Const, |
74b04a01 XL |
152 | kw::Continue, |
153 | kw::False, | |
154 | kw::For, | |
155 | kw::If, | |
156 | kw::Let, | |
157 | kw::Loop, | |
158 | kw::Match, | |
159 | kw::Move, | |
160 | kw::Return, | |
161 | kw::True, | |
1b1a35ee | 162 | kw::Try, |
74b04a01 XL |
163 | kw::Unsafe, |
164 | kw::While, | |
165 | kw::Yield, | |
166 | kw::Static, | |
167 | ] | |
168 | .contains(&name) | |
32a655c1 SL |
169 | } |
170 | ||
f9f354fc | 171 | fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool { |
dc9dc135 | 172 | let ident_token = Token::new(Ident(name, is_raw), span); |
cc61c64b | 173 | |
dfeec247 XL |
174 | !ident_token.is_reserved_ident() |
175 | || ident_token.is_path_segment_keyword() | |
176 | || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn] | |
177 | .contains(&name) | |
cc61c64b XL |
178 | } |
179 | ||
3dfed10e | 180 | #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 | 181 | pub enum TokenKind { |
223e47cc | 182 | /* Expression-operator symbols. */ |
1a4d82fc JJ |
183 | Eq, |
184 | Lt, | |
185 | Le, | |
186 | EqEq, | |
187 | Ne, | |
188 | Ge, | |
189 | Gt, | |
190 | AndAnd, | |
191 | OrOr, | |
192 | Not, | |
193 | Tilde, | |
194 | BinOp(BinOpToken), | |
195 | BinOpEq(BinOpToken), | |
223e47cc LB |
196 | |
197 | /* Structural symbols */ | |
1a4d82fc JJ |
198 | At, |
199 | Dot, | |
200 | DotDot, | |
201 | DotDotDot, | |
ea8adc8c | 202 | DotDotEq, |
1a4d82fc JJ |
203 | Comma, |
204 | Semi, | |
205 | Colon, | |
206 | ModSep, | |
207 | RArrow, | |
208 | LArrow, | |
209 | FatArrow, | |
210 | Pound, | |
211 | Dollar, | |
212 | Question, | |
94b46f34 XL |
213 | /// Used by proc macros for representing lifetimes, not generated by lexer right now. |
214 | SingleQuote, | |
9fa01778 | 215 | /// An opening delimiter (e.g., `{`). |
1a4d82fc | 216 | OpenDelim(DelimToken), |
9fa01778 | 217 | /// A closing delimiter (e.g., `}`). |
1a4d82fc | 218 | CloseDelim(DelimToken), |
223e47cc LB |
219 | |
220 | /* Literals */ | |
dc9dc135 | 221 | Literal(Lit), |
223e47cc | 222 | |
74b04a01 XL |
223 | /// Identifier token. |
224 | /// Do not forget about `NtIdent` when you want to match on identifiers. | |
225 | /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to | |
226 | /// treat regular and interpolated identifiers in the same way. | |
f9f354fc | 227 | Ident(Symbol, /* is_raw */ bool), |
74b04a01 XL |
228 | /// Lifetime identifier token. |
229 | /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. | |
230 | /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to | |
231 | /// treat regular and interpolated lifetime identifiers in the same way. | |
f9f354fc | 232 | Lifetime(Symbol), |
223e47cc | 233 | |
9fa01778 XL |
234 | Interpolated(Lrc<Nonterminal>), |
235 | ||
3dfed10e XL |
236 | /// A doc comment token. |
237 | /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc) | |
238 | /// similarly to symbols in string literal tokens. | |
239 | DocComment(CommentKind, ast::AttrStyle, Symbol), | |
1a4d82fc | 240 | |
1a4d82fc JJ |
241 | Eof, |
242 | } | |
243 | ||
dc9dc135 | 244 | // `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. |
6a06907d | 245 | #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] |
60c5eb7d | 246 | rustc_data_structures::static_assert_size!(TokenKind, 16); |
dc9dc135 | 247 | |
3dfed10e | 248 | #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
249 | pub struct Token { |
250 | pub kind: TokenKind, | |
251 | pub span: Span, | |
252 | } | |
253 | ||
254 | impl TokenKind { | |
255 | pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind { | |
256 | Literal(Lit::new(kind, symbol, suffix)) | |
257 | } | |
258 | ||
74b04a01 XL |
259 | // An approximation to proc-macro-style single-character operators used by rustc parser. |
260 | // If the operator token can be broken into two tokens, the first of which is single-character, | |
261 | // then this function performs that operation, otherwise it returns `None`. | |
262 | pub fn break_two_token_op(&self) -> Option<(TokenKind, TokenKind)> { | |
263 | Some(match *self { | |
264 | Le => (Lt, Eq), | |
265 | EqEq => (Eq, Eq), | |
266 | Ne => (Not, Eq), | |
267 | Ge => (Gt, Eq), | |
268 | AndAnd => (BinOp(And), BinOp(And)), | |
269 | OrOr => (BinOp(Or), BinOp(Or)), | |
270 | BinOp(Shl) => (Lt, Lt), | |
271 | BinOp(Shr) => (Gt, Gt), | |
272 | BinOpEq(Plus) => (BinOp(Plus), Eq), | |
273 | BinOpEq(Minus) => (BinOp(Minus), Eq), | |
274 | BinOpEq(Star) => (BinOp(Star), Eq), | |
275 | BinOpEq(Slash) => (BinOp(Slash), Eq), | |
276 | BinOpEq(Percent) => (BinOp(Percent), Eq), | |
277 | BinOpEq(Caret) => (BinOp(Caret), Eq), | |
278 | BinOpEq(And) => (BinOp(And), Eq), | |
279 | BinOpEq(Or) => (BinOp(Or), Eq), | |
280 | BinOpEq(Shl) => (Lt, Le), | |
281 | BinOpEq(Shr) => (Gt, Ge), | |
282 | DotDot => (Dot, Dot), | |
283 | DotDotDot => (Dot, DotDot), | |
284 | ModSep => (Colon, Colon), | |
285 | RArrow => (BinOp(Minus), Gt), | |
286 | LArrow => (Lt, BinOp(Minus)), | |
287 | FatArrow => (Eq, Gt), | |
288 | _ => return None, | |
289 | }) | |
290 | } | |
291 | ||
dc9dc135 XL |
292 | /// Returns tokens that are likely to be typed accidentally instead of the current token. |
293 | /// Enables better error recovery when the wrong token is found. | |
60c5eb7d | 294 | pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> { |
dc9dc135 XL |
295 | match *self { |
296 | Comma => Some(vec![Dot, Lt, Semi]), | |
297 | Semi => Some(vec![Colon, Comma]), | |
dfeec247 | 298 | _ => None, |
dc9dc135 XL |
299 | } |
300 | } | |
29967ef6 XL |
301 | |
302 | pub fn should_end_const_arg(&self) -> bool { | |
5869c6ff | 303 | matches!(self, Gt | Ge | BinOp(Shr) | BinOpEq(Shr)) |
29967ef6 | 304 | } |
dc9dc135 | 305 | } |
0731742a | 306 | |
1a4d82fc | 307 | impl Token { |
e74abb32 | 308 | pub fn new(kind: TokenKind, span: Span) -> Self { |
dc9dc135 XL |
309 | Token { kind, span } |
310 | } | |
311 | ||
312 | /// Some token that will be thrown away later. | |
60c5eb7d | 313 | pub fn dummy() -> Self { |
1b1a35ee | 314 | Token::new(TokenKind::Question, DUMMY_SP) |
dc9dc135 XL |
315 | } |
316 | ||
f9f354fc XL |
317 | /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary. |
318 | pub fn from_ast_ident(ident: Ident) -> Self { | |
dc9dc135 XL |
319 | Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) |
320 | } | |
321 | ||
322 | /// Return this token by value and leave a dummy token in its place. | |
e74abb32 | 323 | pub fn take(&mut self) -> Self { |
dc9dc135 XL |
324 | mem::replace(self, Token::dummy()) |
325 | } | |
326 | ||
74b04a01 XL |
327 | /// For interpolated tokens, returns a span of the fragment to which the interpolated |
328 | /// token refers. For all other tokens this is just a regular span. | |
329 | /// It is particularly important to use this for identifiers and lifetimes | |
330 | /// for which spans affect name resolution and edition checks. | |
331 | /// Note that keywords are also identifiers, so they should use this | |
332 | /// if they keep spans or perform edition checks. | |
333 | pub fn uninterpolated_span(&self) -> Span { | |
334 | match &self.kind { | |
335 | Interpolated(nt) => nt.span(), | |
336 | _ => self.span, | |
337 | } | |
338 | } | |
339 | ||
60c5eb7d | 340 | pub fn is_op(&self) -> bool { |
5869c6ff XL |
341 | !matches!( |
342 | self.kind, | |
343 | OpenDelim(..) | |
344 | | CloseDelim(..) | |
345 | | Literal(..) | |
346 | | DocComment(..) | |
347 | | Ident(..) | |
348 | | Lifetime(..) | |
349 | | Interpolated(..) | |
350 | | Eof | |
351 | ) | |
041b39d2 XL |
352 | } |
353 | ||
60c5eb7d | 354 | pub fn is_like_plus(&self) -> bool { |
5869c6ff | 355 | matches!(self.kind, BinOp(Plus) | BinOpEq(Plus)) |
bd371182 AL |
356 | } |
357 | ||
1a4d82fc | 358 | /// Returns `true` if the token can appear at the start of an expression. |
e74abb32 | 359 | pub fn can_begin_expr(&self) -> bool { |
74b04a01 | 360 | match self.uninterpolate().kind { |
dc9dc135 XL |
361 | Ident(name, is_raw) => |
362 | ident_can_begin_expr(name, self.span, is_raw), // value name or keyword | |
ea8adc8c XL |
363 | OpenDelim(..) | // tuple, array or block |
364 | Literal(..) | // literal | |
365 | Not | // operator not | |
366 | BinOp(Minus) | // unary minus | |
367 | BinOp(Star) | // dereference | |
368 | BinOp(Or) | OrOr | // closure | |
369 | BinOp(And) | // reference | |
370 | AndAnd | // double reference | |
abe05a73 | 371 | // DotDotDot is no longer supported, but we need some way to display the error |
ea8adc8c | 372 | DotDot | DotDotDot | DotDotEq | // range notation |
ea8adc8c XL |
373 | Lt | BinOp(Shl) | // associated path |
374 | ModSep | // global path | |
83c7162d | 375 | Lifetime(..) | // labeled loop |
ea8adc8c | 376 | Pound => true, // expression attributes |
5869c6ff | 377 | Interpolated(ref nt) => matches!(**nt, NtLiteral(..) | |
94b46f34 XL |
378 | NtExpr(..) | |
379 | NtBlock(..) | | |
5869c6ff | 380 | NtPath(..)), |
c30ab7b3 | 381 | _ => false, |
1a4d82fc JJ |
382 | } |
383 | } | |
223e47cc | 384 | |
32a655c1 | 385 | /// Returns `true` if the token can appear at the start of a type. |
e74abb32 | 386 | pub fn can_begin_type(&self) -> bool { |
74b04a01 | 387 | match self.uninterpolate().kind { |
dc9dc135 XL |
388 | Ident(name, is_raw) => |
389 | ident_can_begin_type(name, self.span, is_raw), // type name or keyword | |
7cac9316 XL |
390 | OpenDelim(Paren) | // tuple |
391 | OpenDelim(Bracket) | // array | |
7cac9316 XL |
392 | Not | // never |
393 | BinOp(Star) | // raw pointer | |
394 | BinOp(And) | // reference | |
395 | AndAnd | // double reference | |
396 | Question | // maybe bound in trait object | |
397 | Lifetime(..) | // lifetime bound in trait object | |
398 | Lt | BinOp(Shl) | // associated path | |
32a655c1 | 399 | ModSep => true, // global path |
5869c6ff | 400 | Interpolated(ref nt) => matches!(**nt, NtTy(..) | NtPath(..)), |
32a655c1 SL |
401 | _ => false, |
402 | } | |
403 | } | |
404 | ||
9fa01778 | 405 | /// Returns `true` if the token can appear at the start of a const param. |
60c5eb7d | 406 | pub fn can_begin_const_arg(&self) -> bool { |
dc9dc135 | 407 | match self.kind { |
9fa01778 | 408 | OpenDelim(Brace) => true, |
5869c6ff | 409 | Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)), |
74b04a01 | 410 | _ => self.can_begin_literal_maybe_minus(), |
9fa01778 XL |
411 | } |
412 | } | |
413 | ||
abe05a73 | 414 | /// Returns `true` if the token can appear at the start of a generic bound. |
60c5eb7d | 415 | pub fn can_begin_bound(&self) -> bool { |
dfeec247 XL |
416 | self.is_path_start() |
417 | || self.is_lifetime() | |
418 | || self.is_keyword(kw::For) | |
419 | || self == &Question | |
420 | || self == &OpenDelim(Paren) | |
abe05a73 XL |
421 | } |
422 | ||
fc512014 | 423 | /// Returns `true` if the token is any literal. |
e74abb32 | 424 | pub fn is_lit(&self) -> bool { |
5869c6ff | 425 | matches!(self.kind, Literal(..)) |
1a4d82fc | 426 | } |
223e47cc | 427 | |
9fa01778 | 428 | /// Returns `true` if the token is any literal, a minus (which can prefix a literal, |
94b46f34 | 429 | /// for example a '-42', or one of the boolean idents). |
74b04a01 XL |
430 | /// |
431 | /// In other words, would this token be a valid start of `parse_literal_maybe_minus`? | |
432 | /// | |
433 | /// Keep this in sync with and `Lit::from_token`, excluding unary negation. | |
434 | pub fn can_begin_literal_maybe_minus(&self) -> bool { | |
435 | match self.uninterpolate().kind { | |
e1599b0c XL |
436 | Literal(..) | BinOp(Minus) => true, |
437 | Ident(name, false) if name.is_bool_lit() => true, | |
74b04a01 XL |
438 | Interpolated(ref nt) => match &**nt { |
439 | NtLiteral(_) => true, | |
440 | NtExpr(e) => match &e.kind { | |
441 | ast::ExprKind::Lit(_) => true, | |
442 | ast::ExprKind::Unary(ast::UnOp::Neg, e) => { | |
443 | matches!(&e.kind, ast::ExprKind::Lit(_)) | |
444 | } | |
445 | _ => false, | |
446 | }, | |
dfeec247 | 447 | _ => false, |
b7449926 | 448 | }, |
dfeec247 | 449 | _ => false, |
94b46f34 XL |
450 | } |
451 | } | |
452 | ||
74b04a01 XL |
453 | // A convenience function for matching on identifiers during parsing. |
454 | // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token | |
455 | // into the regular identifier or lifetime token it refers to, | |
456 | // otherwise returns the original token. | |
457 | pub fn uninterpolate(&self) -> Cow<'_, Token> { | |
458 | match &self.kind { | |
459 | Interpolated(nt) => match **nt { | |
460 | NtIdent(ident, is_raw) => { | |
461 | Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)) | |
462 | } | |
463 | NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), | |
464 | _ => Cow::Borrowed(self), | |
465 | }, | |
466 | _ => Cow::Borrowed(self), | |
467 | } | |
468 | } | |
469 | ||
83c7162d | 470 | /// Returns an identifier if this token is an identifier. |
f9f354fc | 471 | pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> { |
74b04a01 XL |
472 | let token = self.uninterpolate(); |
473 | match token.kind { | |
f9f354fc | 474 | Ident(name, is_raw) => Some((Ident::new(name, token.span), is_raw)), |
83c7162d XL |
475 | _ => None, |
476 | } | |
477 | } | |
dc9dc135 | 478 | |
83c7162d | 479 | /// Returns a lifetime identifier if this token is a lifetime. |
f9f354fc | 480 | pub fn lifetime(&self) -> Option<Ident> { |
74b04a01 XL |
481 | let token = self.uninterpolate(); |
482 | match token.kind { | |
f9f354fc | 483 | Lifetime(name) => Some(Ident::new(name, token.span)), |
cc61c64b | 484 | _ => None, |
223e47cc | 485 | } |
1a4d82fc | 486 | } |
dc9dc135 | 487 | |
cc61c64b XL |
488 | /// Returns `true` if the token is an identifier. |
489 | pub fn is_ident(&self) -> bool { | |
490 | self.ident().is_some() | |
491 | } | |
e1599b0c | 492 | |
83c7162d | 493 | /// Returns `true` if the token is a lifetime. |
60c5eb7d | 494 | pub fn is_lifetime(&self) -> bool { |
83c7162d XL |
495 | self.lifetime().is_some() |
496 | } | |
497 | ||
498 | /// Returns `true` if the token is a identifier whose name is the given | |
499 | /// string slice. | |
60c5eb7d | 500 | pub fn is_ident_named(&self, name: Symbol) -> bool { |
dc9dc135 | 501 | self.ident().map_or(false, |(ident, _)| ident.name == name) |
83c7162d | 502 | } |
cc61c64b | 503 | |
1a4d82fc | 504 | /// Returns `true` if the token is an interpolated path. |
94b46f34 | 505 | fn is_path(&self) -> bool { |
dc9dc135 | 506 | if let Interpolated(ref nt) = self.kind { |
9fa01778 | 507 | if let NtPath(..) = **nt { |
c30ab7b3 SL |
508 | return true; |
509 | } | |
1a4d82fc | 510 | } |
c30ab7b3 | 511 | false |
1a4d82fc JJ |
512 | } |
513 | ||
416331ca XL |
514 | /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? |
515 | /// That is, is this a pre-parsed expression dropped into the token stream | |
516 | /// (which happens while parsing the result of macro expansion)? | |
60c5eb7d | 517 | pub fn is_whole_expr(&self) -> bool { |
416331ca XL |
518 | if let Interpolated(ref nt) = self.kind { |
519 | if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { | |
520 | return true; | |
521 | } | |
522 | } | |
523 | ||
524 | false | |
525 | } | |
526 | ||
ba9703b0 XL |
527 | // Is the token an interpolated block (`$b:block`)? |
528 | pub fn is_whole_block(&self) -> bool { | |
529 | if let Interpolated(ref nt) = self.kind { | |
530 | if let NtBlock(..) = **nt { | |
531 | return true; | |
532 | } | |
533 | } | |
534 | false | |
535 | } | |
536 | ||
1a4d82fc | 537 | /// Returns `true` if the token is either the `mut` or `const` keyword. |
60c5eb7d | 538 | pub fn is_mutability(&self) -> bool { |
dfeec247 | 539 | self.is_keyword(kw::Mut) || self.is_keyword(kw::Const) |
1a4d82fc JJ |
540 | } |
541 | ||
60c5eb7d | 542 | pub fn is_qpath_start(&self) -> bool { |
c30ab7b3 SL |
543 | self == &Lt || self == &BinOp(Shl) |
544 | } | |
545 | ||
60c5eb7d | 546 | pub fn is_path_start(&self) -> bool { |
dfeec247 XL |
547 | self == &ModSep |
548 | || self.is_qpath_start() | |
549 | || self.is_path() | |
550 | || self.is_path_segment_keyword() | |
551 | || self.is_ident() && !self.is_reserved_ident() | |
a7813a04 XL |
552 | } |
553 | ||
1a4d82fc | 554 | /// Returns `true` if the token is a given keyword, `kw`. |
dc9dc135 | 555 | pub fn is_keyword(&self, kw: Symbol) -> bool { |
e1599b0c | 556 | self.is_non_raw_ident_where(|id| id.name == kw) |
1a4d82fc JJ |
557 | } |
558 | ||
60c5eb7d | 559 | pub fn is_path_segment_keyword(&self) -> bool { |
f9f354fc | 560 | self.is_non_raw_ident_where(Ident::is_path_segment_keyword) |
1a4d82fc JJ |
561 | } |
562 | ||
041b39d2 XL |
563 | // Returns true for reserved identifiers used internally for elided lifetimes, |
564 | // unnamed method parameters, crate root module, error recovery etc. | |
60c5eb7d | 565 | pub fn is_special_ident(&self) -> bool { |
f9f354fc | 566 | self.is_non_raw_ident_where(Ident::is_special) |
1a4d82fc JJ |
567 | } |
568 | ||
041b39d2 | 569 | /// Returns `true` if the token is a keyword used in the language. |
60c5eb7d | 570 | pub fn is_used_keyword(&self) -> bool { |
f9f354fc | 571 | self.is_non_raw_ident_where(Ident::is_used_keyword) |
1a4d82fc JJ |
572 | } |
573 | ||
a7813a04 | 574 | /// Returns `true` if the token is a keyword reserved for possible future use. |
60c5eb7d | 575 | pub fn is_unused_keyword(&self) -> bool { |
f9f354fc | 576 | self.is_non_raw_ident_where(Ident::is_unused_keyword) |
1a4d82fc | 577 | } |
041b39d2 | 578 | |
83c7162d XL |
579 | /// Returns `true` if the token is either a special identifier or a keyword. |
580 | pub fn is_reserved_ident(&self) -> bool { | |
f9f354fc | 581 | self.is_non_raw_ident_where(Ident::is_reserved) |
e1599b0c XL |
582 | } |
583 | ||
584 | /// Returns `true` if the token is the identifier `true` or `false`. | |
60c5eb7d | 585 | pub fn is_bool_lit(&self) -> bool { |
e1599b0c XL |
586 | self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) |
587 | } | |
588 | ||
589 | /// Returns `true` if the token is a non-raw identifier for which `pred` holds. | |
f9f354fc | 590 | pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool { |
83c7162d | 591 | match self.ident() { |
e1599b0c | 592 | Some((id, false)) => pred(id), |
83c7162d XL |
593 | _ => false, |
594 | } | |
595 | } | |
596 | ||
60c5eb7d | 597 | pub fn glue(&self, joint: &Token) -> Option<Token> { |
dc9dc135 XL |
598 | let kind = match self.kind { |
599 | Eq => match joint.kind { | |
041b39d2 XL |
600 | Eq => EqEq, |
601 | Gt => FatArrow, | |
602 | _ => return None, | |
603 | }, | |
dc9dc135 | 604 | Lt => match joint.kind { |
041b39d2 XL |
605 | Eq => Le, |
606 | Lt => BinOp(Shl), | |
607 | Le => BinOpEq(Shl), | |
608 | BinOp(Minus) => LArrow, | |
609 | _ => return None, | |
610 | }, | |
dc9dc135 | 611 | Gt => match joint.kind { |
041b39d2 XL |
612 | Eq => Ge, |
613 | Gt => BinOp(Shr), | |
614 | Ge => BinOpEq(Shr), | |
615 | _ => return None, | |
616 | }, | |
dc9dc135 | 617 | Not => match joint.kind { |
041b39d2 XL |
618 | Eq => Ne, |
619 | _ => return None, | |
620 | }, | |
dc9dc135 | 621 | BinOp(op) => match joint.kind { |
041b39d2 XL |
622 | Eq => BinOpEq(op), |
623 | BinOp(And) if op == And => AndAnd, | |
624 | BinOp(Or) if op == Or => OrOr, | |
625 | Gt if op == Minus => RArrow, | |
626 | _ => return None, | |
627 | }, | |
dc9dc135 | 628 | Dot => match joint.kind { |
041b39d2 XL |
629 | Dot => DotDot, |
630 | DotDot => DotDotDot, | |
631 | _ => return None, | |
632 | }, | |
dc9dc135 | 633 | DotDot => match joint.kind { |
041b39d2 | 634 | Dot => DotDotDot, |
ea8adc8c | 635 | Eq => DotDotEq, |
041b39d2 XL |
636 | _ => return None, |
637 | }, | |
dc9dc135 | 638 | Colon => match joint.kind { |
041b39d2 XL |
639 | Colon => ModSep, |
640 | _ => return None, | |
641 | }, | |
dc9dc135 XL |
642 | SingleQuote => match joint.kind { |
643 | Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), | |
94b46f34 XL |
644 | _ => return None, |
645 | }, | |
041b39d2 | 646 | |
dfeec247 XL |
647 | Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot |
648 | | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | |
649 | | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | |
1b1a35ee | 650 | | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None, |
dc9dc135 | 651 | }; |
041b39d2 | 652 | |
dc9dc135 | 653 | Some(Token::new(kind, self.span.to(joint.span))) |
abe05a73 | 654 | } |
223e47cc LB |
655 | } |
656 | ||
dc9dc135 XL |
657 | impl PartialEq<TokenKind> for Token { |
658 | fn eq(&self, rhs: &TokenKind) -> bool { | |
659 | self.kind == *rhs | |
660 | } | |
661 | } | |
662 | ||
3dfed10e | 663 | #[derive(Clone, Encodable, Decodable)] |
1a4d82fc JJ |
664 | /// For interpolation during macro expansion. |
665 | pub enum Nonterminal { | |
666 | NtItem(P<ast::Item>), | |
667 | NtBlock(P<ast::Block>), | |
c30ab7b3 | 668 | NtStmt(ast::Stmt), |
1a4d82fc JJ |
669 | NtPat(P<ast::Pat>), |
670 | NtExpr(P<ast::Expr>), | |
671 | NtTy(P<ast::Ty>), | |
f9f354fc XL |
672 | NtIdent(Ident, /* is_raw */ bool), |
673 | NtLifetime(Ident), | |
94b46f34 | 674 | NtLiteral(P<ast::Expr>), |
1a4d82fc | 675 | /// Stuff inside brackets for attributes |
74b04a01 | 676 | NtMeta(P<ast::AttrItem>), |
c30ab7b3 | 677 | NtPath(ast::Path), |
cc61c64b XL |
678 | NtVis(ast::Visibility), |
679 | NtTT(TokenTree), | |
74b04a01 XL |
680 | } |
681 | ||
682 | // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger. | |
6a06907d | 683 | #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] |
1b1a35ee | 684 | rustc_data_structures::static_assert_size!(Nonterminal, 48); |
74b04a01 | 685 | |
3dfed10e XL |
686 | #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)] |
687 | pub enum NonterminalKind { | |
688 | Item, | |
689 | Block, | |
690 | Stmt, | |
5869c6ff XL |
691 | Pat2018 { |
692 | /// Keep track of whether the user used `:pat2018` or `:pat` and we inferred it from the | |
693 | /// edition of the span. This is used for diagnostics. | |
694 | inferred: bool, | |
695 | }, | |
696 | Pat2021 { | |
697 | /// Keep track of whether the user used `:pat2018` or `:pat` and we inferred it from the | |
698 | /// edition of the span. This is used for diagnostics. | |
699 | inferred: bool, | |
700 | }, | |
3dfed10e XL |
701 | Expr, |
702 | Ty, | |
703 | Ident, | |
704 | Lifetime, | |
705 | Literal, | |
706 | Meta, | |
707 | Path, | |
708 | Vis, | |
709 | TT, | |
710 | } | |
711 | ||
712 | impl NonterminalKind { | |
5869c6ff XL |
713 | /// The `edition` closure is used to get the edition for the given symbol. Doing |
714 | /// `span.edition()` is expensive, so we do it lazily. | |
715 | pub fn from_symbol( | |
716 | symbol: Symbol, | |
717 | edition: impl FnOnce() -> Edition, | |
718 | ) -> Option<NonterminalKind> { | |
3dfed10e XL |
719 | Some(match symbol { |
720 | sym::item => NonterminalKind::Item, | |
721 | sym::block => NonterminalKind::Block, | |
722 | sym::stmt => NonterminalKind::Stmt, | |
5869c6ff XL |
723 | sym::pat => match edition() { |
724 | Edition::Edition2015 | Edition::Edition2018 => { | |
725 | NonterminalKind::Pat2018 { inferred: true } | |
726 | } | |
727 | Edition::Edition2021 => NonterminalKind::Pat2021 { inferred: true }, | |
728 | }, | |
729 | sym::pat2018 => NonterminalKind::Pat2018 { inferred: false }, | |
730 | sym::pat2021 => NonterminalKind::Pat2021 { inferred: false }, | |
3dfed10e XL |
731 | sym::expr => NonterminalKind::Expr, |
732 | sym::ty => NonterminalKind::Ty, | |
733 | sym::ident => NonterminalKind::Ident, | |
734 | sym::lifetime => NonterminalKind::Lifetime, | |
735 | sym::literal => NonterminalKind::Literal, | |
736 | sym::meta => NonterminalKind::Meta, | |
737 | sym::path => NonterminalKind::Path, | |
738 | sym::vis => NonterminalKind::Vis, | |
739 | sym::tt => NonterminalKind::TT, | |
740 | _ => return None, | |
741 | }) | |
742 | } | |
743 | fn symbol(self) -> Symbol { | |
744 | match self { | |
745 | NonterminalKind::Item => sym::item, | |
746 | NonterminalKind::Block => sym::block, | |
747 | NonterminalKind::Stmt => sym::stmt, | |
5869c6ff XL |
748 | NonterminalKind::Pat2018 { inferred: false } => sym::pat2018, |
749 | NonterminalKind::Pat2021 { inferred: false } => sym::pat2021, | |
750 | NonterminalKind::Pat2018 { inferred: true } | |
751 | | NonterminalKind::Pat2021 { inferred: true } => sym::pat, | |
3dfed10e XL |
752 | NonterminalKind::Expr => sym::expr, |
753 | NonterminalKind::Ty => sym::ty, | |
754 | NonterminalKind::Ident => sym::ident, | |
755 | NonterminalKind::Lifetime => sym::lifetime, | |
756 | NonterminalKind::Literal => sym::literal, | |
757 | NonterminalKind::Meta => sym::meta, | |
758 | NonterminalKind::Path => sym::path, | |
759 | NonterminalKind::Vis => sym::vis, | |
760 | NonterminalKind::TT => sym::tt, | |
761 | } | |
762 | } | |
763 | } | |
764 | ||
765 | impl fmt::Display for NonterminalKind { | |
766 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
767 | write!(f, "{}", self.symbol()) | |
768 | } | |
769 | } | |
770 | ||
74b04a01 | 771 | impl Nonterminal { |
5869c6ff | 772 | pub fn span(&self) -> Span { |
74b04a01 XL |
773 | match self { |
774 | NtItem(item) => item.span, | |
775 | NtBlock(block) => block.span, | |
776 | NtStmt(stmt) => stmt.span, | |
777 | NtPat(pat) => pat.span, | |
778 | NtExpr(expr) | NtLiteral(expr) => expr.span, | |
779 | NtTy(ty) => ty.span, | |
780 | NtIdent(ident, _) | NtLifetime(ident) => ident.span, | |
781 | NtMeta(attr_item) => attr_item.span(), | |
782 | NtPath(path) => path.span, | |
783 | NtVis(vis) => vis.span, | |
784 | NtTT(tt) => tt.span(), | |
785 | } | |
786 | } | |
83c7162d XL |
787 | } |
788 | ||
789 | impl PartialEq for Nonterminal { | |
790 | fn eq(&self, rhs: &Self) -> bool { | |
791 | match (self, rhs) { | |
dfeec247 XL |
792 | (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { |
793 | ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs | |
794 | } | |
83c7162d XL |
795 | (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, |
796 | (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, | |
797 | // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them | |
798 | // correctly based on data from AST. This will prevent them from matching each other | |
799 | // in macros. The comparison will become possible only when each nonterminal has an | |
800 | // attached token stream from which it was parsed. | |
801 | _ => false, | |
802 | } | |
803 | } | |
1a4d82fc JJ |
804 | } |
805 | ||
85aaf69f | 806 | impl fmt::Debug for Nonterminal { |
9fa01778 | 807 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
1a4d82fc JJ |
808 | match *self { |
809 | NtItem(..) => f.pad("NtItem(..)"), | |
810 | NtBlock(..) => f.pad("NtBlock(..)"), | |
811 | NtStmt(..) => f.pad("NtStmt(..)"), | |
812 | NtPat(..) => f.pad("NtPat(..)"), | |
813 | NtExpr(..) => f.pad("NtExpr(..)"), | |
814 | NtTy(..) => f.pad("NtTy(..)"), | |
815 | NtIdent(..) => f.pad("NtIdent(..)"), | |
94b46f34 | 816 | NtLiteral(..) => f.pad("NtLiteral(..)"), |
1a4d82fc JJ |
817 | NtMeta(..) => f.pad("NtMeta(..)"), |
818 | NtPath(..) => f.pad("NtPath(..)"), | |
819 | NtTT(..) => f.pad("NtTT(..)"), | |
cc61c64b | 820 | NtVis(..) => f.pad("NtVis(..)"), |
ff7c6d11 | 821 | NtLifetime(..) => f.pad("NtLifetime(..)"), |
1a4d82fc | 822 | } |
970d7e83 LB |
823 | } |
824 | } | |
60c5eb7d XL |
825 | |
826 | impl<CTX> HashStable<CTX> for Nonterminal | |
dfeec247 XL |
827 | where |
828 | CTX: crate::HashStableContext, | |
60c5eb7d XL |
829 | { |
830 | fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { | |
831 | panic!("interpolated tokens should not be present in the HIR") | |
832 | } | |
833 | } |