]>
Commit | Line | Data |
---|---|---|
9fa01778 | 1 | pub use BinOpToken::*; |
9fa01778 | 2 | pub use DelimToken::*; |
dc9dc135 | 3 | pub use LitKind::*; |
dfeec247 | 4 | pub use Nonterminal::*; |
dc9dc135 | 5 | pub use TokenKind::*; |
9fa01778 | 6 | |
e74abb32 | 7 | use crate::ast; |
9fa01778 | 8 | use crate::ptr::P; |
e74abb32 | 9 | use crate::tokenstream::TokenTree; |
9fa01778 | 10 | |
60c5eb7d | 11 | use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; |
9fa01778 | 12 | use rustc_data_structures::sync::Lrc; |
60c5eb7d | 13 | use rustc_macros::HashStable_Generic; |
dfeec247 XL |
14 | use rustc_span::symbol::kw; |
15 | use rustc_span::symbol::Symbol; | |
16 | use rustc_span::{self, Span, DUMMY_SP}; | |
74b04a01 XL |
17 | use std::borrow::Cow; |
18 | use std::{fmt, mem}; | |
1a4d82fc | 19 | |
8faf50e0 | 20 | #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] |
60c5eb7d | 21 | #[derive(HashStable_Generic)] |
1a4d82fc JJ |
22 | pub enum BinOpToken { |
23 | Plus, | |
24 | Minus, | |
25 | Star, | |
26 | Slash, | |
27 | Percent, | |
28 | Caret, | |
29 | And, | |
30 | Or, | |
31 | Shl, | |
32 | Shr, | |
33 | } | |
34 | ||
9fa01778 | 35 | /// A delimiter token. |
e74abb32 | 36 | #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] |
60c5eb7d | 37 | #[derive(HashStable_Generic)] |
1a4d82fc | 38 | pub enum DelimToken { |
9fa01778 | 39 | /// A round parenthesis (i.e., `(` or `)`). |
1a4d82fc | 40 | Paren, |
9fa01778 | 41 | /// A square bracket (i.e., `[` or `]`). |
1a4d82fc | 42 | Bracket, |
9fa01778 | 43 | /// A curly brace (i.e., `{` or `}`). |
1a4d82fc | 44 | Brace, |
9fa01778 | 45 | /// An empty delimiter. |
5bcae85e | 46 | NoDelim, |
1a4d82fc JJ |
47 | } |
48 | ||
32a655c1 | 49 | impl DelimToken { |
8bb4bdeb XL |
50 | pub fn len(self) -> usize { |
51 | if self == NoDelim { 0 } else { 1 } | |
32a655c1 | 52 | } |
7cac9316 XL |
53 | |
54 | pub fn is_empty(self) -> bool { | |
55 | self == NoDelim | |
56 | } | |
32a655c1 SL |
57 | } |
58 | ||
60c5eb7d | 59 | #[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
60 | pub enum LitKind { |
61 | Bool, // AST only, must never appear in a `Token` | |
62 | Byte, | |
63 | Char, | |
64 | Integer, | |
65 | Float, | |
66 | Str, | |
67 | StrRaw(u16), // raw string delimited by `n` hash symbols | |
68 | ByteStr, | |
69 | ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols | |
70 | Err, | |
1a4d82fc JJ |
71 | } |
72 | ||
dc9dc135 | 73 | /// A literal token. |
60c5eb7d | 74 | #[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
75 | pub struct Lit { |
76 | pub kind: LitKind, | |
77 | pub symbol: Symbol, | |
78 | pub suffix: Option<Symbol>, | |
79 | } | |
48663c56 | 80 | |
416331ca XL |
81 | impl fmt::Display for Lit { |
82 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
83 | let Lit { kind, symbol, suffix } = *self; | |
84 | match kind { | |
dfeec247 XL |
85 | Byte => write!(f, "b'{}'", symbol)?, |
86 | Char => write!(f, "'{}'", symbol)?, | |
87 | Str => write!(f, "\"{}\"", symbol)?, | |
88 | StrRaw(n) => write!( | |
89 | f, | |
90 | "r{delim}\"{string}\"{delim}", | |
91 | delim = "#".repeat(n as usize), | |
92 | string = symbol | |
93 | )?, | |
94 | ByteStr => write!(f, "b\"{}\"", symbol)?, | |
95 | ByteStrRaw(n) => write!( | |
96 | f, | |
97 | "br{delim}\"{string}\"{delim}", | |
98 | delim = "#".repeat(n as usize), | |
99 | string = symbol | |
100 | )?, | |
101 | Integer | Float | Bool | Err => write!(f, "{}", symbol)?, | |
416331ca XL |
102 | } |
103 | ||
104 | if let Some(suffix) = suffix { | |
105 | write!(f, "{}", suffix)?; | |
106 | } | |
107 | ||
108 | Ok(()) | |
109 | } | |
110 | } | |
111 | ||
dc9dc135 XL |
112 | impl LitKind { |
113 | /// An English article for the literal token kind. | |
60c5eb7d | 114 | pub fn article(self) -> &'static str { |
dc9dc135 XL |
115 | match self { |
116 | Integer | Err => "an", | |
117 | _ => "a", | |
1a4d82fc JJ |
118 | } |
119 | } | |
83c7162d | 120 | |
60c5eb7d | 121 | pub fn descr(self) -> &'static str { |
dc9dc135 XL |
122 | match self { |
123 | Bool => panic!("literal token contains `Lit::Bool`"), | |
124 | Byte => "byte", | |
125 | Char => "char", | |
126 | Integer => "integer", | |
127 | Float => "float", | |
128 | Str | StrRaw(..) => "string", | |
129 | ByteStr | ByteStrRaw(..) => "byte string", | |
130 | Err => "error", | |
131 | } | |
132 | } | |
133 | ||
134 | crate fn may_have_suffix(self) -> bool { | |
135 | match self { | |
136 | Integer | Float | Err => true, | |
48663c56 XL |
137 | _ => false, |
138 | } | |
139 | } | |
dc9dc135 | 140 | } |
48663c56 | 141 | |
dc9dc135 XL |
142 | impl Lit { |
143 | pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit { | |
144 | Lit { kind, symbol, suffix } | |
83c7162d | 145 | } |
1a4d82fc JJ |
146 | } |
147 | ||
60c5eb7d | 148 | pub fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool { |
dc9dc135 | 149 | let ident_token = Token::new(Ident(name, is_raw), span); |
32a655c1 | 150 | |
dfeec247 XL |
151 | !ident_token.is_reserved_ident() |
152 | || ident_token.is_path_segment_keyword() | |
74b04a01 XL |
153 | || [ |
154 | kw::Async, | |
155 | kw::Do, | |
156 | kw::Box, | |
157 | kw::Break, | |
158 | kw::Continue, | |
159 | kw::False, | |
160 | kw::For, | |
161 | kw::If, | |
162 | kw::Let, | |
163 | kw::Loop, | |
164 | kw::Match, | |
165 | kw::Move, | |
166 | kw::Return, | |
167 | kw::True, | |
168 | kw::Unsafe, | |
169 | kw::While, | |
170 | kw::Yield, | |
171 | kw::Static, | |
172 | ] | |
173 | .contains(&name) | |
32a655c1 SL |
174 | } |
175 | ||
dc9dc135 XL |
176 | fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool { |
177 | let ident_token = Token::new(Ident(name, is_raw), span); | |
cc61c64b | 178 | |
dfeec247 XL |
179 | !ident_token.is_reserved_ident() |
180 | || ident_token.is_path_segment_keyword() | |
181 | || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn] | |
182 | .contains(&name) | |
cc61c64b XL |
183 | } |
184 | ||
60c5eb7d | 185 | #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] |
dc9dc135 | 186 | pub enum TokenKind { |
223e47cc | 187 | /* Expression-operator symbols. */ |
1a4d82fc JJ |
188 | Eq, |
189 | Lt, | |
190 | Le, | |
191 | EqEq, | |
192 | Ne, | |
193 | Ge, | |
194 | Gt, | |
195 | AndAnd, | |
196 | OrOr, | |
197 | Not, | |
198 | Tilde, | |
199 | BinOp(BinOpToken), | |
200 | BinOpEq(BinOpToken), | |
223e47cc LB |
201 | |
202 | /* Structural symbols */ | |
1a4d82fc JJ |
203 | At, |
204 | Dot, | |
205 | DotDot, | |
206 | DotDotDot, | |
ea8adc8c | 207 | DotDotEq, |
1a4d82fc JJ |
208 | Comma, |
209 | Semi, | |
210 | Colon, | |
211 | ModSep, | |
212 | RArrow, | |
213 | LArrow, | |
214 | FatArrow, | |
215 | Pound, | |
216 | Dollar, | |
217 | Question, | |
94b46f34 XL |
218 | /// Used by proc macros for representing lifetimes, not generated by lexer right now. |
219 | SingleQuote, | |
9fa01778 | 220 | /// An opening delimiter (e.g., `{`). |
1a4d82fc | 221 | OpenDelim(DelimToken), |
9fa01778 | 222 | /// A closing delimiter (e.g., `}`). |
1a4d82fc | 223 | CloseDelim(DelimToken), |
223e47cc LB |
224 | |
225 | /* Literals */ | |
dc9dc135 | 226 | Literal(Lit), |
223e47cc | 227 | |
74b04a01 XL |
228 | /// Identifier token. |
229 | /// Do not forget about `NtIdent` when you want to match on identifiers. | |
230 | /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to | |
231 | /// treat regular and interpolated identifiers in the same way. | |
dc9dc135 | 232 | Ident(ast::Name, /* is_raw */ bool), |
74b04a01 XL |
233 | /// Lifetime identifier token. |
234 | /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. | |
235 | /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to | |
236 | /// treat regular and interpolated lifetime identifiers in the same way. | |
dc9dc135 | 237 | Lifetime(ast::Name), |
223e47cc | 238 | |
9fa01778 XL |
239 | Interpolated(Lrc<Nonterminal>), |
240 | ||
1a4d82fc | 241 | // Can be expanded into several tokens. |
9fa01778 | 242 | /// A doc comment. |
1a4d82fc | 243 | DocComment(ast::Name), |
1a4d82fc JJ |
244 | |
245 | // Junk. These carry no data because we don't really care about the data | |
246 | // they *would* carry, and don't really want to allocate a new ident for | |
247 | // them. Instead, users could extract that from the associated span. | |
9fa01778 | 248 | /// Whitespace. |
1a4d82fc | 249 | Whitespace, |
9fa01778 | 250 | /// A comment. |
1a4d82fc JJ |
251 | Comment, |
252 | Shebang(ast::Name), | |
416331ca XL |
253 | /// A completely invalid token which should be skipped. |
254 | Unknown(ast::Name), | |
1a4d82fc JJ |
255 | |
256 | Eof, | |
257 | } | |
258 | ||
dc9dc135 | 259 | // `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. |
0731742a | 260 | #[cfg(target_arch = "x86_64")] |
60c5eb7d | 261 | rustc_data_structures::static_assert_size!(TokenKind, 16); |
dc9dc135 | 262 | |
60c5eb7d | 263 | #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
264 | pub struct Token { |
265 | pub kind: TokenKind, | |
266 | pub span: Span, | |
267 | } | |
268 | ||
269 | impl TokenKind { | |
270 | pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind { | |
271 | Literal(Lit::new(kind, symbol, suffix)) | |
272 | } | |
273 | ||
74b04a01 XL |
274 | // An approximation to proc-macro-style single-character operators used by rustc parser. |
275 | // If the operator token can be broken into two tokens, the first of which is single-character, | |
276 | // then this function performs that operation, otherwise it returns `None`. | |
277 | pub fn break_two_token_op(&self) -> Option<(TokenKind, TokenKind)> { | |
278 | Some(match *self { | |
279 | Le => (Lt, Eq), | |
280 | EqEq => (Eq, Eq), | |
281 | Ne => (Not, Eq), | |
282 | Ge => (Gt, Eq), | |
283 | AndAnd => (BinOp(And), BinOp(And)), | |
284 | OrOr => (BinOp(Or), BinOp(Or)), | |
285 | BinOp(Shl) => (Lt, Lt), | |
286 | BinOp(Shr) => (Gt, Gt), | |
287 | BinOpEq(Plus) => (BinOp(Plus), Eq), | |
288 | BinOpEq(Minus) => (BinOp(Minus), Eq), | |
289 | BinOpEq(Star) => (BinOp(Star), Eq), | |
290 | BinOpEq(Slash) => (BinOp(Slash), Eq), | |
291 | BinOpEq(Percent) => (BinOp(Percent), Eq), | |
292 | BinOpEq(Caret) => (BinOp(Caret), Eq), | |
293 | BinOpEq(And) => (BinOp(And), Eq), | |
294 | BinOpEq(Or) => (BinOp(Or), Eq), | |
295 | BinOpEq(Shl) => (Lt, Le), | |
296 | BinOpEq(Shr) => (Gt, Ge), | |
297 | DotDot => (Dot, Dot), | |
298 | DotDotDot => (Dot, DotDot), | |
299 | ModSep => (Colon, Colon), | |
300 | RArrow => (BinOp(Minus), Gt), | |
301 | LArrow => (Lt, BinOp(Minus)), | |
302 | FatArrow => (Eq, Gt), | |
303 | _ => return None, | |
304 | }) | |
305 | } | |
306 | ||
dc9dc135 XL |
307 | /// Returns tokens that are likely to be typed accidentally instead of the current token. |
308 | /// Enables better error recovery when the wrong token is found. | |
60c5eb7d | 309 | pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> { |
dc9dc135 XL |
310 | match *self { |
311 | Comma => Some(vec![Dot, Lt, Semi]), | |
312 | Semi => Some(vec![Colon, Comma]), | |
dfeec247 | 313 | _ => None, |
dc9dc135 XL |
314 | } |
315 | } | |
316 | } | |
0731742a | 317 | |
1a4d82fc | 318 | impl Token { |
e74abb32 | 319 | pub fn new(kind: TokenKind, span: Span) -> Self { |
dc9dc135 XL |
320 | Token { kind, span } |
321 | } | |
322 | ||
323 | /// Some token that will be thrown away later. | |
60c5eb7d | 324 | pub fn dummy() -> Self { |
dc9dc135 XL |
325 | Token::new(TokenKind::Whitespace, DUMMY_SP) |
326 | } | |
327 | ||
0531ce1d | 328 | /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. |
e74abb32 | 329 | pub fn from_ast_ident(ident: ast::Ident) -> Self { |
dc9dc135 XL |
330 | Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) |
331 | } | |
332 | ||
333 | /// Return this token by value and leave a dummy token in its place. | |
e74abb32 | 334 | pub fn take(&mut self) -> Self { |
dc9dc135 XL |
335 | mem::replace(self, Token::dummy()) |
336 | } | |
337 | ||
74b04a01 XL |
338 | /// For interpolated tokens, returns a span of the fragment to which the interpolated |
339 | /// token refers. For all other tokens this is just a regular span. | |
340 | /// It is particularly important to use this for identifiers and lifetimes | |
341 | /// for which spans affect name resolution and edition checks. | |
342 | /// Note that keywords are also identifiers, so they should use this | |
343 | /// if they keep spans or perform edition checks. | |
344 | pub fn uninterpolated_span(&self) -> Span { | |
345 | match &self.kind { | |
346 | Interpolated(nt) => nt.span(), | |
347 | _ => self.span, | |
348 | } | |
349 | } | |
350 | ||
60c5eb7d | 351 | pub fn is_op(&self) -> bool { |
dc9dc135 | 352 | match self.kind { |
dfeec247 XL |
353 | OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) |
354 | | Lifetime(..) | Interpolated(..) | Whitespace | Comment | Shebang(..) | Eof => false, | |
dc9dc135 XL |
355 | _ => true, |
356 | } | |
041b39d2 XL |
357 | } |
358 | ||
60c5eb7d | 359 | pub fn is_like_plus(&self) -> bool { |
dc9dc135 | 360 | match self.kind { |
94b46f34 | 361 | BinOp(Plus) | BinOpEq(Plus) => true, |
bd371182 AL |
362 | _ => false, |
363 | } | |
364 | } | |
365 | ||
1a4d82fc | 366 | /// Returns `true` if the token can appear at the start of an expression. |
e74abb32 | 367 | pub fn can_begin_expr(&self) -> bool { |
74b04a01 | 368 | match self.uninterpolate().kind { |
dc9dc135 XL |
369 | Ident(name, is_raw) => |
370 | ident_can_begin_expr(name, self.span, is_raw), // value name or keyword | |
ea8adc8c XL |
371 | OpenDelim(..) | // tuple, array or block |
372 | Literal(..) | // literal | |
373 | Not | // operator not | |
374 | BinOp(Minus) | // unary minus | |
375 | BinOp(Star) | // dereference | |
376 | BinOp(Or) | OrOr | // closure | |
377 | BinOp(And) | // reference | |
378 | AndAnd | // double reference | |
abe05a73 | 379 | // DotDotDot is no longer supported, but we need some way to display the error |
ea8adc8c | 380 | DotDot | DotDotDot | DotDotEq | // range notation |
ea8adc8c XL |
381 | Lt | BinOp(Shl) | // associated path |
382 | ModSep | // global path | |
83c7162d | 383 | Lifetime(..) | // labeled loop |
ea8adc8c | 384 | Pound => true, // expression attributes |
9fa01778 | 385 | Interpolated(ref nt) => match **nt { |
94b46f34 | 386 | NtLiteral(..) | |
94b46f34 XL |
387 | NtExpr(..) | |
388 | NtBlock(..) | | |
74b04a01 | 389 | NtPath(..) => true, |
c30ab7b3 SL |
390 | _ => false, |
391 | }, | |
392 | _ => false, | |
1a4d82fc JJ |
393 | } |
394 | } | |
223e47cc | 395 | |
32a655c1 | 396 | /// Returns `true` if the token can appear at the start of a type. |
e74abb32 | 397 | pub fn can_begin_type(&self) -> bool { |
74b04a01 | 398 | match self.uninterpolate().kind { |
dc9dc135 XL |
399 | Ident(name, is_raw) => |
400 | ident_can_begin_type(name, self.span, is_raw), // type name or keyword | |
7cac9316 XL |
401 | OpenDelim(Paren) | // tuple |
402 | OpenDelim(Bracket) | // array | |
7cac9316 XL |
403 | Not | // never |
404 | BinOp(Star) | // raw pointer | |
405 | BinOp(And) | // reference | |
406 | AndAnd | // double reference | |
407 | Question | // maybe bound in trait object | |
408 | Lifetime(..) | // lifetime bound in trait object | |
409 | Lt | BinOp(Shl) | // associated path | |
32a655c1 | 410 | ModSep => true, // global path |
9fa01778 | 411 | Interpolated(ref nt) => match **nt { |
74b04a01 | 412 | NtTy(..) | NtPath(..) => true, |
32a655c1 SL |
413 | _ => false, |
414 | }, | |
415 | _ => false, | |
416 | } | |
417 | } | |
418 | ||
9fa01778 | 419 | /// Returns `true` if the token can appear at the start of a const param. |
60c5eb7d | 420 | pub fn can_begin_const_arg(&self) -> bool { |
dc9dc135 | 421 | match self.kind { |
9fa01778 XL |
422 | OpenDelim(Brace) => true, |
423 | Interpolated(ref nt) => match **nt { | |
60c5eb7d | 424 | NtExpr(..) | NtBlock(..) | NtLiteral(..) => true, |
9fa01778 | 425 | _ => false, |
dfeec247 | 426 | }, |
74b04a01 | 427 | _ => self.can_begin_literal_maybe_minus(), |
9fa01778 XL |
428 | } |
429 | } | |
430 | ||
abe05a73 | 431 | /// Returns `true` if the token can appear at the start of a generic bound. |
60c5eb7d | 432 | pub fn can_begin_bound(&self) -> bool { |
dfeec247 XL |
433 | self.is_path_start() |
434 | || self.is_lifetime() | |
435 | || self.is_keyword(kw::For) | |
436 | || self == &Question | |
437 | || self == &OpenDelim(Paren) | |
abe05a73 XL |
438 | } |
439 | ||
1a4d82fc | 440 | /// Returns `true` if the token is any literal |
e74abb32 | 441 | pub fn is_lit(&self) -> bool { |
dc9dc135 | 442 | match self.kind { |
9e0c209e | 443 | Literal(..) => true, |
dfeec247 | 444 | _ => false, |
1a4d82fc JJ |
445 | } |
446 | } | |
223e47cc | 447 | |
9fa01778 | 448 | /// Returns `true` if the token is any literal, a minus (which can prefix a literal, |
94b46f34 | 449 | /// for example a '-42', or one of the boolean idents). |
74b04a01 XL |
450 | /// |
451 | /// In other words, would this token be a valid start of `parse_literal_maybe_minus`? | |
452 | /// | |
453 | /// Keep this in sync with and `Lit::from_token`, excluding unary negation. | |
454 | pub fn can_begin_literal_maybe_minus(&self) -> bool { | |
455 | match self.uninterpolate().kind { | |
e1599b0c XL |
456 | Literal(..) | BinOp(Minus) => true, |
457 | Ident(name, false) if name.is_bool_lit() => true, | |
74b04a01 XL |
458 | Interpolated(ref nt) => match &**nt { |
459 | NtLiteral(_) => true, | |
460 | NtExpr(e) => match &e.kind { | |
461 | ast::ExprKind::Lit(_) => true, | |
462 | ast::ExprKind::Unary(ast::UnOp::Neg, e) => { | |
463 | matches!(&e.kind, ast::ExprKind::Lit(_)) | |
464 | } | |
465 | _ => false, | |
466 | }, | |
dfeec247 | 467 | _ => false, |
b7449926 | 468 | }, |
dfeec247 | 469 | _ => false, |
94b46f34 XL |
470 | } |
471 | } | |
472 | ||
74b04a01 XL |
473 | // A convenience function for matching on identifiers during parsing. |
474 | // Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token | |
475 | // into the regular identifier or lifetime token it refers to, | |
476 | // otherwise returns the original token. | |
477 | pub fn uninterpolate(&self) -> Cow<'_, Token> { | |
478 | match &self.kind { | |
479 | Interpolated(nt) => match **nt { | |
480 | NtIdent(ident, is_raw) => { | |
481 | Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)) | |
482 | } | |
483 | NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), | |
484 | _ => Cow::Borrowed(self), | |
485 | }, | |
486 | _ => Cow::Borrowed(self), | |
487 | } | |
488 | } | |
489 | ||
83c7162d XL |
490 | /// Returns an identifier if this token is an identifier. |
491 | pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { | |
74b04a01 XL |
492 | let token = self.uninterpolate(); |
493 | match token.kind { | |
494 | Ident(name, is_raw) => Some((ast::Ident::new(name, token.span), is_raw)), | |
83c7162d XL |
495 | _ => None, |
496 | } | |
497 | } | |
dc9dc135 | 498 | |
83c7162d XL |
499 | /// Returns a lifetime identifier if this token is a lifetime. |
500 | pub fn lifetime(&self) -> Option<ast::Ident> { | |
74b04a01 XL |
501 | let token = self.uninterpolate(); |
502 | match token.kind { | |
503 | Lifetime(name) => Some(ast::Ident::new(name, token.span)), | |
cc61c64b | 504 | _ => None, |
223e47cc | 505 | } |
1a4d82fc | 506 | } |
dc9dc135 | 507 | |
cc61c64b XL |
508 | /// Returns `true` if the token is an identifier. |
509 | pub fn is_ident(&self) -> bool { | |
510 | self.ident().is_some() | |
511 | } | |
e1599b0c | 512 | |
83c7162d | 513 | /// Returns `true` if the token is a lifetime. |
60c5eb7d | 514 | pub fn is_lifetime(&self) -> bool { |
83c7162d XL |
515 | self.lifetime().is_some() |
516 | } | |
517 | ||
518 | /// Returns `true` if the token is a identifier whose name is the given | |
519 | /// string slice. | |
60c5eb7d | 520 | pub fn is_ident_named(&self, name: Symbol) -> bool { |
dc9dc135 | 521 | self.ident().map_or(false, |(ident, _)| ident.name == name) |
83c7162d | 522 | } |
cc61c64b | 523 | |
1a4d82fc | 524 | /// Returns `true` if the token is an interpolated path. |
94b46f34 | 525 | fn is_path(&self) -> bool { |
dc9dc135 | 526 | if let Interpolated(ref nt) = self.kind { |
9fa01778 | 527 | if let NtPath(..) = **nt { |
c30ab7b3 SL |
528 | return true; |
529 | } | |
1a4d82fc | 530 | } |
c30ab7b3 | 531 | false |
1a4d82fc JJ |
532 | } |
533 | ||
416331ca XL |
534 | /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? |
535 | /// That is, is this a pre-parsed expression dropped into the token stream | |
536 | /// (which happens while parsing the result of macro expansion)? | |
60c5eb7d | 537 | pub fn is_whole_expr(&self) -> bool { |
416331ca XL |
538 | if let Interpolated(ref nt) = self.kind { |
539 | if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { | |
540 | return true; | |
541 | } | |
542 | } | |
543 | ||
544 | false | |
545 | } | |
546 | ||
ba9703b0 XL |
547 | // Is the token an interpolated block (`$b:block`)? |
548 | pub fn is_whole_block(&self) -> bool { | |
549 | if let Interpolated(ref nt) = self.kind { | |
550 | if let NtBlock(..) = **nt { | |
551 | return true; | |
552 | } | |
553 | } | |
554 | false | |
555 | } | |
556 | ||
1a4d82fc | 557 | /// Returns `true` if the token is either the `mut` or `const` keyword. |
60c5eb7d | 558 | pub fn is_mutability(&self) -> bool { |
dfeec247 | 559 | self.is_keyword(kw::Mut) || self.is_keyword(kw::Const) |
1a4d82fc JJ |
560 | } |
561 | ||
60c5eb7d | 562 | pub fn is_qpath_start(&self) -> bool { |
c30ab7b3 SL |
563 | self == &Lt || self == &BinOp(Shl) |
564 | } | |
565 | ||
60c5eb7d | 566 | pub fn is_path_start(&self) -> bool { |
dfeec247 XL |
567 | self == &ModSep |
568 | || self.is_qpath_start() | |
569 | || self.is_path() | |
570 | || self.is_path_segment_keyword() | |
571 | || self.is_ident() && !self.is_reserved_ident() | |
a7813a04 XL |
572 | } |
573 | ||
1a4d82fc | 574 | /// Returns `true` if the token is a given keyword, `kw`. |
dc9dc135 | 575 | pub fn is_keyword(&self, kw: Symbol) -> bool { |
e1599b0c | 576 | self.is_non_raw_ident_where(|id| id.name == kw) |
1a4d82fc JJ |
577 | } |
578 | ||
60c5eb7d | 579 | pub fn is_path_segment_keyword(&self) -> bool { |
e1599b0c | 580 | self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword) |
1a4d82fc JJ |
581 | } |
582 | ||
041b39d2 XL |
583 | // Returns true for reserved identifiers used internally for elided lifetimes, |
584 | // unnamed method parameters, crate root module, error recovery etc. | |
60c5eb7d | 585 | pub fn is_special_ident(&self) -> bool { |
e1599b0c | 586 | self.is_non_raw_ident_where(ast::Ident::is_special) |
1a4d82fc JJ |
587 | } |
588 | ||
041b39d2 | 589 | /// Returns `true` if the token is a keyword used in the language. |
60c5eb7d | 590 | pub fn is_used_keyword(&self) -> bool { |
e1599b0c | 591 | self.is_non_raw_ident_where(ast::Ident::is_used_keyword) |
1a4d82fc JJ |
592 | } |
593 | ||
a7813a04 | 594 | /// Returns `true` if the token is a keyword reserved for possible future use. |
60c5eb7d | 595 | pub fn is_unused_keyword(&self) -> bool { |
e1599b0c | 596 | self.is_non_raw_ident_where(ast::Ident::is_unused_keyword) |
1a4d82fc | 597 | } |
041b39d2 | 598 | |
83c7162d XL |
599 | /// Returns `true` if the token is either a special identifier or a keyword. |
600 | pub fn is_reserved_ident(&self) -> bool { | |
e1599b0c XL |
601 | self.is_non_raw_ident_where(ast::Ident::is_reserved) |
602 | } | |
603 | ||
604 | /// Returns `true` if the token is the identifier `true` or `false`. | |
60c5eb7d | 605 | pub fn is_bool_lit(&self) -> bool { |
e1599b0c XL |
606 | self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) |
607 | } | |
608 | ||
609 | /// Returns `true` if the token is a non-raw identifier for which `pred` holds. | |
74b04a01 | 610 | pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool { |
83c7162d | 611 | match self.ident() { |
e1599b0c | 612 | Some((id, false)) => pred(id), |
83c7162d XL |
613 | _ => false, |
614 | } | |
615 | } | |
616 | ||
60c5eb7d | 617 | pub fn glue(&self, joint: &Token) -> Option<Token> { |
dc9dc135 XL |
618 | let kind = match self.kind { |
619 | Eq => match joint.kind { | |
041b39d2 XL |
620 | Eq => EqEq, |
621 | Gt => FatArrow, | |
622 | _ => return None, | |
623 | }, | |
dc9dc135 | 624 | Lt => match joint.kind { |
041b39d2 XL |
625 | Eq => Le, |
626 | Lt => BinOp(Shl), | |
627 | Le => BinOpEq(Shl), | |
628 | BinOp(Minus) => LArrow, | |
629 | _ => return None, | |
630 | }, | |
dc9dc135 | 631 | Gt => match joint.kind { |
041b39d2 XL |
632 | Eq => Ge, |
633 | Gt => BinOp(Shr), | |
634 | Ge => BinOpEq(Shr), | |
635 | _ => return None, | |
636 | }, | |
dc9dc135 | 637 | Not => match joint.kind { |
041b39d2 XL |
638 | Eq => Ne, |
639 | _ => return None, | |
640 | }, | |
dc9dc135 | 641 | BinOp(op) => match joint.kind { |
041b39d2 XL |
642 | Eq => BinOpEq(op), |
643 | BinOp(And) if op == And => AndAnd, | |
644 | BinOp(Or) if op == Or => OrOr, | |
645 | Gt if op == Minus => RArrow, | |
646 | _ => return None, | |
647 | }, | |
dc9dc135 | 648 | Dot => match joint.kind { |
041b39d2 XL |
649 | Dot => DotDot, |
650 | DotDot => DotDotDot, | |
651 | _ => return None, | |
652 | }, | |
dc9dc135 | 653 | DotDot => match joint.kind { |
041b39d2 | 654 | Dot => DotDotDot, |
ea8adc8c | 655 | Eq => DotDotEq, |
041b39d2 XL |
656 | _ => return None, |
657 | }, | |
dc9dc135 | 658 | Colon => match joint.kind { |
041b39d2 XL |
659 | Colon => ModSep, |
660 | _ => return None, | |
661 | }, | |
dc9dc135 XL |
662 | SingleQuote => match joint.kind { |
663 | Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), | |
94b46f34 XL |
664 | _ => return None, |
665 | }, | |
041b39d2 | 666 | |
dfeec247 XL |
667 | Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot |
668 | | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | |
669 | | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | |
670 | | Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment | |
671 | | Shebang(..) | Unknown(..) | Eof => return None, | |
dc9dc135 | 672 | }; |
041b39d2 | 673 | |
dc9dc135 | 674 | Some(Token::new(kind, self.span.to(joint.span))) |
abe05a73 XL |
675 | } |
676 | ||
9fa01778 | 677 | // See comments in `Nonterminal::to_tokenstream` for why we care about |
83c7162d | 678 | // *probably* equal here rather than actual equality |
94b46f34 | 679 | crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { |
dc9dc135 | 680 | if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) { |
dfeec247 | 681 | return false; |
83c7162d | 682 | } |
dc9dc135 | 683 | match (&self.kind, &other.kind) { |
dfeec247 XL |
684 | (&Eq, &Eq) |
685 | | (&Lt, &Lt) | |
686 | | (&Le, &Le) | |
687 | | (&EqEq, &EqEq) | |
688 | | (&Ne, &Ne) | |
689 | | (&Ge, &Ge) | |
690 | | (&Gt, &Gt) | |
691 | | (&AndAnd, &AndAnd) | |
692 | | (&OrOr, &OrOr) | |
693 | | (&Not, &Not) | |
694 | | (&Tilde, &Tilde) | |
695 | | (&At, &At) | |
696 | | (&Dot, &Dot) | |
697 | | (&DotDot, &DotDot) | |
698 | | (&DotDotDot, &DotDotDot) | |
699 | | (&DotDotEq, &DotDotEq) | |
700 | | (&Comma, &Comma) | |
701 | | (&Semi, &Semi) | |
702 | | (&Colon, &Colon) | |
703 | | (&ModSep, &ModSep) | |
704 | | (&RArrow, &RArrow) | |
705 | | (&LArrow, &LArrow) | |
706 | | (&FatArrow, &FatArrow) | |
707 | | (&Pound, &Pound) | |
708 | | (&Dollar, &Dollar) | |
709 | | (&Question, &Question) | |
710 | | (&Whitespace, &Whitespace) | |
711 | | (&Comment, &Comment) | |
712 | | (&Eof, &Eof) => true, | |
713 | ||
714 | (&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b, | |
715 | ||
716 | (&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b, | |
717 | ||
718 | (&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b, | |
83c7162d | 719 | |
dc9dc135 | 720 | (&Literal(a), &Literal(b)) => a == b, |
83c7162d | 721 | |
dc9dc135 | 722 | (&Lifetime(a), &Lifetime(b)) => a == b, |
dfeec247 XL |
723 | (&Ident(a, b), &Ident(c, d)) => { |
724 | b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate) | |
725 | } | |
83c7162d XL |
726 | |
727 | (&Interpolated(_), &Interpolated(_)) => false, | |
728 | ||
729 | _ => panic!("forgot to add a token?"), | |
730 | } | |
ea8adc8c | 731 | } |
223e47cc LB |
732 | } |
733 | ||
dc9dc135 XL |
734 | impl PartialEq<TokenKind> for Token { |
735 | fn eq(&self, rhs: &TokenKind) -> bool { | |
736 | self.kind == *rhs | |
737 | } | |
738 | } | |
739 | ||
8faf50e0 | 740 | #[derive(Clone, RustcEncodable, RustcDecodable)] |
1a4d82fc JJ |
741 | /// For interpolation during macro expansion. |
742 | pub enum Nonterminal { | |
743 | NtItem(P<ast::Item>), | |
744 | NtBlock(P<ast::Block>), | |
c30ab7b3 | 745 | NtStmt(ast::Stmt), |
1a4d82fc JJ |
746 | NtPat(P<ast::Pat>), |
747 | NtExpr(P<ast::Expr>), | |
748 | NtTy(P<ast::Ty>), | |
83c7162d XL |
749 | NtIdent(ast::Ident, /* is_raw */ bool), |
750 | NtLifetime(ast::Ident), | |
94b46f34 | 751 | NtLiteral(P<ast::Expr>), |
1a4d82fc | 752 | /// Stuff inside brackets for attributes |
74b04a01 | 753 | NtMeta(P<ast::AttrItem>), |
c30ab7b3 | 754 | NtPath(ast::Path), |
cc61c64b XL |
755 | NtVis(ast::Visibility), |
756 | NtTT(TokenTree), | |
74b04a01 XL |
757 | } |
758 | ||
759 | // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger. | |
760 | #[cfg(target_arch = "x86_64")] | |
761 | rustc_data_structures::static_assert_size!(Nonterminal, 40); | |
762 | ||
763 | impl Nonterminal { | |
764 | fn span(&self) -> Span { | |
765 | match self { | |
766 | NtItem(item) => item.span, | |
767 | NtBlock(block) => block.span, | |
768 | NtStmt(stmt) => stmt.span, | |
769 | NtPat(pat) => pat.span, | |
770 | NtExpr(expr) | NtLiteral(expr) => expr.span, | |
771 | NtTy(ty) => ty.span, | |
772 | NtIdent(ident, _) | NtLifetime(ident) => ident.span, | |
773 | NtMeta(attr_item) => attr_item.span(), | |
774 | NtPath(path) => path.span, | |
775 | NtVis(vis) => vis.span, | |
776 | NtTT(tt) => tt.span(), | |
777 | } | |
778 | } | |
83c7162d XL |
779 | } |
780 | ||
781 | impl PartialEq for Nonterminal { | |
782 | fn eq(&self, rhs: &Self) -> bool { | |
783 | match (self, rhs) { | |
dfeec247 XL |
784 | (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { |
785 | ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs | |
786 | } | |
83c7162d XL |
787 | (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, |
788 | (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, | |
789 | // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them | |
790 | // correctly based on data from AST. This will prevent them from matching each other | |
791 | // in macros. The comparison will become possible only when each nonterminal has an | |
792 | // attached token stream from which it was parsed. | |
793 | _ => false, | |
794 | } | |
795 | } | |
1a4d82fc JJ |
796 | } |
797 | ||
85aaf69f | 798 | impl fmt::Debug for Nonterminal { |
9fa01778 | 799 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
1a4d82fc JJ |
800 | match *self { |
801 | NtItem(..) => f.pad("NtItem(..)"), | |
802 | NtBlock(..) => f.pad("NtBlock(..)"), | |
803 | NtStmt(..) => f.pad("NtStmt(..)"), | |
804 | NtPat(..) => f.pad("NtPat(..)"), | |
805 | NtExpr(..) => f.pad("NtExpr(..)"), | |
806 | NtTy(..) => f.pad("NtTy(..)"), | |
807 | NtIdent(..) => f.pad("NtIdent(..)"), | |
94b46f34 | 808 | NtLiteral(..) => f.pad("NtLiteral(..)"), |
1a4d82fc JJ |
809 | NtMeta(..) => f.pad("NtMeta(..)"), |
810 | NtPath(..) => f.pad("NtPath(..)"), | |
811 | NtTT(..) => f.pad("NtTT(..)"), | |
cc61c64b | 812 | NtVis(..) => f.pad("NtVis(..)"), |
ff7c6d11 | 813 | NtLifetime(..) => f.pad("NtLifetime(..)"), |
1a4d82fc | 814 | } |
970d7e83 LB |
815 | } |
816 | } | |
60c5eb7d XL |
817 | |
818 | impl<CTX> HashStable<CTX> for Nonterminal | |
dfeec247 XL |
819 | where |
820 | CTX: crate::HashStableContext, | |
60c5eb7d XL |
821 | { |
822 | fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { | |
823 | panic!("interpolated tokens should not be present in the HIR") | |
824 | } | |
825 | } |