]>
Commit | Line | Data |
---|---|---|
9fa01778 | 1 | pub use BinOpToken::*; |
9fa01778 | 2 | pub use DelimToken::*; |
dc9dc135 | 3 | pub use LitKind::*; |
46de9a89 | 4 | pub use Nonterminal::*; |
dc9dc135 | 5 | pub use TokenKind::*; |
9fa01778 | 6 | |
e74abb32 | 7 | use crate::ast; |
9fa01778 | 8 | use crate::ptr::P; |
e74abb32 | 9 | use crate::tokenstream::TokenTree; |
9fa01778 | 10 | |
d9bb1a4e | 11 | use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; |
9fa01778 | 12 | use rustc_data_structures::sync::Lrc; |
d9bb1a4e | 13 | use rustc_macros::HashStable_Generic; |
46de9a89 FG |
14 | use rustc_span::symbol::kw; |
15 | use rustc_span::symbol::Symbol; | |
16 | use rustc_span::{self, Span, DUMMY_SP}; | |
17 | use std::fmt; | |
18 | use std::mem; | |
1a4d82fc | 19 | |
8faf50e0 | 20 | #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] |
d9bb1a4e | 21 | #[derive(HashStable_Generic)] |
1a4d82fc JJ |
22 | pub enum BinOpToken { |
23 | Plus, | |
24 | Minus, | |
25 | Star, | |
26 | Slash, | |
27 | Percent, | |
28 | Caret, | |
29 | And, | |
30 | Or, | |
31 | Shl, | |
32 | Shr, | |
33 | } | |
34 | ||
9fa01778 | 35 | /// A delimiter token. |
e74abb32 | 36 | #[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)] |
d9bb1a4e | 37 | #[derive(HashStable_Generic)] |
1a4d82fc | 38 | pub enum DelimToken { |
9fa01778 | 39 | /// A round parenthesis (i.e., `(` or `)`). |
1a4d82fc | 40 | Paren, |
9fa01778 | 41 | /// A square bracket (i.e., `[` or `]`). |
1a4d82fc | 42 | Bracket, |
9fa01778 | 43 | /// A curly brace (i.e., `{` or `}`). |
1a4d82fc | 44 | Brace, |
9fa01778 | 45 | /// An empty delimiter. |
5bcae85e | 46 | NoDelim, |
1a4d82fc JJ |
47 | } |
48 | ||
32a655c1 | 49 | impl DelimToken { |
8bb4bdeb XL |
50 | pub fn len(self) -> usize { |
51 | if self == NoDelim { 0 } else { 1 } | |
32a655c1 | 52 | } |
7cac9316 XL |
53 | |
54 | pub fn is_empty(self) -> bool { | |
55 | self == NoDelim | |
56 | } | |
32a655c1 SL |
57 | } |
58 | ||
d9bb1a4e | 59 | #[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
60 | pub enum LitKind { |
61 | Bool, // AST only, must never appear in a `Token` | |
62 | Byte, | |
63 | Char, | |
64 | Integer, | |
65 | Float, | |
66 | Str, | |
67 | StrRaw(u16), // raw string delimited by `n` hash symbols | |
68 | ByteStr, | |
69 | ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols | |
70 | Err, | |
1a4d82fc JJ |
71 | } |
72 | ||
dc9dc135 | 73 | /// A literal token. |
d9bb1a4e | 74 | #[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
75 | pub struct Lit { |
76 | pub kind: LitKind, | |
77 | pub symbol: Symbol, | |
78 | pub suffix: Option<Symbol>, | |
79 | } | |
48663c56 | 80 | |
416331ca XL |
81 | impl fmt::Display for Lit { |
82 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
83 | let Lit { kind, symbol, suffix } = *self; | |
84 | match kind { | |
46de9a89 FG |
85 | Byte => write!(f, "b'{}'", symbol)?, |
86 | Char => write!(f, "'{}'", symbol)?, | |
87 | Str => write!(f, "\"{}\"", symbol)?, | |
88 | StrRaw(n) => write!( | |
89 | f, | |
90 | "r{delim}\"{string}\"{delim}", | |
91 | delim = "#".repeat(n as usize), | |
92 | string = symbol | |
93 | )?, | |
94 | ByteStr => write!(f, "b\"{}\"", symbol)?, | |
95 | ByteStrRaw(n) => write!( | |
96 | f, | |
97 | "br{delim}\"{string}\"{delim}", | |
98 | delim = "#".repeat(n as usize), | |
99 | string = symbol | |
100 | )?, | |
101 | Integer | Float | Bool | Err => write!(f, "{}", symbol)?, | |
416331ca XL |
102 | } |
103 | ||
104 | if let Some(suffix) = suffix { | |
105 | write!(f, "{}", suffix)?; | |
106 | } | |
107 | ||
108 | Ok(()) | |
109 | } | |
110 | } | |
111 | ||
dc9dc135 XL |
112 | impl LitKind { |
113 | /// An English article for the literal token kind. | |
d9bb1a4e | 114 | pub fn article(self) -> &'static str { |
dc9dc135 XL |
115 | match self { |
116 | Integer | Err => "an", | |
117 | _ => "a", | |
1a4d82fc JJ |
118 | } |
119 | } | |
83c7162d | 120 | |
d9bb1a4e | 121 | pub fn descr(self) -> &'static str { |
dc9dc135 XL |
122 | match self { |
123 | Bool => panic!("literal token contains `Lit::Bool`"), | |
124 | Byte => "byte", | |
125 | Char => "char", | |
126 | Integer => "integer", | |
127 | Float => "float", | |
128 | Str | StrRaw(..) => "string", | |
129 | ByteStr | ByteStrRaw(..) => "byte string", | |
130 | Err => "error", | |
131 | } | |
132 | } | |
133 | ||
134 | crate fn may_have_suffix(self) -> bool { | |
135 | match self { | |
136 | Integer | Float | Err => true, | |
48663c56 XL |
137 | _ => false, |
138 | } | |
139 | } | |
dc9dc135 | 140 | } |
48663c56 | 141 | |
dc9dc135 XL |
142 | impl Lit { |
143 | pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit { | |
144 | Lit { kind, symbol, suffix } | |
83c7162d | 145 | } |
1a4d82fc JJ |
146 | } |
147 | ||
d9bb1a4e | 148 | pub fn ident_can_begin_expr(name: ast::Name, span: Span, is_raw: bool) -> bool { |
dc9dc135 | 149 | let ident_token = Token::new(Ident(name, is_raw), span); |
e74abb32 XL |
150 | token_can_begin_expr(&ident_token) |
151 | } | |
32a655c1 | 152 | |
d9bb1a4e | 153 | pub fn token_can_begin_expr(ident_token: &Token) -> bool { |
46de9a89 FG |
154 | !ident_token.is_reserved_ident() |
155 | || ident_token.is_path_segment_keyword() | |
156 | || match ident_token.kind { | |
157 | TokenKind::Ident(ident, _) => [ | |
158 | kw::Async, | |
159 | kw::Do, | |
160 | kw::Box, | |
161 | kw::Break, | |
162 | kw::Continue, | |
163 | kw::False, | |
164 | kw::For, | |
165 | kw::If, | |
166 | kw::Let, | |
167 | kw::Loop, | |
168 | kw::Match, | |
169 | kw::Move, | |
170 | kw::Return, | |
171 | kw::True, | |
172 | kw::Unsafe, | |
173 | kw::While, | |
174 | kw::Yield, | |
175 | kw::Static, | |
176 | ] | |
177 | .contains(&ident), | |
178 | _ => false, | |
179 | } | |
32a655c1 SL |
180 | } |
181 | ||
dc9dc135 XL |
182 | fn ident_can_begin_type(name: ast::Name, span: Span, is_raw: bool) -> bool { |
183 | let ident_token = Token::new(Ident(name, is_raw), span); | |
cc61c64b | 184 | |
46de9a89 FG |
185 | !ident_token.is_reserved_ident() |
186 | || ident_token.is_path_segment_keyword() | |
187 | || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn] | |
188 | .contains(&name) | |
cc61c64b XL |
189 | } |
190 | ||
d9bb1a4e | 191 | #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] |
dc9dc135 | 192 | pub enum TokenKind { |
223e47cc | 193 | /* Expression-operator symbols. */ |
1a4d82fc JJ |
194 | Eq, |
195 | Lt, | |
196 | Le, | |
197 | EqEq, | |
198 | Ne, | |
199 | Ge, | |
200 | Gt, | |
201 | AndAnd, | |
202 | OrOr, | |
203 | Not, | |
204 | Tilde, | |
205 | BinOp(BinOpToken), | |
206 | BinOpEq(BinOpToken), | |
223e47cc LB |
207 | |
208 | /* Structural symbols */ | |
1a4d82fc JJ |
209 | At, |
210 | Dot, | |
211 | DotDot, | |
212 | DotDotDot, | |
ea8adc8c | 213 | DotDotEq, |
1a4d82fc JJ |
214 | Comma, |
215 | Semi, | |
216 | Colon, | |
217 | ModSep, | |
218 | RArrow, | |
219 | LArrow, | |
220 | FatArrow, | |
221 | Pound, | |
222 | Dollar, | |
223 | Question, | |
94b46f34 XL |
224 | /// Used by proc macros for representing lifetimes, not generated by lexer right now. |
225 | SingleQuote, | |
9fa01778 | 226 | /// An opening delimiter (e.g., `{`). |
1a4d82fc | 227 | OpenDelim(DelimToken), |
9fa01778 | 228 | /// A closing delimiter (e.g., `}`). |
1a4d82fc | 229 | CloseDelim(DelimToken), |
223e47cc LB |
230 | |
231 | /* Literals */ | |
dc9dc135 | 232 | Literal(Lit), |
223e47cc LB |
233 | |
234 | /* Name components */ | |
dc9dc135 XL |
235 | Ident(ast::Name, /* is_raw */ bool), |
236 | Lifetime(ast::Name), | |
223e47cc | 237 | |
9fa01778 XL |
238 | Interpolated(Lrc<Nonterminal>), |
239 | ||
1a4d82fc | 240 | // Can be expanded into several tokens. |
9fa01778 | 241 | /// A doc comment. |
1a4d82fc | 242 | DocComment(ast::Name), |
1a4d82fc JJ |
243 | |
244 | // Junk. These carry no data because we don't really care about the data | |
245 | // they *would* carry, and don't really want to allocate a new ident for | |
246 | // them. Instead, users could extract that from the associated span. | |
9fa01778 | 247 | /// Whitespace. |
1a4d82fc | 248 | Whitespace, |
9fa01778 | 249 | /// A comment. |
1a4d82fc JJ |
250 | Comment, |
251 | Shebang(ast::Name), | |
416331ca XL |
252 | /// A completely invalid token which should be skipped. |
253 | Unknown(ast::Name), | |
1a4d82fc JJ |
254 | |
255 | Eof, | |
256 | } | |
257 | ||
dc9dc135 | 258 | // `TokenKind` is used a lot. Make sure it doesn't unintentionally get bigger. |
0731742a | 259 | #[cfg(target_arch = "x86_64")] |
d9bb1a4e | 260 | rustc_data_structures::static_assert_size!(TokenKind, 16); |
dc9dc135 | 261 | |
d9bb1a4e | 262 | #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
263 | pub struct Token { |
264 | pub kind: TokenKind, | |
265 | pub span: Span, | |
266 | } | |
267 | ||
268 | impl TokenKind { | |
269 | pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind { | |
270 | Literal(Lit::new(kind, symbol, suffix)) | |
271 | } | |
272 | ||
273 | /// Returns tokens that are likely to be typed accidentally instead of the current token. | |
274 | /// Enables better error recovery when the wrong token is found. | |
d9bb1a4e | 275 | pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> { |
dc9dc135 XL |
276 | match *self { |
277 | Comma => Some(vec![Dot, Lt, Semi]), | |
278 | Semi => Some(vec![Colon, Comma]), | |
46de9a89 | 279 | _ => None, |
dc9dc135 XL |
280 | } |
281 | } | |
282 | } | |
0731742a | 283 | |
1a4d82fc | 284 | impl Token { |
e74abb32 | 285 | pub fn new(kind: TokenKind, span: Span) -> Self { |
dc9dc135 XL |
286 | Token { kind, span } |
287 | } | |
288 | ||
289 | /// Some token that will be thrown away later. | |
d9bb1a4e | 290 | pub fn dummy() -> Self { |
dc9dc135 XL |
291 | Token::new(TokenKind::Whitespace, DUMMY_SP) |
292 | } | |
293 | ||
0531ce1d | 294 | /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary. |
e74abb32 | 295 | pub fn from_ast_ident(ident: ast::Ident) -> Self { |
dc9dc135 XL |
296 | Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) |
297 | } | |
298 | ||
299 | /// Return this token by value and leave a dummy token in its place. | |
e74abb32 | 300 | pub fn take(&mut self) -> Self { |
dc9dc135 XL |
301 | mem::replace(self, Token::dummy()) |
302 | } | |
303 | ||
d9bb1a4e | 304 | pub fn is_op(&self) -> bool { |
dc9dc135 | 305 | match self.kind { |
46de9a89 FG |
306 | OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) |
307 | | Lifetime(..) | Interpolated(..) | Whitespace | Comment | Shebang(..) | Eof => false, | |
dc9dc135 XL |
308 | _ => true, |
309 | } | |
041b39d2 XL |
310 | } |
311 | ||
d9bb1a4e | 312 | pub fn is_like_plus(&self) -> bool { |
dc9dc135 | 313 | match self.kind { |
94b46f34 | 314 | BinOp(Plus) | BinOpEq(Plus) => true, |
bd371182 AL |
315 | _ => false, |
316 | } | |
317 | } | |
318 | ||
1a4d82fc | 319 | /// Returns `true` if the token can appear at the start of an expression. |
e74abb32 | 320 | pub fn can_begin_expr(&self) -> bool { |
dc9dc135 XL |
321 | match self.kind { |
322 | Ident(name, is_raw) => | |
323 | ident_can_begin_expr(name, self.span, is_raw), // value name or keyword | |
ea8adc8c XL |
324 | OpenDelim(..) | // tuple, array or block |
325 | Literal(..) | // literal | |
326 | Not | // operator not | |
327 | BinOp(Minus) | // unary minus | |
328 | BinOp(Star) | // dereference | |
329 | BinOp(Or) | OrOr | // closure | |
330 | BinOp(And) | // reference | |
331 | AndAnd | // double reference | |
abe05a73 | 332 | // DotDotDot is no longer supported, but we need some way to display the error |
ea8adc8c | 333 | DotDot | DotDotDot | DotDotEq | // range notation |
ea8adc8c XL |
334 | Lt | BinOp(Shl) | // associated path |
335 | ModSep | // global path | |
83c7162d | 336 | Lifetime(..) | // labeled loop |
ea8adc8c | 337 | Pound => true, // expression attributes |
9fa01778 | 338 | Interpolated(ref nt) => match **nt { |
94b46f34 XL |
339 | NtLiteral(..) | |
340 | NtIdent(..) | | |
341 | NtExpr(..) | | |
342 | NtBlock(..) | | |
343 | NtPath(..) | | |
344 | NtLifetime(..) => true, | |
c30ab7b3 SL |
345 | _ => false, |
346 | }, | |
347 | _ => false, | |
1a4d82fc JJ |
348 | } |
349 | } | |
223e47cc | 350 | |
32a655c1 | 351 | /// Returns `true` if the token can appear at the start of a type. |
e74abb32 | 352 | pub fn can_begin_type(&self) -> bool { |
dc9dc135 XL |
353 | match self.kind { |
354 | Ident(name, is_raw) => | |
355 | ident_can_begin_type(name, self.span, is_raw), // type name or keyword | |
7cac9316 XL |
356 | OpenDelim(Paren) | // tuple |
357 | OpenDelim(Bracket) | // array | |
7cac9316 XL |
358 | Not | // never |
359 | BinOp(Star) | // raw pointer | |
360 | BinOp(And) | // reference | |
361 | AndAnd | // double reference | |
362 | Question | // maybe bound in trait object | |
363 | Lifetime(..) | // lifetime bound in trait object | |
364 | Lt | BinOp(Shl) | // associated path | |
32a655c1 | 365 | ModSep => true, // global path |
9fa01778 | 366 | Interpolated(ref nt) => match **nt { |
ff7c6d11 | 367 | NtIdent(..) | NtTy(..) | NtPath(..) | NtLifetime(..) => true, |
32a655c1 SL |
368 | _ => false, |
369 | }, | |
370 | _ => false, | |
371 | } | |
372 | } | |
373 | ||
9fa01778 | 374 | /// Returns `true` if the token can appear at the start of a const param. |
d9bb1a4e | 375 | pub fn can_begin_const_arg(&self) -> bool { |
dc9dc135 | 376 | match self.kind { |
9fa01778 XL |
377 | OpenDelim(Brace) => true, |
378 | Interpolated(ref nt) => match **nt { | |
d9bb1a4e | 379 | NtExpr(..) | NtBlock(..) | NtLiteral(..) => true, |
9fa01778 | 380 | _ => false, |
46de9a89 | 381 | }, |
9fa01778 XL |
382 | _ => self.can_begin_literal_or_bool(), |
383 | } | |
384 | } | |
385 | ||
abe05a73 | 386 | /// Returns `true` if the token can appear at the start of a generic bound. |
d9bb1a4e | 387 | pub fn can_begin_bound(&self) -> bool { |
46de9a89 FG |
388 | self.is_path_start() |
389 | || self.is_lifetime() | |
390 | || self.is_keyword(kw::For) | |
391 | || self == &Question | |
392 | || self == &OpenDelim(Paren) | |
abe05a73 XL |
393 | } |
394 | ||
1a4d82fc | 395 | /// Returns `true` if the token is any literal |
e74abb32 | 396 | pub fn is_lit(&self) -> bool { |
dc9dc135 | 397 | match self.kind { |
9e0c209e | 398 | Literal(..) => true, |
46de9a89 | 399 | _ => false, |
1a4d82fc JJ |
400 | } |
401 | } | |
223e47cc | 402 | |
9fa01778 | 403 | /// Returns `true` if the token is any literal, a minus (which can prefix a literal, |
94b46f34 | 404 | /// for example a '-42', or one of the boolean idents). |
e74abb32 | 405 | pub fn can_begin_literal_or_bool(&self) -> bool { |
dc9dc135 | 406 | match self.kind { |
e1599b0c XL |
407 | Literal(..) | BinOp(Minus) => true, |
408 | Ident(name, false) if name.is_bool_lit() => true, | |
9fa01778 | 409 | Interpolated(ref nt) => match **nt { |
b7449926 | 410 | NtLiteral(..) => true, |
46de9a89 | 411 | _ => false, |
b7449926 | 412 | }, |
46de9a89 | 413 | _ => false, |
94b46f34 XL |
414 | } |
415 | } | |
416 | ||
83c7162d XL |
417 | /// Returns an identifier if this token is an identifier. |
418 | pub fn ident(&self) -> Option<(ast::Ident, /* is_raw */ bool)> { | |
dc9dc135 XL |
419 | match self.kind { |
420 | Ident(name, is_raw) => Some((ast::Ident::new(name, self.span), is_raw)), | |
9fa01778 | 421 | Interpolated(ref nt) => match **nt { |
83c7162d XL |
422 | NtIdent(ident, is_raw) => Some((ident, is_raw)), |
423 | _ => None, | |
424 | }, | |
425 | _ => None, | |
426 | } | |
427 | } | |
dc9dc135 | 428 | |
83c7162d XL |
429 | /// Returns a lifetime identifier if this token is a lifetime. |
430 | pub fn lifetime(&self) -> Option<ast::Ident> { | |
dc9dc135 XL |
431 | match self.kind { |
432 | Lifetime(name) => Some(ast::Ident::new(name, self.span)), | |
9fa01778 | 433 | Interpolated(ref nt) => match **nt { |
83c7162d | 434 | NtLifetime(ident) => Some(ident), |
cc61c64b XL |
435 | _ => None, |
436 | }, | |
437 | _ => None, | |
223e47cc | 438 | } |
1a4d82fc | 439 | } |
dc9dc135 | 440 | |
cc61c64b XL |
441 | /// Returns `true` if the token is an identifier. |
442 | pub fn is_ident(&self) -> bool { | |
443 | self.ident().is_some() | |
444 | } | |
e1599b0c | 445 | |
83c7162d | 446 | /// Returns `true` if the token is a lifetime. |
d9bb1a4e | 447 | pub fn is_lifetime(&self) -> bool { |
83c7162d XL |
448 | self.lifetime().is_some() |
449 | } | |
450 | ||
451 | /// Returns `true` if the token is a identifier whose name is the given | |
452 | /// string slice. | |
d9bb1a4e | 453 | pub fn is_ident_named(&self, name: Symbol) -> bool { |
dc9dc135 | 454 | self.ident().map_or(false, |(ident, _)| ident.name == name) |
83c7162d | 455 | } |
cc61c64b | 456 | |
1a4d82fc | 457 | /// Returns `true` if the token is an interpolated path. |
94b46f34 | 458 | fn is_path(&self) -> bool { |
dc9dc135 | 459 | if let Interpolated(ref nt) = self.kind { |
9fa01778 | 460 | if let NtPath(..) = **nt { |
c30ab7b3 SL |
461 | return true; |
462 | } | |
1a4d82fc | 463 | } |
c30ab7b3 | 464 | false |
1a4d82fc JJ |
465 | } |
466 | ||
416331ca XL |
467 | /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? |
468 | /// That is, is this a pre-parsed expression dropped into the token stream | |
469 | /// (which happens while parsing the result of macro expansion)? | |
d9bb1a4e | 470 | pub fn is_whole_expr(&self) -> bool { |
416331ca XL |
471 | if let Interpolated(ref nt) = self.kind { |
472 | if let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtIdent(..) | NtBlock(_) = **nt { | |
473 | return true; | |
474 | } | |
475 | } | |
476 | ||
477 | false | |
478 | } | |
479 | ||
1a4d82fc | 480 | /// Returns `true` if the token is either the `mut` or `const` keyword. |
d9bb1a4e | 481 | pub fn is_mutability(&self) -> bool { |
46de9a89 | 482 | self.is_keyword(kw::Mut) || self.is_keyword(kw::Const) |
1a4d82fc JJ |
483 | } |
484 | ||
d9bb1a4e | 485 | pub fn is_qpath_start(&self) -> bool { |
c30ab7b3 SL |
486 | self == &Lt || self == &BinOp(Shl) |
487 | } | |
488 | ||
d9bb1a4e | 489 | pub fn is_path_start(&self) -> bool { |
46de9a89 FG |
490 | self == &ModSep |
491 | || self.is_qpath_start() | |
492 | || self.is_path() | |
493 | || self.is_path_segment_keyword() | |
494 | || self.is_ident() && !self.is_reserved_ident() | |
a7813a04 XL |
495 | } |
496 | ||
1a4d82fc | 497 | /// Returns `true` if the token is a given keyword, `kw`. |
dc9dc135 | 498 | pub fn is_keyword(&self, kw: Symbol) -> bool { |
e1599b0c | 499 | self.is_non_raw_ident_where(|id| id.name == kw) |
1a4d82fc JJ |
500 | } |
501 | ||
d9bb1a4e | 502 | pub fn is_path_segment_keyword(&self) -> bool { |
e1599b0c | 503 | self.is_non_raw_ident_where(ast::Ident::is_path_segment_keyword) |
1a4d82fc JJ |
504 | } |
505 | ||
041b39d2 XL |
506 | // Returns true for reserved identifiers used internally for elided lifetimes, |
507 | // unnamed method parameters, crate root module, error recovery etc. | |
d9bb1a4e | 508 | pub fn is_special_ident(&self) -> bool { |
e1599b0c | 509 | self.is_non_raw_ident_where(ast::Ident::is_special) |
1a4d82fc JJ |
510 | } |
511 | ||
041b39d2 | 512 | /// Returns `true` if the token is a keyword used in the language. |
d9bb1a4e | 513 | pub fn is_used_keyword(&self) -> bool { |
e1599b0c | 514 | self.is_non_raw_ident_where(ast::Ident::is_used_keyword) |
1a4d82fc JJ |
515 | } |
516 | ||
a7813a04 | 517 | /// Returns `true` if the token is a keyword reserved for possible future use. |
d9bb1a4e | 518 | pub fn is_unused_keyword(&self) -> bool { |
e1599b0c | 519 | self.is_non_raw_ident_where(ast::Ident::is_unused_keyword) |
1a4d82fc | 520 | } |
041b39d2 | 521 | |
83c7162d XL |
522 | /// Returns `true` if the token is either a special identifier or a keyword. |
523 | pub fn is_reserved_ident(&self) -> bool { | |
e1599b0c XL |
524 | self.is_non_raw_ident_where(ast::Ident::is_reserved) |
525 | } | |
526 | ||
527 | /// Returns `true` if the token is the identifier `true` or `false`. | |
d9bb1a4e | 528 | pub fn is_bool_lit(&self) -> bool { |
e1599b0c XL |
529 | self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) |
530 | } | |
531 | ||
532 | /// Returns `true` if the token is a non-raw identifier for which `pred` holds. | |
533 | fn is_non_raw_ident_where(&self, pred: impl FnOnce(ast::Ident) -> bool) -> bool { | |
83c7162d | 534 | match self.ident() { |
e1599b0c | 535 | Some((id, false)) => pred(id), |
83c7162d XL |
536 | _ => false, |
537 | } | |
538 | } | |
539 | ||
d9bb1a4e | 540 | pub fn glue(&self, joint: &Token) -> Option<Token> { |
dc9dc135 XL |
541 | let kind = match self.kind { |
542 | Eq => match joint.kind { | |
041b39d2 XL |
543 | Eq => EqEq, |
544 | Gt => FatArrow, | |
545 | _ => return None, | |
546 | }, | |
dc9dc135 | 547 | Lt => match joint.kind { |
041b39d2 XL |
548 | Eq => Le, |
549 | Lt => BinOp(Shl), | |
550 | Le => BinOpEq(Shl), | |
551 | BinOp(Minus) => LArrow, | |
552 | _ => return None, | |
553 | }, | |
dc9dc135 | 554 | Gt => match joint.kind { |
041b39d2 XL |
555 | Eq => Ge, |
556 | Gt => BinOp(Shr), | |
557 | Ge => BinOpEq(Shr), | |
558 | _ => return None, | |
559 | }, | |
dc9dc135 | 560 | Not => match joint.kind { |
041b39d2 XL |
561 | Eq => Ne, |
562 | _ => return None, | |
563 | }, | |
dc9dc135 | 564 | BinOp(op) => match joint.kind { |
041b39d2 XL |
565 | Eq => BinOpEq(op), |
566 | BinOp(And) if op == And => AndAnd, | |
567 | BinOp(Or) if op == Or => OrOr, | |
568 | Gt if op == Minus => RArrow, | |
569 | _ => return None, | |
570 | }, | |
dc9dc135 | 571 | Dot => match joint.kind { |
041b39d2 XL |
572 | Dot => DotDot, |
573 | DotDot => DotDotDot, | |
574 | _ => return None, | |
575 | }, | |
dc9dc135 | 576 | DotDot => match joint.kind { |
041b39d2 | 577 | Dot => DotDotDot, |
ea8adc8c | 578 | Eq => DotDotEq, |
041b39d2 XL |
579 | _ => return None, |
580 | }, | |
dc9dc135 | 581 | Colon => match joint.kind { |
041b39d2 XL |
582 | Colon => ModSep, |
583 | _ => return None, | |
584 | }, | |
dc9dc135 XL |
585 | SingleQuote => match joint.kind { |
586 | Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), | |
94b46f34 XL |
587 | _ => return None, |
588 | }, | |
041b39d2 | 589 | |
46de9a89 FG |
590 | Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot |
591 | | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | |
592 | | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | |
593 | | Lifetime(..) | Interpolated(..) | DocComment(..) | Whitespace | Comment | |
594 | | Shebang(..) | Unknown(..) | Eof => return None, | |
dc9dc135 | 595 | }; |
041b39d2 | 596 | |
dc9dc135 | 597 | Some(Token::new(kind, self.span.to(joint.span))) |
abe05a73 XL |
598 | } |
599 | ||
9fa01778 | 600 | // See comments in `Nonterminal::to_tokenstream` for why we care about |
83c7162d | 601 | // *probably* equal here rather than actual equality |
94b46f34 | 602 | crate fn probably_equal_for_proc_macro(&self, other: &Token) -> bool { |
dc9dc135 | 603 | if mem::discriminant(&self.kind) != mem::discriminant(&other.kind) { |
46de9a89 | 604 | return false; |
83c7162d | 605 | } |
dc9dc135 | 606 | match (&self.kind, &other.kind) { |
46de9a89 FG |
607 | (&Eq, &Eq) |
608 | | (&Lt, &Lt) | |
609 | | (&Le, &Le) | |
610 | | (&EqEq, &EqEq) | |
611 | | (&Ne, &Ne) | |
612 | | (&Ge, &Ge) | |
613 | | (&Gt, &Gt) | |
614 | | (&AndAnd, &AndAnd) | |
615 | | (&OrOr, &OrOr) | |
616 | | (&Not, &Not) | |
617 | | (&Tilde, &Tilde) | |
618 | | (&At, &At) | |
619 | | (&Dot, &Dot) | |
620 | | (&DotDot, &DotDot) | |
621 | | (&DotDotDot, &DotDotDot) | |
622 | | (&DotDotEq, &DotDotEq) | |
623 | | (&Comma, &Comma) | |
624 | | (&Semi, &Semi) | |
625 | | (&Colon, &Colon) | |
626 | | (&ModSep, &ModSep) | |
627 | | (&RArrow, &RArrow) | |
628 | | (&LArrow, &LArrow) | |
629 | | (&FatArrow, &FatArrow) | |
630 | | (&Pound, &Pound) | |
631 | | (&Dollar, &Dollar) | |
632 | | (&Question, &Question) | |
633 | | (&Whitespace, &Whitespace) | |
634 | | (&Comment, &Comment) | |
635 | | (&Eof, &Eof) => true, | |
636 | ||
637 | (&BinOp(a), &BinOp(b)) | (&BinOpEq(a), &BinOpEq(b)) => a == b, | |
638 | ||
639 | (&OpenDelim(a), &OpenDelim(b)) | (&CloseDelim(a), &CloseDelim(b)) => a == b, | |
640 | ||
641 | (&DocComment(a), &DocComment(b)) | (&Shebang(a), &Shebang(b)) => a == b, | |
83c7162d | 642 | |
dc9dc135 | 643 | (&Literal(a), &Literal(b)) => a == b, |
83c7162d | 644 | |
dc9dc135 | 645 | (&Lifetime(a), &Lifetime(b)) => a == b, |
46de9a89 FG |
646 | (&Ident(a, b), &Ident(c, d)) => { |
647 | b == d && (a == c || a == kw::DollarCrate || c == kw::DollarCrate) | |
648 | } | |
83c7162d XL |
649 | |
650 | (&Interpolated(_), &Interpolated(_)) => false, | |
651 | ||
652 | _ => panic!("forgot to add a token?"), | |
653 | } | |
ea8adc8c | 654 | } |
223e47cc LB |
655 | } |
656 | ||
dc9dc135 XL |
657 | impl PartialEq<TokenKind> for Token { |
658 | fn eq(&self, rhs: &TokenKind) -> bool { | |
659 | self.kind == *rhs | |
660 | } | |
661 | } | |
662 | ||
8faf50e0 | 663 | #[derive(Clone, RustcEncodable, RustcDecodable)] |
1a4d82fc JJ |
664 | /// For interpolation during macro expansion. |
665 | pub enum Nonterminal { | |
666 | NtItem(P<ast::Item>), | |
667 | NtBlock(P<ast::Block>), | |
c30ab7b3 | 668 | NtStmt(ast::Stmt), |
1a4d82fc JJ |
669 | NtPat(P<ast::Pat>), |
670 | NtExpr(P<ast::Expr>), | |
671 | NtTy(P<ast::Ty>), | |
83c7162d XL |
672 | NtIdent(ast::Ident, /* is_raw */ bool), |
673 | NtLifetime(ast::Ident), | |
94b46f34 | 674 | NtLiteral(P<ast::Expr>), |
1a4d82fc | 675 | /// Stuff inside brackets for attributes |
e74abb32 | 676 | NtMeta(ast::AttrItem), |
c30ab7b3 | 677 | NtPath(ast::Path), |
cc61c64b XL |
678 | NtVis(ast::Visibility), |
679 | NtTT(TokenTree), | |
48663c56 XL |
680 | // Used only for passing items to proc macro attributes (they are not |
681 | // strictly necessary for that, `Annotatable` can be converted into | |
682 | // tokens directly, but doing that naively regresses pretty-printing). | |
46de9a89 FG |
683 | NtTraitItem(ast::AssocItem), |
684 | NtImplItem(ast::AssocItem), | |
83c7162d | 685 | NtForeignItem(ast::ForeignItem), |
83c7162d XL |
686 | } |
687 | ||
688 | impl PartialEq for Nonterminal { | |
689 | fn eq(&self, rhs: &Self) -> bool { | |
690 | match (self, rhs) { | |
46de9a89 FG |
691 | (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { |
692 | ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs | |
693 | } | |
83c7162d XL |
694 | (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, |
695 | (NtTT(tt_lhs), NtTT(tt_rhs)) => tt_lhs == tt_rhs, | |
696 | // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them | |
697 | // correctly based on data from AST. This will prevent them from matching each other | |
698 | // in macros. The comparison will become possible only when each nonterminal has an | |
699 | // attached token stream from which it was parsed. | |
700 | _ => false, | |
701 | } | |
702 | } | |
1a4d82fc JJ |
703 | } |
704 | ||
85aaf69f | 705 | impl fmt::Debug for Nonterminal { |
9fa01778 | 706 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
1a4d82fc JJ |
707 | match *self { |
708 | NtItem(..) => f.pad("NtItem(..)"), | |
709 | NtBlock(..) => f.pad("NtBlock(..)"), | |
710 | NtStmt(..) => f.pad("NtStmt(..)"), | |
711 | NtPat(..) => f.pad("NtPat(..)"), | |
712 | NtExpr(..) => f.pad("NtExpr(..)"), | |
713 | NtTy(..) => f.pad("NtTy(..)"), | |
714 | NtIdent(..) => f.pad("NtIdent(..)"), | |
94b46f34 | 715 | NtLiteral(..) => f.pad("NtLiteral(..)"), |
1a4d82fc JJ |
716 | NtMeta(..) => f.pad("NtMeta(..)"), |
717 | NtPath(..) => f.pad("NtPath(..)"), | |
718 | NtTT(..) => f.pad("NtTT(..)"), | |
d9579d0f AL |
719 | NtImplItem(..) => f.pad("NtImplItem(..)"), |
720 | NtTraitItem(..) => f.pad("NtTraitItem(..)"), | |
83c7162d | 721 | NtForeignItem(..) => f.pad("NtForeignItem(..)"), |
cc61c64b | 722 | NtVis(..) => f.pad("NtVis(..)"), |
ff7c6d11 | 723 | NtLifetime(..) => f.pad("NtLifetime(..)"), |
1a4d82fc | 724 | } |
970d7e83 LB |
725 | } |
726 | } | |
d9bb1a4e FG |
727 | |
728 | impl<CTX> HashStable<CTX> for Nonterminal | |
46de9a89 FG |
729 | where |
730 | CTX: crate::HashStableContext, | |
d9bb1a4e FG |
731 | { |
732 | fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { | |
733 | panic!("interpolated tokens should not be present in the HIR") | |
734 | } | |
735 | } |