]>
Commit | Line | Data |
---|---|---|
9fa01778 | 1 | pub use BinOpToken::*; |
dc9dc135 | 2 | pub use LitKind::*; |
dfeec247 | 3 | pub use Nonterminal::*; |
dc9dc135 | 4 | pub use TokenKind::*; |
9fa01778 | 5 | |
e74abb32 | 6 | use crate::ast; |
9fa01778 | 7 | use crate::ptr::P; |
487cf647 | 8 | use crate::util::case::Case; |
9fa01778 | 9 | |
60c5eb7d | 10 | use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; |
9fa01778 | 11 | use rustc_data_structures::sync::Lrc; |
60c5eb7d | 12 | use rustc_macros::HashStable_Generic; |
f035d41b | 13 | use rustc_span::symbol::{kw, sym}; |
f9f354fc | 14 | use rustc_span::symbol::{Ident, Symbol}; |
6a06907d | 15 | use rustc_span::{self, edition::Edition, Span, DUMMY_SP}; |
74b04a01 | 16 | use std::borrow::Cow; |
2b03887a | 17 | use std::fmt; |
1a4d82fc | 18 | |
3dfed10e XL |
19 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
20 | pub enum CommentKind { | |
21 | Line, | |
22 | Block, | |
23 | } | |
24 | ||
25 | #[derive(Clone, PartialEq, Encodable, Decodable, Hash, Debug, Copy)] | |
60c5eb7d | 26 | #[derive(HashStable_Generic)] |
1a4d82fc JJ |
27 | pub enum BinOpToken { |
28 | Plus, | |
29 | Minus, | |
30 | Star, | |
31 | Slash, | |
32 | Percent, | |
33 | Caret, | |
34 | And, | |
35 | Or, | |
36 | Shl, | |
37 | Shr, | |
38 | } | |
39 | ||
04454e1e FG |
40 | /// Describes how a sequence of token trees is delimited. |
41 | /// Cannot use `proc_macro::Delimiter` directly because this | |
42 | /// structure should implement some additional traits. | |
43 | /// The `None` variant is also renamed to `Invisible` to be | |
44 | /// less confusing and better convey the semantics. | |
45 | #[derive(Copy, Clone, Debug, PartialEq, Eq)] | |
46 | #[derive(Encodable, Decodable, Hash, HashStable_Generic)] | |
47 | pub enum Delimiter { | |
48 | /// `( ... )` | |
49 | Parenthesis, | |
50 | /// `{ ... }` | |
1a4d82fc | 51 | Brace, |
04454e1e FG |
52 | /// `[ ... ]` |
53 | Bracket, | |
54 | /// `Ø ... Ø` | |
55 | /// An invisible delimiter, that may, for example, appear around tokens coming from a | |
56 | /// "macro variable" `$var`. It is important to preserve operator priorities in cases like | |
57 | /// `$var * 3` where `$var` is `1 + 2`. | |
58 | /// Invisible delimiters might not survive roundtrip of a token stream through a string. | |
59 | Invisible, | |
1a4d82fc JJ |
60 | } |
61 | ||
487cf647 FG |
62 | // Note that the suffix is *not* considered when deciding the `LitKind` in this |
63 | // type. This means that float literals like `1f32` are classified by this type | |
64 | // as `Int`. Only upon conversion to `ast::LitKind` will such a literal be | |
65 | // given the `Float` kind. | |
3dfed10e | 66 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
67 | pub enum LitKind { |
68 | Bool, // AST only, must never appear in a `Token` | |
69 | Byte, | |
70 | Char, | |
487cf647 FG |
71 | Integer, // e.g. `1`, `1u8`, `1f32` |
72 | Float, // e.g. `1.`, `1.0`, `1e3f32` | |
dc9dc135 | 73 | Str, |
5e7ed085 | 74 | StrRaw(u8), // raw string delimited by `n` hash symbols |
dc9dc135 | 75 | ByteStr, |
5e7ed085 | 76 | ByteStrRaw(u8), // raw byte string delimited by `n` hash symbols |
dc9dc135 | 77 | Err, |
1a4d82fc JJ |
78 | } |
79 | ||
dc9dc135 | 80 | /// A literal token. |
3dfed10e | 81 | #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
82 | pub struct Lit { |
83 | pub kind: LitKind, | |
84 | pub symbol: Symbol, | |
85 | pub suffix: Option<Symbol>, | |
86 | } | |
48663c56 | 87 | |
487cf647 FG |
88 | impl Lit { |
89 | pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit { | |
90 | Lit { kind, symbol, suffix } | |
91 | } | |
92 | ||
93 | /// Returns `true` if this is semantically a float literal. This includes | |
94 | /// ones like `1f32` that have an `Integer` kind but a float suffix. | |
95 | pub fn is_semantic_float(&self) -> bool { | |
96 | match self.kind { | |
97 | LitKind::Float => true, | |
98 | LitKind::Integer => match self.suffix { | |
99 | Some(sym) => sym == sym::f32 || sym == sym::f64, | |
100 | None => false, | |
101 | }, | |
102 | _ => false, | |
103 | } | |
104 | } | |
105 | ||
106 | /// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation. | |
107 | pub fn from_token(token: &Token) -> Option<Lit> { | |
108 | match token.uninterpolate().kind { | |
109 | Ident(name, false) if name.is_bool_lit() => { | |
110 | Some(Lit::new(Bool, name, None)) | |
111 | } | |
112 | Literal(token_lit) => Some(token_lit), | |
113 | Interpolated(ref nt) | |
114 | if let NtExpr(expr) | NtLiteral(expr) = &**nt | |
115 | && let ast::ExprKind::Lit(token_lit) = expr.kind => | |
116 | { | |
117 | Some(token_lit.clone()) | |
118 | } | |
119 | _ => None, | |
120 | } | |
121 | } | |
122 | } | |
123 | ||
416331ca XL |
124 | impl fmt::Display for Lit { |
125 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
126 | let Lit { kind, symbol, suffix } = *self; | |
127 | match kind { | |
dfeec247 XL |
128 | Byte => write!(f, "b'{}'", symbol)?, |
129 | Char => write!(f, "'{}'", symbol)?, | |
130 | Str => write!(f, "\"{}\"", symbol)?, | |
131 | StrRaw(n) => write!( | |
132 | f, | |
133 | "r{delim}\"{string}\"{delim}", | |
134 | delim = "#".repeat(n as usize), | |
135 | string = symbol | |
136 | )?, | |
137 | ByteStr => write!(f, "b\"{}\"", symbol)?, | |
138 | ByteStrRaw(n) => write!( | |
139 | f, | |
140 | "br{delim}\"{string}\"{delim}", | |
141 | delim = "#".repeat(n as usize), | |
142 | string = symbol | |
143 | )?, | |
144 | Integer | Float | Bool | Err => write!(f, "{}", symbol)?, | |
416331ca XL |
145 | } |
146 | ||
147 | if let Some(suffix) = suffix { | |
148 | write!(f, "{}", suffix)?; | |
149 | } | |
150 | ||
151 | Ok(()) | |
152 | } | |
153 | } | |
154 | ||
dc9dc135 XL |
155 | impl LitKind { |
156 | /// An English article for the literal token kind. | |
60c5eb7d | 157 | pub fn article(self) -> &'static str { |
dc9dc135 XL |
158 | match self { |
159 | Integer | Err => "an", | |
160 | _ => "a", | |
1a4d82fc JJ |
161 | } |
162 | } | |
83c7162d | 163 | |
60c5eb7d | 164 | pub fn descr(self) -> &'static str { |
dc9dc135 XL |
165 | match self { |
166 | Bool => panic!("literal token contains `Lit::Bool`"), | |
167 | Byte => "byte", | |
168 | Char => "char", | |
169 | Integer => "integer", | |
170 | Float => "float", | |
171 | Str | StrRaw(..) => "string", | |
172 | ByteStr | ByteStrRaw(..) => "byte string", | |
173 | Err => "error", | |
174 | } | |
175 | } | |
176 | ||
923072b8 | 177 | pub(crate) fn may_have_suffix(self) -> bool { |
5869c6ff | 178 | matches!(self, Integer | Float | Err) |
48663c56 | 179 | } |
dc9dc135 | 180 | } |
48663c56 | 181 | |
f9f354fc | 182 | pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool { |
dc9dc135 | 183 | let ident_token = Token::new(Ident(name, is_raw), span); |
32a655c1 | 184 | |
dfeec247 XL |
185 | !ident_token.is_reserved_ident() |
186 | || ident_token.is_path_segment_keyword() | |
74b04a01 XL |
187 | || [ |
188 | kw::Async, | |
189 | kw::Do, | |
190 | kw::Box, | |
191 | kw::Break, | |
29967ef6 | 192 | kw::Const, |
74b04a01 XL |
193 | kw::Continue, |
194 | kw::False, | |
195 | kw::For, | |
196 | kw::If, | |
197 | kw::Let, | |
198 | kw::Loop, | |
199 | kw::Match, | |
200 | kw::Move, | |
201 | kw::Return, | |
202 | kw::True, | |
1b1a35ee | 203 | kw::Try, |
74b04a01 XL |
204 | kw::Unsafe, |
205 | kw::While, | |
206 | kw::Yield, | |
207 | kw::Static, | |
208 | ] | |
209 | .contains(&name) | |
32a655c1 SL |
210 | } |
211 | ||
f9f354fc | 212 | fn ident_can_begin_type(name: Symbol, span: Span, is_raw: bool) -> bool { |
dc9dc135 | 213 | let ident_token = Token::new(Ident(name, is_raw), span); |
cc61c64b | 214 | |
dfeec247 XL |
215 | !ident_token.is_reserved_ident() |
216 | || ident_token.is_path_segment_keyword() | |
217 | || [kw::Underscore, kw::For, kw::Impl, kw::Fn, kw::Unsafe, kw::Extern, kw::Typeof, kw::Dyn] | |
218 | .contains(&name) | |
cc61c64b XL |
219 | } |
220 | ||
3dfed10e | 221 | #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 | 222 | pub enum TokenKind { |
223e47cc | 223 | /* Expression-operator symbols. */ |
1a4d82fc JJ |
224 | Eq, |
225 | Lt, | |
226 | Le, | |
227 | EqEq, | |
228 | Ne, | |
229 | Ge, | |
230 | Gt, | |
231 | AndAnd, | |
232 | OrOr, | |
233 | Not, | |
234 | Tilde, | |
235 | BinOp(BinOpToken), | |
236 | BinOpEq(BinOpToken), | |
223e47cc LB |
237 | |
238 | /* Structural symbols */ | |
1a4d82fc JJ |
239 | At, |
240 | Dot, | |
241 | DotDot, | |
242 | DotDotDot, | |
ea8adc8c | 243 | DotDotEq, |
1a4d82fc JJ |
244 | Comma, |
245 | Semi, | |
246 | Colon, | |
247 | ModSep, | |
248 | RArrow, | |
249 | LArrow, | |
250 | FatArrow, | |
251 | Pound, | |
252 | Dollar, | |
253 | Question, | |
94b46f34 XL |
254 | /// Used by proc macros for representing lifetimes, not generated by lexer right now. |
255 | SingleQuote, | |
9fa01778 | 256 | /// An opening delimiter (e.g., `{`). |
04454e1e | 257 | OpenDelim(Delimiter), |
9fa01778 | 258 | /// A closing delimiter (e.g., `}`). |
04454e1e | 259 | CloseDelim(Delimiter), |
223e47cc LB |
260 | |
261 | /* Literals */ | |
dc9dc135 | 262 | Literal(Lit), |
223e47cc | 263 | |
74b04a01 XL |
264 | /// Identifier token. |
265 | /// Do not forget about `NtIdent` when you want to match on identifiers. | |
266 | /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to | |
267 | /// treat regular and interpolated identifiers in the same way. | |
f9f354fc | 268 | Ident(Symbol, /* is_raw */ bool), |
74b04a01 XL |
269 | /// Lifetime identifier token. |
270 | /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. | |
271 | /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to | |
272 | /// treat regular and interpolated lifetime identifiers in the same way. | |
f9f354fc | 273 | Lifetime(Symbol), |
223e47cc | 274 | |
04454e1e FG |
275 | /// An embedded AST node, as produced by a macro. This only exists for |
276 | /// historical reasons. We'd like to get rid of it, for multiple reasons. | |
277 | /// - It's conceptually very strange. Saying a token can contain an AST | |
278 | /// node is like saying, in natural language, that a word can contain a | |
279 | /// sentence. | |
280 | /// - It requires special handling in a bunch of places in the parser. | |
281 | /// - It prevents `Token` from implementing `Copy`. | |
282 | /// It adds complexity and likely slows things down. Please don't add new | |
283 | /// occurrences of this token kind! | |
9fa01778 XL |
284 | Interpolated(Lrc<Nonterminal>), |
285 | ||
3dfed10e XL |
286 | /// A doc comment token. |
287 | /// `Symbol` is the doc comment's data excluding its "quotes" (`///`, `/**`, etc) | |
288 | /// similarly to symbols in string literal tokens. | |
289 | DocComment(CommentKind, ast::AttrStyle, Symbol), | |
1a4d82fc | 290 | |
1a4d82fc JJ |
291 | Eof, |
292 | } | |
293 | ||
3dfed10e | 294 | #[derive(Clone, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)] |
dc9dc135 XL |
295 | pub struct Token { |
296 | pub kind: TokenKind, | |
297 | pub span: Span, | |
298 | } | |
299 | ||
300 | impl TokenKind { | |
301 | pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> TokenKind { | |
302 | Literal(Lit::new(kind, symbol, suffix)) | |
303 | } | |
304 | ||
487cf647 FG |
305 | /// An approximation to proc-macro-style single-character operators used by rustc parser. |
306 | /// If the operator token can be broken into two tokens, the first of which is single-character, | |
307 | /// then this function performs that operation, otherwise it returns `None`. | |
74b04a01 XL |
308 | pub fn break_two_token_op(&self) -> Option<(TokenKind, TokenKind)> { |
309 | Some(match *self { | |
310 | Le => (Lt, Eq), | |
311 | EqEq => (Eq, Eq), | |
312 | Ne => (Not, Eq), | |
313 | Ge => (Gt, Eq), | |
314 | AndAnd => (BinOp(And), BinOp(And)), | |
315 | OrOr => (BinOp(Or), BinOp(Or)), | |
316 | BinOp(Shl) => (Lt, Lt), | |
317 | BinOp(Shr) => (Gt, Gt), | |
318 | BinOpEq(Plus) => (BinOp(Plus), Eq), | |
319 | BinOpEq(Minus) => (BinOp(Minus), Eq), | |
320 | BinOpEq(Star) => (BinOp(Star), Eq), | |
321 | BinOpEq(Slash) => (BinOp(Slash), Eq), | |
322 | BinOpEq(Percent) => (BinOp(Percent), Eq), | |
323 | BinOpEq(Caret) => (BinOp(Caret), Eq), | |
324 | BinOpEq(And) => (BinOp(And), Eq), | |
325 | BinOpEq(Or) => (BinOp(Or), Eq), | |
326 | BinOpEq(Shl) => (Lt, Le), | |
327 | BinOpEq(Shr) => (Gt, Ge), | |
328 | DotDot => (Dot, Dot), | |
329 | DotDotDot => (Dot, DotDot), | |
330 | ModSep => (Colon, Colon), | |
331 | RArrow => (BinOp(Minus), Gt), | |
332 | LArrow => (Lt, BinOp(Minus)), | |
333 | FatArrow => (Eq, Gt), | |
334 | _ => return None, | |
335 | }) | |
336 | } | |
337 | ||
dc9dc135 XL |
338 | /// Returns tokens that are likely to be typed accidentally instead of the current token. |
339 | /// Enables better error recovery when the wrong token is found. | |
60c5eb7d | 340 | pub fn similar_tokens(&self) -> Option<Vec<TokenKind>> { |
dc9dc135 XL |
341 | match *self { |
342 | Comma => Some(vec![Dot, Lt, Semi]), | |
343 | Semi => Some(vec![Colon, Comma]), | |
c295e0f8 | 344 | FatArrow => Some(vec![Eq, RArrow]), |
dfeec247 | 345 | _ => None, |
dc9dc135 XL |
346 | } |
347 | } | |
29967ef6 XL |
348 | |
349 | pub fn should_end_const_arg(&self) -> bool { | |
5869c6ff | 350 | matches!(self, Gt | Ge | BinOp(Shr) | BinOpEq(Shr)) |
29967ef6 | 351 | } |
dc9dc135 | 352 | } |
0731742a | 353 | |
1a4d82fc | 354 | impl Token { |
e74abb32 | 355 | pub fn new(kind: TokenKind, span: Span) -> Self { |
dc9dc135 XL |
356 | Token { kind, span } |
357 | } | |
358 | ||
359 | /// Some token that will be thrown away later. | |
60c5eb7d | 360 | pub fn dummy() -> Self { |
1b1a35ee | 361 | Token::new(TokenKind::Question, DUMMY_SP) |
dc9dc135 XL |
362 | } |
363 | ||
f9f354fc XL |
364 | /// Recovers a `Token` from an `Ident`. This creates a raw identifier if necessary. |
365 | pub fn from_ast_ident(ident: Ident) -> Self { | |
dc9dc135 XL |
366 | Token::new(Ident(ident.name, ident.is_raw_guess()), ident.span) |
367 | } | |
368 | ||
74b04a01 XL |
369 | /// For interpolated tokens, returns a span of the fragment to which the interpolated |
370 | /// token refers. For all other tokens this is just a regular span. | |
371 | /// It is particularly important to use this for identifiers and lifetimes | |
372 | /// for which spans affect name resolution and edition checks. | |
373 | /// Note that keywords are also identifiers, so they should use this | |
374 | /// if they keep spans or perform edition checks. | |
375 | pub fn uninterpolated_span(&self) -> Span { | |
376 | match &self.kind { | |
377 | Interpolated(nt) => nt.span(), | |
378 | _ => self.span, | |
379 | } | |
380 | } | |
381 | ||
60c5eb7d | 382 | pub fn is_op(&self) -> bool { |
2b03887a FG |
383 | match self.kind { |
384 | Eq | Lt | Le | EqEq | Ne | Ge | Gt | AndAnd | OrOr | Not | Tilde | BinOp(_) | |
385 | | BinOpEq(_) | At | Dot | DotDot | DotDotDot | DotDotEq | Comma | Semi | Colon | |
386 | | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | Question | SingleQuote => true, | |
387 | ||
388 | OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) | |
389 | | Lifetime(..) | Interpolated(..) | Eof => false, | |
390 | } | |
041b39d2 XL |
391 | } |
392 | ||
60c5eb7d | 393 | pub fn is_like_plus(&self) -> bool { |
5869c6ff | 394 | matches!(self.kind, BinOp(Plus) | BinOpEq(Plus)) |
bd371182 AL |
395 | } |
396 | ||
1a4d82fc | 397 | /// Returns `true` if the token can appear at the start of an expression. |
e74abb32 | 398 | pub fn can_begin_expr(&self) -> bool { |
74b04a01 | 399 | match self.uninterpolate().kind { |
dc9dc135 XL |
400 | Ident(name, is_raw) => |
401 | ident_can_begin_expr(name, self.span, is_raw), // value name or keyword | |
ea8adc8c XL |
402 | OpenDelim(..) | // tuple, array or block |
403 | Literal(..) | // literal | |
404 | Not | // operator not | |
405 | BinOp(Minus) | // unary minus | |
406 | BinOp(Star) | // dereference | |
407 | BinOp(Or) | OrOr | // closure | |
408 | BinOp(And) | // reference | |
409 | AndAnd | // double reference | |
abe05a73 | 410 | // DotDotDot is no longer supported, but we need some way to display the error |
ea8adc8c | 411 | DotDot | DotDotDot | DotDotEq | // range notation |
ea8adc8c XL |
412 | Lt | BinOp(Shl) | // associated path |
413 | ModSep | // global path | |
83c7162d | 414 | Lifetime(..) | // labeled loop |
ea8adc8c | 415 | Pound => true, // expression attributes |
5869c6ff | 416 | Interpolated(ref nt) => matches!(**nt, NtLiteral(..) | |
94b46f34 XL |
417 | NtExpr(..) | |
418 | NtBlock(..) | | |
5869c6ff | 419 | NtPath(..)), |
c30ab7b3 | 420 | _ => false, |
1a4d82fc JJ |
421 | } |
422 | } | |
223e47cc | 423 | |
f2b60f7d FG |
424 | /// Returns `true` if the token can appear at the start of an pattern. |
425 | /// | |
426 | /// Shamelessly borrowed from `can_begin_expr`, only used for diagnostics right now. | |
427 | pub fn can_begin_pattern(&self) -> bool { | |
428 | match self.uninterpolate().kind { | |
429 | Ident(name, is_raw) => | |
430 | ident_can_begin_expr(name, self.span, is_raw), // value name or keyword | |
431 | | OpenDelim(Delimiter::Bracket | Delimiter::Parenthesis) // tuple or array | |
432 | | Literal(..) // literal | |
433 | | BinOp(Minus) // unary minus | |
434 | | BinOp(And) // reference | |
435 | | AndAnd // double reference | |
436 | // DotDotDot is no longer supported | |
437 | | DotDot | DotDotDot | DotDotEq // ranges | |
438 | | Lt | BinOp(Shl) // associated path | |
439 | | ModSep => true, // global path | |
440 | Interpolated(ref nt) => matches!(**nt, NtLiteral(..) | | |
441 | NtPat(..) | | |
442 | NtBlock(..) | | |
443 | NtPath(..)), | |
444 | _ => false, | |
445 | } | |
446 | } | |
447 | ||
32a655c1 | 448 | /// Returns `true` if the token can appear at the start of a type. |
e74abb32 | 449 | pub fn can_begin_type(&self) -> bool { |
74b04a01 | 450 | match self.uninterpolate().kind { |
dc9dc135 XL |
451 | Ident(name, is_raw) => |
452 | ident_can_begin_type(name, self.span, is_raw), // type name or keyword | |
04454e1e FG |
453 | OpenDelim(Delimiter::Parenthesis) | // tuple |
454 | OpenDelim(Delimiter::Bracket) | // array | |
7cac9316 XL |
455 | Not | // never |
456 | BinOp(Star) | // raw pointer | |
457 | BinOp(And) | // reference | |
458 | AndAnd | // double reference | |
459 | Question | // maybe bound in trait object | |
460 | Lifetime(..) | // lifetime bound in trait object | |
461 | Lt | BinOp(Shl) | // associated path | |
32a655c1 | 462 | ModSep => true, // global path |
5869c6ff | 463 | Interpolated(ref nt) => matches!(**nt, NtTy(..) | NtPath(..)), |
32a655c1 SL |
464 | _ => false, |
465 | } | |
466 | } | |
467 | ||
9fa01778 | 468 | /// Returns `true` if the token can appear at the start of a const param. |
60c5eb7d | 469 | pub fn can_begin_const_arg(&self) -> bool { |
dc9dc135 | 470 | match self.kind { |
04454e1e | 471 | OpenDelim(Delimiter::Brace) => true, |
5869c6ff | 472 | Interpolated(ref nt) => matches!(**nt, NtExpr(..) | NtBlock(..) | NtLiteral(..)), |
74b04a01 | 473 | _ => self.can_begin_literal_maybe_minus(), |
9fa01778 XL |
474 | } |
475 | } | |
476 | ||
abe05a73 | 477 | /// Returns `true` if the token can appear at the start of a generic bound. |
60c5eb7d | 478 | pub fn can_begin_bound(&self) -> bool { |
dfeec247 XL |
479 | self.is_path_start() |
480 | || self.is_lifetime() | |
481 | || self.is_keyword(kw::For) | |
482 | || self == &Question | |
04454e1e | 483 | || self == &OpenDelim(Delimiter::Parenthesis) |
abe05a73 XL |
484 | } |
485 | ||
f2b60f7d FG |
486 | /// Returns `true` if the token can appear at the start of an item. |
487 | pub fn can_begin_item(&self) -> bool { | |
488 | match self.kind { | |
489 | Ident(name, _) => [ | |
490 | kw::Fn, | |
491 | kw::Use, | |
492 | kw::Struct, | |
493 | kw::Enum, | |
494 | kw::Pub, | |
495 | kw::Trait, | |
496 | kw::Extern, | |
497 | kw::Impl, | |
498 | kw::Unsafe, | |
499 | kw::Const, | |
500 | kw::Static, | |
501 | kw::Union, | |
502 | kw::Macro, | |
503 | kw::Mod, | |
504 | kw::Type, | |
505 | ] | |
506 | .contains(&name), | |
507 | _ => false, | |
508 | } | |
509 | } | |
510 | ||
fc512014 | 511 | /// Returns `true` if the token is any literal. |
e74abb32 | 512 | pub fn is_lit(&self) -> bool { |
5869c6ff | 513 | matches!(self.kind, Literal(..)) |
1a4d82fc | 514 | } |
223e47cc | 515 | |
9fa01778 | 516 | /// Returns `true` if the token is any literal, a minus (which can prefix a literal, |
94b46f34 | 517 | /// for example a '-42', or one of the boolean idents). |
74b04a01 XL |
518 | /// |
519 | /// In other words, would this token be a valid start of `parse_literal_maybe_minus`? | |
520 | /// | |
521 | /// Keep this in sync with and `Lit::from_token`, excluding unary negation. | |
522 | pub fn can_begin_literal_maybe_minus(&self) -> bool { | |
523 | match self.uninterpolate().kind { | |
e1599b0c XL |
524 | Literal(..) | BinOp(Minus) => true, |
525 | Ident(name, false) if name.is_bool_lit() => true, | |
74b04a01 XL |
526 | Interpolated(ref nt) => match &**nt { |
527 | NtLiteral(_) => true, | |
528 | NtExpr(e) => match &e.kind { | |
529 | ast::ExprKind::Lit(_) => true, | |
530 | ast::ExprKind::Unary(ast::UnOp::Neg, e) => { | |
531 | matches!(&e.kind, ast::ExprKind::Lit(_)) | |
532 | } | |
533 | _ => false, | |
534 | }, | |
dfeec247 | 535 | _ => false, |
b7449926 | 536 | }, |
dfeec247 | 537 | _ => false, |
94b46f34 XL |
538 | } |
539 | } | |
540 | ||
487cf647 FG |
541 | /// A convenience function for matching on identifiers during parsing. |
542 | /// Turns interpolated identifier (`$i: ident`) or lifetime (`$l: lifetime`) token | |
543 | /// into the regular identifier or lifetime token it refers to, | |
544 | /// otherwise returns the original token. | |
74b04a01 XL |
545 | pub fn uninterpolate(&self) -> Cow<'_, Token> { |
546 | match &self.kind { | |
547 | Interpolated(nt) => match **nt { | |
548 | NtIdent(ident, is_raw) => { | |
549 | Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)) | |
550 | } | |
551 | NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)), | |
552 | _ => Cow::Borrowed(self), | |
553 | }, | |
554 | _ => Cow::Borrowed(self), | |
555 | } | |
556 | } | |
557 | ||
83c7162d | 558 | /// Returns an identifier if this token is an identifier. |
04454e1e | 559 | #[inline] |
f9f354fc | 560 | pub fn ident(&self) -> Option<(Ident, /* is_raw */ bool)> { |
04454e1e FG |
561 | // We avoid using `Token::uninterpolate` here because it's slow. |
562 | match &self.kind { | |
563 | &Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), | |
564 | Interpolated(nt) => match **nt { | |
565 | NtIdent(ident, is_raw) => Some((ident, is_raw)), | |
566 | _ => None, | |
567 | }, | |
83c7162d XL |
568 | _ => None, |
569 | } | |
570 | } | |
dc9dc135 | 571 | |
83c7162d | 572 | /// Returns a lifetime identifier if this token is a lifetime. |
04454e1e | 573 | #[inline] |
f9f354fc | 574 | pub fn lifetime(&self) -> Option<Ident> { |
04454e1e FG |
575 | // We avoid using `Token::uninterpolate` here because it's slow. |
576 | match &self.kind { | |
577 | &Lifetime(name) => Some(Ident::new(name, self.span)), | |
578 | Interpolated(nt) => match **nt { | |
579 | NtLifetime(ident) => Some(ident), | |
580 | _ => None, | |
581 | }, | |
cc61c64b | 582 | _ => None, |
223e47cc | 583 | } |
1a4d82fc | 584 | } |
dc9dc135 | 585 | |
cc61c64b XL |
586 | /// Returns `true` if the token is an identifier. |
587 | pub fn is_ident(&self) -> bool { | |
588 | self.ident().is_some() | |
589 | } | |
e1599b0c | 590 | |
83c7162d | 591 | /// Returns `true` if the token is a lifetime. |
60c5eb7d | 592 | pub fn is_lifetime(&self) -> bool { |
83c7162d XL |
593 | self.lifetime().is_some() |
594 | } | |
595 | ||
94222f64 | 596 | /// Returns `true` if the token is an identifier whose name is the given |
83c7162d | 597 | /// string slice. |
60c5eb7d | 598 | pub fn is_ident_named(&self, name: Symbol) -> bool { |
dc9dc135 | 599 | self.ident().map_or(false, |(ident, _)| ident.name == name) |
83c7162d | 600 | } |
cc61c64b | 601 | |
1a4d82fc | 602 | /// Returns `true` if the token is an interpolated path. |
94b46f34 | 603 | fn is_path(&self) -> bool { |
487cf647 | 604 | if let Interpolated(nt) = &self.kind && let NtPath(..) = **nt { |
5e7ed085 | 605 | return true; |
1a4d82fc | 606 | } |
487cf647 | 607 | |
c30ab7b3 | 608 | false |
1a4d82fc JJ |
609 | } |
610 | ||
416331ca XL |
611 | /// Would `maybe_whole_expr` in `parser.rs` return `Ok(..)`? |
612 | /// That is, is this a pre-parsed expression dropped into the token stream | |
613 | /// (which happens while parsing the result of macro expansion)? | |
60c5eb7d | 614 | pub fn is_whole_expr(&self) -> bool { |
487cf647 | 615 | if let Interpolated(nt) = &self.kind |
04454e1e | 616 | && let NtExpr(_) | NtLiteral(_) | NtPath(_) | NtBlock(_) = **nt |
5e7ed085 FG |
617 | { |
618 | return true; | |
416331ca XL |
619 | } |
620 | ||
621 | false | |
622 | } | |
623 | ||
487cf647 | 624 | /// Is the token an interpolated block (`$b:block`)? |
ba9703b0 | 625 | pub fn is_whole_block(&self) -> bool { |
487cf647 | 626 | if let Interpolated(nt) = &self.kind && let NtBlock(..) = **nt { |
5e7ed085 | 627 | return true; |
ba9703b0 | 628 | } |
487cf647 | 629 | |
ba9703b0 XL |
630 | false |
631 | } | |
632 | ||
1a4d82fc | 633 | /// Returns `true` if the token is either the `mut` or `const` keyword. |
60c5eb7d | 634 | pub fn is_mutability(&self) -> bool { |
dfeec247 | 635 | self.is_keyword(kw::Mut) || self.is_keyword(kw::Const) |
1a4d82fc JJ |
636 | } |
637 | ||
60c5eb7d | 638 | pub fn is_qpath_start(&self) -> bool { |
c30ab7b3 SL |
639 | self == &Lt || self == &BinOp(Shl) |
640 | } | |
641 | ||
60c5eb7d | 642 | pub fn is_path_start(&self) -> bool { |
dfeec247 XL |
643 | self == &ModSep |
644 | || self.is_qpath_start() | |
645 | || self.is_path() | |
646 | || self.is_path_segment_keyword() | |
647 | || self.is_ident() && !self.is_reserved_ident() | |
a7813a04 XL |
648 | } |
649 | ||
1a4d82fc | 650 | /// Returns `true` if the token is a given keyword, `kw`. |
dc9dc135 | 651 | pub fn is_keyword(&self, kw: Symbol) -> bool { |
e1599b0c | 652 | self.is_non_raw_ident_where(|id| id.name == kw) |
1a4d82fc JJ |
653 | } |
654 | ||
487cf647 FG |
655 | /// Returns `true` if the token is a given keyword, `kw` or if `case` is `Insensitive` and this token is an identifier equal to `kw` ignoring the case. |
656 | pub fn is_keyword_case(&self, kw: Symbol, case: Case) -> bool { | |
657 | self.is_keyword(kw) | |
658 | || (case == Case::Insensitive | |
659 | && self.is_non_raw_ident_where(|id| { | |
660 | id.name.as_str().to_lowercase() == kw.as_str().to_lowercase() | |
661 | })) | |
662 | } | |
663 | ||
60c5eb7d | 664 | pub fn is_path_segment_keyword(&self) -> bool { |
f9f354fc | 665 | self.is_non_raw_ident_where(Ident::is_path_segment_keyword) |
1a4d82fc JJ |
666 | } |
667 | ||
487cf647 FG |
668 | /// Returns true for reserved identifiers used internally for elided lifetimes, |
669 | /// unnamed method parameters, crate root module, error recovery etc. | |
60c5eb7d | 670 | pub fn is_special_ident(&self) -> bool { |
f9f354fc | 671 | self.is_non_raw_ident_where(Ident::is_special) |
1a4d82fc JJ |
672 | } |
673 | ||
041b39d2 | 674 | /// Returns `true` if the token is a keyword used in the language. |
60c5eb7d | 675 | pub fn is_used_keyword(&self) -> bool { |
f9f354fc | 676 | self.is_non_raw_ident_where(Ident::is_used_keyword) |
1a4d82fc JJ |
677 | } |
678 | ||
a7813a04 | 679 | /// Returns `true` if the token is a keyword reserved for possible future use. |
60c5eb7d | 680 | pub fn is_unused_keyword(&self) -> bool { |
f9f354fc | 681 | self.is_non_raw_ident_where(Ident::is_unused_keyword) |
1a4d82fc | 682 | } |
041b39d2 | 683 | |
83c7162d XL |
684 | /// Returns `true` if the token is either a special identifier or a keyword. |
685 | pub fn is_reserved_ident(&self) -> bool { | |
f9f354fc | 686 | self.is_non_raw_ident_where(Ident::is_reserved) |
e1599b0c XL |
687 | } |
688 | ||
689 | /// Returns `true` if the token is the identifier `true` or `false`. | |
60c5eb7d | 690 | pub fn is_bool_lit(&self) -> bool { |
e1599b0c XL |
691 | self.is_non_raw_ident_where(|id| id.name.is_bool_lit()) |
692 | } | |
693 | ||
c295e0f8 XL |
694 | pub fn is_numeric_lit(&self) -> bool { |
695 | matches!( | |
696 | self.kind, | |
697 | Literal(Lit { kind: LitKind::Integer, .. }) | Literal(Lit { kind: LitKind::Float, .. }) | |
698 | ) | |
699 | } | |
700 | ||
e1599b0c | 701 | /// Returns `true` if the token is a non-raw identifier for which `pred` holds. |
f9f354fc | 702 | pub fn is_non_raw_ident_where(&self, pred: impl FnOnce(Ident) -> bool) -> bool { |
83c7162d | 703 | match self.ident() { |
e1599b0c | 704 | Some((id, false)) => pred(id), |
83c7162d XL |
705 | _ => false, |
706 | } | |
707 | } | |
708 | ||
60c5eb7d | 709 | pub fn glue(&self, joint: &Token) -> Option<Token> { |
dc9dc135 XL |
710 | let kind = match self.kind { |
711 | Eq => match joint.kind { | |
041b39d2 XL |
712 | Eq => EqEq, |
713 | Gt => FatArrow, | |
714 | _ => return None, | |
715 | }, | |
dc9dc135 | 716 | Lt => match joint.kind { |
041b39d2 XL |
717 | Eq => Le, |
718 | Lt => BinOp(Shl), | |
719 | Le => BinOpEq(Shl), | |
720 | BinOp(Minus) => LArrow, | |
721 | _ => return None, | |
722 | }, | |
dc9dc135 | 723 | Gt => match joint.kind { |
041b39d2 XL |
724 | Eq => Ge, |
725 | Gt => BinOp(Shr), | |
726 | Ge => BinOpEq(Shr), | |
727 | _ => return None, | |
728 | }, | |
dc9dc135 | 729 | Not => match joint.kind { |
041b39d2 XL |
730 | Eq => Ne, |
731 | _ => return None, | |
732 | }, | |
dc9dc135 | 733 | BinOp(op) => match joint.kind { |
041b39d2 XL |
734 | Eq => BinOpEq(op), |
735 | BinOp(And) if op == And => AndAnd, | |
736 | BinOp(Or) if op == Or => OrOr, | |
737 | Gt if op == Minus => RArrow, | |
738 | _ => return None, | |
739 | }, | |
dc9dc135 | 740 | Dot => match joint.kind { |
041b39d2 XL |
741 | Dot => DotDot, |
742 | DotDot => DotDotDot, | |
743 | _ => return None, | |
744 | }, | |
dc9dc135 | 745 | DotDot => match joint.kind { |
041b39d2 | 746 | Dot => DotDotDot, |
ea8adc8c | 747 | Eq => DotDotEq, |
041b39d2 XL |
748 | _ => return None, |
749 | }, | |
dc9dc135 | 750 | Colon => match joint.kind { |
041b39d2 XL |
751 | Colon => ModSep, |
752 | _ => return None, | |
753 | }, | |
dc9dc135 XL |
754 | SingleQuote => match joint.kind { |
755 | Ident(name, false) => Lifetime(Symbol::intern(&format!("'{}", name))), | |
94b46f34 XL |
756 | _ => return None, |
757 | }, | |
041b39d2 | 758 | |
dfeec247 XL |
759 | Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot |
760 | | DotDotEq | Comma | Semi | ModSep | RArrow | LArrow | FatArrow | Pound | Dollar | |
761 | | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | |
1b1a35ee | 762 | | Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None, |
dc9dc135 | 763 | }; |
041b39d2 | 764 | |
dc9dc135 | 765 | Some(Token::new(kind, self.span.to(joint.span))) |
abe05a73 | 766 | } |
223e47cc LB |
767 | } |
768 | ||
dc9dc135 | 769 | impl PartialEq<TokenKind> for Token { |
2b03887a | 770 | #[inline] |
dc9dc135 XL |
771 | fn eq(&self, rhs: &TokenKind) -> bool { |
772 | self.kind == *rhs | |
773 | } | |
774 | } | |
775 | ||
3dfed10e | 776 | #[derive(Clone, Encodable, Decodable)] |
1a4d82fc JJ |
777 | /// For interpolation during macro expansion. |
778 | pub enum Nonterminal { | |
779 | NtItem(P<ast::Item>), | |
780 | NtBlock(P<ast::Block>), | |
04454e1e | 781 | NtStmt(P<ast::Stmt>), |
1a4d82fc JJ |
782 | NtPat(P<ast::Pat>), |
783 | NtExpr(P<ast::Expr>), | |
784 | NtTy(P<ast::Ty>), | |
f9f354fc XL |
785 | NtIdent(Ident, /* is_raw */ bool), |
786 | NtLifetime(Ident), | |
94b46f34 | 787 | NtLiteral(P<ast::Expr>), |
1a4d82fc | 788 | /// Stuff inside brackets for attributes |
74b04a01 | 789 | NtMeta(P<ast::AttrItem>), |
04454e1e FG |
790 | NtPath(P<ast::Path>), |
791 | NtVis(P<ast::Visibility>), | |
74b04a01 XL |
792 | } |
793 | ||
3dfed10e XL |
794 | #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)] |
795 | pub enum NonterminalKind { | |
796 | Item, | |
797 | Block, | |
798 | Stmt, | |
cdc7bbd5 XL |
799 | PatParam { |
800 | /// Keep track of whether the user used `:pat_param` or `:pat` and we inferred it from the | |
5869c6ff XL |
801 | /// edition of the span. This is used for diagnostics. |
802 | inferred: bool, | |
803 | }, | |
cdc7bbd5 | 804 | PatWithOr, |
3dfed10e XL |
805 | Expr, |
806 | Ty, | |
807 | Ident, | |
808 | Lifetime, | |
809 | Literal, | |
810 | Meta, | |
811 | Path, | |
812 | Vis, | |
813 | TT, | |
814 | } | |
815 | ||
816 | impl NonterminalKind { | |
5869c6ff XL |
817 | /// The `edition` closure is used to get the edition for the given symbol. Doing |
818 | /// `span.edition()` is expensive, so we do it lazily. | |
819 | pub fn from_symbol( | |
820 | symbol: Symbol, | |
821 | edition: impl FnOnce() -> Edition, | |
822 | ) -> Option<NonterminalKind> { | |
3dfed10e XL |
823 | Some(match symbol { |
824 | sym::item => NonterminalKind::Item, | |
825 | sym::block => NonterminalKind::Block, | |
826 | sym::stmt => NonterminalKind::Stmt, | |
5869c6ff XL |
827 | sym::pat => match edition() { |
828 | Edition::Edition2015 | Edition::Edition2018 => { | |
cdc7bbd5 | 829 | NonterminalKind::PatParam { inferred: true } |
5869c6ff | 830 | } |
04454e1e | 831 | Edition::Edition2021 | Edition::Edition2024 => NonterminalKind::PatWithOr, |
5869c6ff | 832 | }, |
cdc7bbd5 | 833 | sym::pat_param => NonterminalKind::PatParam { inferred: false }, |
3dfed10e XL |
834 | sym::expr => NonterminalKind::Expr, |
835 | sym::ty => NonterminalKind::Ty, | |
836 | sym::ident => NonterminalKind::Ident, | |
837 | sym::lifetime => NonterminalKind::Lifetime, | |
838 | sym::literal => NonterminalKind::Literal, | |
839 | sym::meta => NonterminalKind::Meta, | |
840 | sym::path => NonterminalKind::Path, | |
841 | sym::vis => NonterminalKind::Vis, | |
842 | sym::tt => NonterminalKind::TT, | |
843 | _ => return None, | |
844 | }) | |
845 | } | |
846 | fn symbol(self) -> Symbol { | |
847 | match self { | |
848 | NonterminalKind::Item => sym::item, | |
849 | NonterminalKind::Block => sym::block, | |
850 | NonterminalKind::Stmt => sym::stmt, | |
cdc7bbd5 XL |
851 | NonterminalKind::PatParam { inferred: false } => sym::pat_param, |
852 | NonterminalKind::PatParam { inferred: true } | NonterminalKind::PatWithOr => sym::pat, | |
3dfed10e XL |
853 | NonterminalKind::Expr => sym::expr, |
854 | NonterminalKind::Ty => sym::ty, | |
855 | NonterminalKind::Ident => sym::ident, | |
856 | NonterminalKind::Lifetime => sym::lifetime, | |
857 | NonterminalKind::Literal => sym::literal, | |
858 | NonterminalKind::Meta => sym::meta, | |
859 | NonterminalKind::Path => sym::path, | |
860 | NonterminalKind::Vis => sym::vis, | |
861 | NonterminalKind::TT => sym::tt, | |
862 | } | |
863 | } | |
864 | } | |
865 | ||
866 | impl fmt::Display for NonterminalKind { | |
867 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
868 | write!(f, "{}", self.symbol()) | |
869 | } | |
870 | } | |
871 | ||
74b04a01 | 872 | impl Nonterminal { |
5869c6ff | 873 | pub fn span(&self) -> Span { |
74b04a01 XL |
874 | match self { |
875 | NtItem(item) => item.span, | |
876 | NtBlock(block) => block.span, | |
877 | NtStmt(stmt) => stmt.span, | |
878 | NtPat(pat) => pat.span, | |
879 | NtExpr(expr) | NtLiteral(expr) => expr.span, | |
880 | NtTy(ty) => ty.span, | |
881 | NtIdent(ident, _) | NtLifetime(ident) => ident.span, | |
882 | NtMeta(attr_item) => attr_item.span(), | |
883 | NtPath(path) => path.span, | |
884 | NtVis(vis) => vis.span, | |
74b04a01 XL |
885 | } |
886 | } | |
83c7162d XL |
887 | } |
888 | ||
889 | impl PartialEq for Nonterminal { | |
890 | fn eq(&self, rhs: &Self) -> bool { | |
891 | match (self, rhs) { | |
dfeec247 XL |
892 | (NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { |
893 | ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs | |
894 | } | |
83c7162d | 895 | (NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, |
83c7162d XL |
896 | // FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them |
897 | // correctly based on data from AST. This will prevent them from matching each other | |
898 | // in macros. The comparison will become possible only when each nonterminal has an | |
899 | // attached token stream from which it was parsed. | |
900 | _ => false, | |
901 | } | |
902 | } | |
1a4d82fc JJ |
903 | } |
904 | ||
85aaf69f | 905 | impl fmt::Debug for Nonterminal { |
9fa01778 | 906 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
1a4d82fc JJ |
907 | match *self { |
908 | NtItem(..) => f.pad("NtItem(..)"), | |
909 | NtBlock(..) => f.pad("NtBlock(..)"), | |
910 | NtStmt(..) => f.pad("NtStmt(..)"), | |
911 | NtPat(..) => f.pad("NtPat(..)"), | |
912 | NtExpr(..) => f.pad("NtExpr(..)"), | |
913 | NtTy(..) => f.pad("NtTy(..)"), | |
914 | NtIdent(..) => f.pad("NtIdent(..)"), | |
94b46f34 | 915 | NtLiteral(..) => f.pad("NtLiteral(..)"), |
1a4d82fc JJ |
916 | NtMeta(..) => f.pad("NtMeta(..)"), |
917 | NtPath(..) => f.pad("NtPath(..)"), | |
cc61c64b | 918 | NtVis(..) => f.pad("NtVis(..)"), |
ff7c6d11 | 919 | NtLifetime(..) => f.pad("NtLifetime(..)"), |
1a4d82fc | 920 | } |
970d7e83 LB |
921 | } |
922 | } | |
60c5eb7d XL |
923 | |
924 | impl<CTX> HashStable<CTX> for Nonterminal | |
dfeec247 XL |
925 | where |
926 | CTX: crate::HashStableContext, | |
60c5eb7d XL |
927 | { |
928 | fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) { | |
929 | panic!("interpolated tokens should not be present in the HIR") | |
930 | } | |
931 | } | |
2b03887a FG |
932 | |
933 | // Some types are used a lot. Make sure they don't unintentionally get bigger. | |
934 | #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] | |
935 | mod size_asserts { | |
936 | use super::*; | |
937 | use rustc_data_structures::static_assert_size; | |
938 | // tidy-alphabetical-start | |
939 | static_assert_size!(Lit, 12); | |
940 | static_assert_size!(LitKind, 2); | |
941 | static_assert_size!(Nonterminal, 16); | |
942 | static_assert_size!(Token, 24); | |
943 | static_assert_size!(TokenKind, 16); | |
944 | // tidy-alphabetical-end | |
945 | } |