]> git.proxmox.com Git - rustc.git/blame - compiler/rustc_parse/src/parser/mod.rs
New upstream version 1.70.0+dfsg1
[rustc.git] / compiler / rustc_parse / src / parser / mod.rs
CommitLineData
e74abb32 1pub mod attr;
6a06907d 2mod attr_wrapper;
3dfed10e 3mod diagnostics;
416331ca 4mod expr;
3dfed10e 5mod generics;
416331ca 6mod item;
3dfed10e 7mod nonterminal;
dfeec247 8mod pat;
416331ca 9mod path;
dfeec247 10mod stmt;
3dfed10e 11mod ty;
416331ca 12
9ffffee4 13use crate::lexer::UnmatchedDelim;
6a06907d 14pub use attr_wrapper::AttrWrapper;
29967ef6 15pub use diagnostics::AttemptLocalParseRecovery;
a2a8927a 16pub(crate) use item::FnParseMode;
5e7ed085 17pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
3dfed10e 18pub use path::PathStyle;
60c5eb7d 19
74b04a01 20use rustc_ast::ptr::P;
04454e1e 21use rustc_ast::token::{self, Delimiter, Nonterminal, Token, TokenKind};
9ffffee4
FG
22use rustc_ast::tokenstream::{AttributesData, DelimSpan, Spacing};
23use rustc_ast::tokenstream::{TokenStream, TokenTree, TokenTreeCursor};
487cf647 24use rustc_ast::util::case::Case;
cdc7bbd5 25use rustc_ast::AttrId;
3dfed10e 26use rustc_ast::DUMMY_NODE_ID;
9c376795 27use rustc_ast::{self as ast, AnonConst, AttrStyle, Const, DelimArgs, Extern};
487cf647 28use rustc_ast::{Async, AttrArgs, AttrArgsEq, Expr, ExprKind, MacDelimiter, Mutability, StrLit};
04454e1e 29use rustc_ast::{HasAttrs, HasTokens, Unsafe, Visibility, VisibilityKind};
74b04a01 30use rustc_ast_pretty::pprust;
cdc7bbd5 31use rustc_data_structures::fx::FxHashMap;
353b0b11 32use rustc_data_structures::sync::Ordering;
29967ef6 33use rustc_errors::PResult;
5e7ed085 34use rustc_errors::{
2b03887a 35 Applicability, DiagnosticBuilder, ErrorGuaranteed, FatalError, IntoDiagnostic, MultiSpan,
5e7ed085 36};
74b04a01 37use rustc_session::parse::ParseSess;
04454e1e 38use rustc_span::source_map::{Span, DUMMY_SP};
f9f354fc 39use rustc_span::symbol::{kw, sym, Ident, Symbol};
cdc7bbd5 40use std::ops::Range;
dfeec247 41use std::{cmp, mem, slice};
9ffffee4
FG
42use thin_vec::ThinVec;
43use tracing::debug;
60c5eb7d 44
2b03887a 45use crate::errors::{
353b0b11 46 IncorrectVisibilityRestriction, MismatchedClosingDelimiter, NonStringAbiLiteral,
2b03887a
FG
47};
48
9fa01778 49bitflags::bitflags! {
94b46f34 50 struct Restrictions: u8 {
ea8adc8c
XL
51 const STMT_EXPR = 1 << 0;
52 const NO_STRUCT_LITERAL = 1 << 1;
29967ef6 53 const CONST_EXPR = 1 << 2;
064997fb 54 const ALLOW_LET = 1 << 3;
1a4d82fc 55 }
223e47cc
LB
56}
57
8faf50e0 58#[derive(Clone, Copy, PartialEq, Debug)]
e74abb32 59enum SemiColonMode {
7453a54e
SL
60 Break,
61 Ignore,
9fa01778 62 Comma,
7453a54e
SL
63}
64
8faf50e0 65#[derive(Clone, Copy, PartialEq, Debug)]
e74abb32 66enum BlockMode {
cc61c64b
XL
67 Break,
68 Ignore,
69}
70
5869c6ff
XL
71/// Whether or not we should force collection of tokens for an AST node,
72/// regardless of whether or not it has attributes
17df50a5 73#[derive(Clone, Copy, PartialEq)]
5869c6ff
XL
74pub enum ForceCollect {
75 Yes,
76 No,
77}
78
cdc7bbd5 79#[derive(Debug, Eq, PartialEq)]
5869c6ff
XL
80pub enum TrailingToken {
81 None,
82 Semi,
2b03887a 83 Gt,
6a06907d
XL
84 /// If the trailing token is a comma, then capture it
85 /// Otherwise, ignore the trailing token
86 MaybeComma,
5869c6ff
XL
87}
88
e1599b0c 89/// Like `maybe_whole_expr`, but for things other than expressions.
416331ca 90#[macro_export]
1a4d82fc 91macro_rules! maybe_whole {
c30ab7b3 92 ($p:expr, $constructor:ident, |$x:ident| $e:expr) => {
dc9dc135 93 if let token::Interpolated(nt) = &$p.token.kind {
532ac7d7
XL
94 if let token::$constructor(x) = &**nt {
95 let $x = x.clone();
c30ab7b3
SL
96 $p.bump();
97 return Ok($e);
223e47cc 98 }
1a4d82fc 99 }
c30ab7b3 100 };
1a4d82fc 101}
223e47cc 102
532ac7d7 103/// If the next tokens are ill-formed `$ty::` recover them as `<$ty>::`.
416331ca 104#[macro_export]
532ac7d7
XL
105macro_rules! maybe_recover_from_interpolated_ty_qpath {
106 ($self: expr, $allow_qpath_recovery: expr) => {
5e7ed085 107 if $allow_qpath_recovery
487cf647 108 && $self.may_recover()
5e7ed085
FG
109 && $self.look_ahead(1, |t| t == &token::ModSep)
110 && let token::Interpolated(nt) = &$self.token.kind
111 && let token::NtTy(ty) = &**nt
112 {
532ac7d7
XL
113 let ty = ty.clone();
114 $self.bump();
74b04a01 115 return $self.maybe_recover_from_bad_qpath_stage_2($self.prev_token.span, ty);
532ac7d7 116 }
dfeec247 117 };
532ac7d7
XL
118}
119
2b03887a
FG
120#[derive(Clone, Copy)]
121pub enum Recovery {
122 Allowed,
123 Forbidden,
124}
125
041b39d2 126#[derive(Clone)]
1a4d82fc
JJ
127pub struct Parser<'a> {
128 pub sess: &'a ParseSess,
74b04a01 129 /// The current token.
dc9dc135 130 pub token: Token,
29967ef6
XL
131 /// The spacing for the current token
132 pub token_spacing: Spacing,
74b04a01
XL
133 /// The previous token.
134 pub prev_token: Token,
cdc7bbd5 135 pub capture_cfg: bool,
94b46f34 136 restrictions: Restrictions,
e74abb32 137 expected_tokens: Vec<TokenType>,
04454e1e
FG
138 // Important: This must only be advanced from `bump` to ensure that
139 // `token_cursor.num_next_calls` is updated properly.
e1599b0c 140 token_cursor: TokenCursor,
94b46f34 141 desugar_doc_comments: bool,
9fa01778
XL
142 /// This field is used to keep track of how many left angle brackets we have seen. This is
143 /// required in order to detect extra leading left angle brackets (`<` characters) and error
144 /// appropriately.
145 ///
146 /// See the comments in the `parse_path_segment` function for more details.
e74abb32
XL
147 unmatched_angle_bracket_count: u32,
148 max_angle_bracket_count: u32,
353b0b11 149
e74abb32 150 last_unexpected_token_span: Option<Span>,
3dfed10e
XL
151 /// Span pointing at the `:` for the last type ascription the parser has seen, and whether it
152 /// looked like it could have been a mistyped path or literal `Option:Some(42)`).
e74abb32 153 pub last_type_ascription: Option<(Span, bool /* likely path typo */)>,
dc9dc135 154 /// If present, this `Parser` is not parsing Rust code but rather a macro call.
e74abb32 155 subparser_name: Option<&'static str>,
cdc7bbd5 156 capture_state: CaptureState,
c295e0f8
XL
157 /// This allows us to recover when the user forget to add braces around
158 /// multiple statements in the closure body.
159 pub current_closure: Option<ClosureSpans>,
2b03887a
FG
160 /// Whether the parser is allowed to do recovery.
161 /// This is disabled when parsing macro arguments, see #103534
162 pub recovery: Recovery,
c295e0f8
XL
163}
164
2b03887a 165// This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure
04454e1e
FG
166// it doesn't unintentionally get bigger.
167#[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
353b0b11 168rustc_data_structures::static_assert_size!(Parser<'_>, 288);
04454e1e 169
5e7ed085 170/// Stores span information about a closure.
c295e0f8
XL
171#[derive(Clone)]
172pub struct ClosureSpans {
173 pub whole_closure: Span,
174 pub closing_pipe: Span,
175 pub body: Span,
cdc7bbd5
XL
176}
177
178/// Indicates a range of tokens that should be replaced by
179/// the tokens in the provided vector. This is used in two
180/// places during token collection:
181///
182/// 1. During the parsing of an AST node that may have a `#[derive]`
183/// attribute, we parse a nested AST node that has `#[cfg]` or `#[cfg_attr]`
184/// In this case, we use a `ReplaceRange` to replace the entire inner AST node
185/// with `FlatToken::AttrTarget`, allowing us to perform eager cfg-expansion
f2b60f7d 186/// on an `AttrTokenStream`.
cdc7bbd5
XL
187///
188/// 2. When we parse an inner attribute while collecting tokens. We
189/// remove inner attributes from the token stream entirely, and
190/// instead track them through the `attrs` field on the AST node.
191/// This allows us to easily manipulate them (for example, removing
192/// the first macro inner attribute to invoke a proc-macro).
193/// When create a `TokenStream`, the inner attributes get inserted
194/// into the proper place in the token stream.
195pub type ReplaceRange = (Range<u32>, Vec<(FlatToken, Spacing)>);
196
197/// Controls how we capture tokens. Capturing can be expensive,
198/// so we try to avoid performing capturing in cases where
f2b60f7d 199/// we will never need an `AttrTokenStream`.
cdc7bbd5
XL
200#[derive(Copy, Clone)]
201pub enum Capturing {
202 /// We aren't performing any capturing - this is the default mode.
203 No,
204 /// We are capturing tokens
205 Yes,
206}
207
208#[derive(Clone)]
209struct CaptureState {
210 capturing: Capturing,
211 replace_ranges: Vec<ReplaceRange>,
212 inner_attr_ranges: FxHashMap<AttrId, ReplaceRange>,
1a4d82fc
JJ
213}
214
9ffffee4
FG
215/// Iterator over a `TokenStream` that produces `Token`s. It's a bit odd that
216/// we (a) lex tokens into a nice tree structure (`TokenStream`), and then (b)
217/// use this type to emit them as a linear sequence. But a linear sequence is
218/// what the parser expects, for the most part.
041b39d2 219#[derive(Clone)]
e1599b0c 220struct TokenCursor {
9ffffee4
FG
221 // Cursor for the current (innermost) token stream. The delimiters for this
222 // token stream are found in `self.stack.last()`; when that is `None` then
223 // we are in the outermost token stream which never has delimiters.
224 tree_cursor: TokenTreeCursor,
225
226 // Token streams surrounding the current one. The delimiters for stack[n]'s
227 // tokens are in `stack[n-1]`. `stack[0]` (when present) has no delimiters
228 // because it's the outermost token stream which never has delimiters.
229 stack: Vec<(TokenTreeCursor, Delimiter, DelimSpan)>,
230
29967ef6 231 desugar_doc_comments: bool,
9ffffee4 232
04454e1e 233 // Counts the number of calls to `{,inlined_}next`.
29967ef6 234 num_next_calls: usize,
9ffffee4 235
fc512014
XL
236 // During parsing, we may sometimes need to 'unglue' a
237 // glued token into two component tokens
238 // (e.g. '>>' into '>' and '>), so that the parser
239 // can consume them one at a time. This process
240 // bypasses the normal capturing mechanism
241 // (e.g. `num_next_calls` will not be incremented),
242 // since the 'unglued' tokens due not exist in
243 // the original `TokenStream`.
244 //
245 // If we end up consuming both unglued tokens,
246 // then this is not an issue - we'll end up
247 // capturing the single 'glued' token.
248 //
249 // However, in certain circumstances, we may
250 // want to capture just the first 'unglued' token.
251 // For example, capturing the `Vec<u8>`
252 // in `Option<Vec<u8>>` requires us to unglue
cdc7bbd5 253 // the trailing `>>` token. The `break_last_token`
fc512014
XL
254 // field is used to track this token - it gets
255 // appended to the captured stream when
f2b60f7d 256 // we evaluate a `LazyAttrTokenStream`.
cdc7bbd5 257 break_last_token: bool,
8bb4bdeb
XL
258}
259
8bb4bdeb 260impl TokenCursor {
04454e1e
FG
261 fn next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
262 self.inlined_next(desugar_doc_comments)
5e7ed085
FG
263 }
264
265 /// This always-inlined version should only be used on hot code paths.
266 #[inline(always)]
04454e1e 267 fn inlined_next(&mut self, desugar_doc_comments: bool) -> (Token, Spacing) {
8bb4bdeb 268 loop {
04454e1e
FG
269 // FIXME: we currently don't return `Delimiter` open/close delims. To fix #67062 we will
270 // need to, whereupon the `delim != Delimiter::Invisible` conditions below can be
271 // removed.
9ffffee4 272 if let Some(tree) = self.tree_cursor.next_ref() {
04454e1e 273 match tree {
064997fb 274 &TokenTree::Token(ref token, spacing) => match (desugar_doc_comments, token) {
04454e1e 275 (true, &Token { kind: token::DocComment(_, attr_style, data), span }) => {
9ffffee4
FG
276 let desugared = self.desugar(attr_style, data, span);
277 self.tree_cursor.replace_prev_and_rewind(desugared);
278 // Continue to get the first token of the desugared doc comment.
279 }
280 _ => {
281 debug_assert!(!matches!(
282 token.kind,
283 token::OpenDelim(_) | token::CloseDelim(_)
284 ));
285 return (token.clone(), spacing);
04454e1e 286 }
04454e1e
FG
287 },
288 &TokenTree::Delimited(sp, delim, ref tts) => {
9ffffee4
FG
289 let trees = tts.clone().into_trees();
290 self.stack.push((mem::replace(&mut self.tree_cursor, trees), delim, sp));
04454e1e
FG
291 if delim != Delimiter::Invisible {
292 return (Token::new(token::OpenDelim(delim), sp.open), Spacing::Alone);
293 }
f2b60f7d 294 // No open delimiter to return; continue on to the next iteration.
04454e1e
FG
295 }
296 };
9ffffee4
FG
297 } else if let Some((tree_cursor, delim, span)) = self.stack.pop() {
298 // We have exhausted this token stream. Move back to its parent token stream.
299 self.tree_cursor = tree_cursor;
300 if delim != Delimiter::Invisible {
04454e1e
FG
301 return (Token::new(token::CloseDelim(delim), span.close), Spacing::Alone);
302 }
04454e1e 303 // No close delimiter to return; continue on to the next iteration.
8bb4bdeb 304 } else {
9ffffee4 305 // We have exhausted the outermost token stream.
04454e1e 306 return (Token::new(token::Eof, DUMMY_SP), Spacing::Alone);
8bb4bdeb
XL
307 }
308 }
309 }
310
9ffffee4
FG
311 // Desugar a doc comment into something like `#[doc = r"foo"]`.
312 fn desugar(&mut self, attr_style: AttrStyle, data: Symbol, span: Span) -> Vec<TokenTree> {
8bb4bdeb 313 // Searches for the occurrences of `"#*` and returns the minimum number of `#`s
2b03887a
FG
314 // required to wrap the text. E.g.
315 // - `abc d` is wrapped as `r"abc d"` (num_of_hashes = 0)
316 // - `abc "d"` is wrapped as `r#"abc "d""#` (num_of_hashes = 1)
9c376795 317 // - `abc "##d##"` is wrapped as `r###"abc ##"d"##"###` (num_of_hashes = 3)
8bb4bdeb
XL
318 let mut num_of_hashes = 0;
319 let mut count = 0;
3dfed10e 320 for ch in data.as_str().chars() {
8bb4bdeb
XL
321 count = match ch {
322 '"' => 1,
323 '#' if count > 0 => count + 1,
324 _ => 0,
325 };
326 num_of_hashes = cmp::max(num_of_hashes, count);
327 }
328
9ffffee4 329 // `/// foo` becomes `doc = r"foo"`.
04454e1e 330 let delim_span = DelimSpan::from_single(span);
0731742a
XL
331 let body = TokenTree::Delimited(
332 delim_span,
04454e1e 333 Delimiter::Bracket,
dc9dc135 334 [
064997fb
FG
335 TokenTree::token_alone(token::Ident(sym::doc, false), span),
336 TokenTree::token_alone(token::Eq, span),
337 TokenTree::token_alone(
338 TokenKind::lit(token::StrRaw(num_of_hashes), data, None),
339 span,
340 ),
0731742a 341 ]
064997fb 342 .into_iter()
74b04a01 343 .collect::<TokenStream>(),
0731742a 344 );
8bb4bdeb 345
9ffffee4
FG
346 if attr_style == AttrStyle::Inner {
347 vec![
348 TokenTree::token_alone(token::Pound, span),
349 TokenTree::token_alone(token::Not, span),
350 body,
351 ]
352 } else {
353 vec![TokenTree::token_alone(token::Pound, span), body]
354 }
8bb4bdeb
XL
355 }
356}
357
5869c6ff 358#[derive(Debug, Clone, PartialEq)]
e74abb32 359enum TokenType {
dc9dc135
XL
360 Token(TokenKind),
361 Keyword(Symbol),
1a4d82fc 362 Operator,
32a655c1
SL
363 Lifetime,
364 Ident,
365 Path,
366 Type,
9fa01778 367 Const,
1a4d82fc 368}
223e47cc 369
1a4d82fc 370impl TokenType {
e74abb32 371 fn to_string(&self) -> String {
487cf647
FG
372 match self {
373 TokenType::Token(t) => format!("`{}`", pprust::token_kind_to_string(t)),
dc9dc135 374 TokenType::Keyword(kw) => format!("`{}`", kw),
32a655c1
SL
375 TokenType::Operator => "an operator".to_string(),
376 TokenType::Lifetime => "lifetime".to_string(),
377 TokenType::Ident => "identifier".to_string(),
378 TokenType::Path => "path".to_string(),
379 TokenType::Type => "type".to_string(),
fc512014 380 TokenType::Const => "a const expression".to_string(),
1a4d82fc
JJ
381 }
382 }
223e47cc
LB
383}
384
8faf50e0 385#[derive(Copy, Clone, Debug)]
e74abb32 386enum TokenExpectType {
ea8adc8c
XL
387 Expect,
388 NoExpect,
389}
390
e74abb32
XL
391/// A sequence separator.
392struct SeqSep {
393 /// The separator token.
394 sep: Option<TokenKind>,
395 /// `true` if a trailing separator is allowed.
396 trailing_sep_allowed: bool,
397}
398
399impl SeqSep {
400 fn trailing_allowed(t: TokenKind) -> SeqSep {
dfeec247 401 SeqSep { sep: Some(t), trailing_sep_allowed: true }
e74abb32
XL
402 }
403
404 fn none() -> SeqSep {
dfeec247 405 SeqSep { sep: None, trailing_sep_allowed: false }
e74abb32
XL
406 }
407}
408
dfeec247
XL
409pub enum FollowedByType {
410 Yes,
411 No,
412}
413
2b03887a
FG
414#[derive(Clone, Copy, PartialEq, Eq)]
415pub enum TokenDescription {
416 ReservedIdentifier,
417 Keyword,
418 ReservedKeyword,
419 DocComment,
dfeec247
XL
420}
421
2b03887a
FG
422impl TokenDescription {
423 pub fn from_token(token: &Token) -> Option<Self> {
424 match token.kind {
425 _ if token.is_special_ident() => Some(TokenDescription::ReservedIdentifier),
426 _ if token.is_used_keyword() => Some(TokenDescription::Keyword),
427 _ if token.is_unused_keyword() => Some(TokenDescription::ReservedKeyword),
428 token::DocComment(..) => Some(TokenDescription::DocComment),
429 _ => None,
430 }
dfeec247
XL
431 }
432}
60c5eb7d 433
2b03887a
FG
434pub(super) fn token_descr(token: &Token) -> String {
435 let name = pprust::token_to_string(token).to_string();
436
437 let kind = TokenDescription::from_token(token).map(|kind| match kind {
438 TokenDescription::ReservedIdentifier => "reserved identifier",
439 TokenDescription::Keyword => "keyword",
440 TokenDescription::ReservedKeyword => "reserved keyword",
441 TokenDescription::DocComment => "doc comment",
442 });
443
444 if let Some(kind) = kind { format!("{} `{}`", kind, name) } else { format!("`{}`", name) }
445}
446
1a4d82fc 447impl<'a> Parser<'a> {
dc9dc135
XL
448 pub fn new(
449 sess: &'a ParseSess,
450 tokens: TokenStream,
dc9dc135
XL
451 desugar_doc_comments: bool,
452 subparser_name: Option<&'static str>,
453 ) -> Self {
c30ab7b3 454 let mut parser = Parser {
3b2f2976 455 sess,
dc9dc135 456 token: Token::dummy(),
29967ef6 457 token_spacing: Spacing::Alone,
74b04a01 458 prev_token: Token::dummy(),
cdc7bbd5 459 capture_cfg: false,
d9579d0f 460 restrictions: Restrictions::empty(),
1a4d82fc 461 expected_tokens: Vec::new(),
8bb4bdeb 462 token_cursor: TokenCursor {
9ffffee4 463 tree_cursor: tokens.into_trees(),
8bb4bdeb 464 stack: Vec::new(),
29967ef6
XL
465 num_next_calls: 0,
466 desugar_doc_comments,
cdc7bbd5 467 break_last_token: false,
8bb4bdeb 468 },
3b2f2976 469 desugar_doc_comments,
9fa01778
XL
470 unmatched_angle_bracket_count: 0,
471 max_angle_bracket_count: 0,
9fa01778 472 last_unexpected_token_span: None,
416331ca 473 last_type_ascription: None,
dc9dc135 474 subparser_name,
cdc7bbd5
XL
475 capture_state: CaptureState {
476 capturing: Capturing::No,
477 replace_ranges: Vec::new(),
478 inner_attr_ranges: Default::default(),
479 },
c295e0f8 480 current_closure: None,
2b03887a 481 recovery: Recovery::Allowed,
c30ab7b3
SL
482 };
483
74b04a01
XL
484 // Make parser point to the first token.
485 parser.bump();
7cac9316 486
c30ab7b3
SL
487 parser
488 }
489
487cf647
FG
490 pub fn recovery(mut self, recovery: Recovery) -> Self {
491 self.recovery = recovery;
2b03887a
FG
492 self
493 }
494
495 /// Whether the parser is allowed to recover from broken code.
496 ///
497 /// If this returns false, recovering broken code into valid code (especially if this recovery does lookahead)
498 /// is not allowed. All recovery done by the parser must be gated behind this check.
499 ///
500 /// Technically, this only needs to restrict eager recovery by doing lookahead at more tokens.
501 /// But making the distinction is very subtle, and simply forbidding all recovery is a lot simpler to uphold.
502 fn may_recover(&self) -> bool {
503 matches!(self.recovery, Recovery::Allowed)
504 }
505
29967ef6 506 pub fn unexpected<T>(&mut self) -> PResult<'a, T> {
9346a6ac 507 match self.expect_one_of(&[], &[]) {
9cc50fc6 508 Err(e) => Err(e),
e74abb32
XL
509 // We can get `Ok(true)` from `recover_closing_delimiter`
510 // which is called in `expected_one_of_not_found`.
511 Ok(_) => FatalError.raise(),
9346a6ac 512 }
1a4d82fc
JJ
513 }
514
9fa01778 515 /// Expects and consumes the token `t`. Signals an error if the next token is not `t`.
dc9dc135 516 pub fn expect(&mut self, t: &TokenKind) -> PResult<'a, bool /* recovered */> {
1a4d82fc
JJ
517 if self.expected_tokens.is_empty() {
518 if self.token == *t {
9cc50fc6 519 self.bump();
9fa01778 520 Ok(false)
1a4d82fc 521 } else {
dc9dc135 522 self.unexpected_try_recover(t)
1a4d82fc
JJ
523 }
524 } else {
94b46f34 525 self.expect_one_of(slice::from_ref(t), &[])
1a4d82fc
JJ
526 }
527 }
528
9c376795 529 /// Expect next token to be edible or inedible token. If edible,
1a4d82fc 530 /// then consume it; if inedible, then return without consuming
9c376795 531 /// anything. Signal a fatal error if next token is unexpected.
9fa01778
XL
532 pub fn expect_one_of(
533 &mut self,
dc9dc135
XL
534 edible: &[TokenKind],
535 inedible: &[TokenKind],
9fa01778 536 ) -> PResult<'a, bool /* recovered */> {
dc9dc135 537 if edible.contains(&self.token.kind) {
9cc50fc6 538 self.bump();
9fa01778 539 Ok(false)
dc9dc135 540 } else if inedible.contains(&self.token.kind) {
1a4d82fc 541 // leave it in the input
9fa01778 542 Ok(false)
dc9dc135 543 } else if self.last_unexpected_token_span == Some(self.token.span) {
9fa01778 544 FatalError.raise();
1a4d82fc 545 } else {
dc9dc135 546 self.expected_one_of_not_found(edible, inedible)
1a4d82fc 547 }
970d7e83
LB
548 }
549
dfeec247 550 // Public for rustfmt usage.
f9f354fc 551 pub fn parse_ident(&mut self) -> PResult<'a, Ident> {
2c00a5a8
XL
552 self.parse_ident_common(true)
553 }
554
f9f354fc 555 fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, Ident> {
353b0b11
FG
556 let (ident, is_raw) = self.ident_or_err(recover)?;
557
17df50a5 558 if !is_raw && ident.is_reserved() {
353b0b11 559 let mut err = self.expected_ident_found_err();
17df50a5
XL
560 if recover {
561 err.emit();
562 } else {
563 return Err(err);
970d7e83 564 }
970d7e83 565 }
17df50a5
XL
566 self.bump();
567 Ok(ident)
970d7e83
LB
568 }
569
353b0b11
FG
570 fn ident_or_err(&mut self, recover: bool) -> PResult<'a, (Ident, /* is_raw */ bool)> {
571 let result = self.token.ident().ok_or_else(|| self.expected_ident_found(recover));
572
573 let (ident, is_raw) = match result {
574 Ok(ident) => ident,
575 Err(err) => match err {
576 // we recovered!
577 Ok(ident) => ident,
578 Err(err) => return Err(err),
579 },
580 };
581
582 Ok((ident, is_raw))
583 }
584
9fa01778 585 /// Checks if the next token is `tok`, and returns `true` if so.
1a4d82fc 586 ///
7453a54e 587 /// This method will automatically add `tok` to `expected_tokens` if `tok` is not
1a4d82fc 588 /// encountered.
e74abb32 589 fn check(&mut self, tok: &TokenKind) -> bool {
1a4d82fc 590 let is_present = self.token == *tok;
dfeec247
XL
591 if !is_present {
592 self.expected_tokens.push(TokenType::Token(tok.clone()));
593 }
1a4d82fc 594 is_present
970d7e83
LB
595 }
596
923072b8
FG
597 fn check_noexpect(&self, tok: &TokenKind) -> bool {
598 self.token == *tok
599 }
600
601 /// Consumes a token 'tok' if it exists. Returns whether the given token was present.
602 ///
603 /// the main purpose of this function is to reduce the cluttering of the suggestions list
604 /// which using the normal eat method could introduce in some cases.
605 pub fn eat_noexpect(&mut self, tok: &TokenKind) -> bool {
606 let is_present = self.check_noexpect(tok);
607 if is_present {
608 self.bump()
609 }
610 is_present
611 }
612
9fa01778 613 /// Consumes a token 'tok' if it exists. Returns whether the given token was present.
dc9dc135 614 pub fn eat(&mut self, tok: &TokenKind) -> bool {
1a4d82fc 615 let is_present = self.check(tok);
dfeec247
XL
616 if is_present {
617 self.bump()
618 }
9cc50fc6 619 is_present
970d7e83
LB
620 }
621
e74abb32
XL
622 /// If the next token is the given keyword, returns `true` without eating it.
623 /// An expectation is also added for diagnostics purposes.
dc9dc135 624 fn check_keyword(&mut self, kw: Symbol) -> bool {
85aaf69f
SL
625 self.expected_tokens.push(TokenType::Keyword(kw));
626 self.token.is_keyword(kw)
627 }
628
487cf647
FG
629 fn check_keyword_case(&mut self, kw: Symbol, case: Case) -> bool {
630 if self.check_keyword(kw) {
631 return true;
632 }
633
634 if case == Case::Insensitive
635 && let Some((ident, /* is_raw */ false)) = self.token.ident()
636 && ident.as_str().to_lowercase() == kw.as_str().to_lowercase() {
637 true
638 } else {
639 false
640 }
641 }
642
e74abb32
XL
643 /// If the next token is the given keyword, eats it and returns `true`.
644 /// Otherwise, returns `false`. An expectation is also added for diagnostics purposes.
dfeec247
XL
645 // Public for rustfmt usage.
646 pub fn eat_keyword(&mut self, kw: Symbol) -> bool {
85aaf69f 647 if self.check_keyword(kw) {
9cc50fc6
SL
648 self.bump();
649 true
85aaf69f 650 } else {
9cc50fc6 651 false
85aaf69f
SL
652 }
653 }
654
487cf647
FG
655 /// Eats a keyword, optionally ignoring the case.
656 /// If the case differs (and is ignored) an error is issued.
657 /// This is useful for recovery.
658 fn eat_keyword_case(&mut self, kw: Symbol, case: Case) -> bool {
659 if self.eat_keyword(kw) {
660 return true;
661 }
662
663 if case == Case::Insensitive
664 && let Some((ident, /* is_raw */ false)) = self.token.ident()
665 && ident.as_str().to_lowercase() == kw.as_str().to_lowercase() {
666 self
667 .struct_span_err(ident.span, format!("keyword `{kw}` is written in a wrong case"))
668 .span_suggestion(
669 ident.span,
670 "write it in the correct case",
671 kw,
672 Applicability::MachineApplicable
673 ).emit();
674
675 self.bump();
676 return true;
677 }
678
679 false
680 }
681
dc9dc135 682 fn eat_keyword_noexpect(&mut self, kw: Symbol) -> bool {
1a4d82fc 683 if self.token.is_keyword(kw) {
9cc50fc6
SL
684 self.bump();
685 true
1a4d82fc 686 } else {
9cc50fc6 687 false
1a4d82fc 688 }
970d7e83
LB
689 }
690
9fa01778
XL
691 /// If the given word is not a keyword, signals an error.
692 /// If the next token is not the given word, signals an error.
693 /// Otherwise, eats it.
dc9dc135 694 fn expect_keyword(&mut self, kw: Symbol) -> PResult<'a, ()> {
dfeec247 695 if !self.eat_keyword(kw) { self.unexpected() } else { Ok(()) }
970d7e83
LB
696 }
697
74b04a01
XL
698 /// Is the given keyword `kw` followed by a non-reserved identifier?
699 fn is_kw_followed_by_ident(&self, kw: Symbol) -> bool {
700 self.token.is_keyword(kw) && self.look_ahead(1, |t| t.is_ident() && !t.is_reserved_ident())
701 }
702
e74abb32
XL
703 fn check_or_expected(&mut self, ok: bool, typ: TokenType) -> bool {
704 if ok {
32a655c1
SL
705 true
706 } else {
e74abb32 707 self.expected_tokens.push(typ);
32a655c1
SL
708 false
709 }
710 }
711
e74abb32
XL
712 fn check_ident(&mut self) -> bool {
713 self.check_or_expected(self.token.is_ident(), TokenType::Ident)
714 }
715
32a655c1 716 fn check_path(&mut self) -> bool {
e74abb32 717 self.check_or_expected(self.token.is_path_start(), TokenType::Path)
32a655c1
SL
718 }
719
720 fn check_type(&mut self) -> bool {
e74abb32 721 self.check_or_expected(self.token.can_begin_type(), TokenType::Type)
32a655c1
SL
722 }
723
9fa01778 724 fn check_const_arg(&mut self) -> bool {
e74abb32
XL
725 self.check_or_expected(self.token.can_begin_const_arg(), TokenType::Const)
726 }
727
9c376795
FG
728 fn check_const_closure(&self) -> bool {
729 self.is_keyword_ahead(0, &[kw::Const])
730 && self.look_ahead(1, |t| match &t.kind {
9ffffee4
FG
731 // async closures do not work with const closures, so we do not parse that here.
732 token::Ident(kw::Move | kw::Static, _) | token::OrOr | token::BinOp(token::Or) => {
733 true
734 }
9c376795
FG
735 _ => false,
736 })
737 }
738
29967ef6
XL
739 fn check_inline_const(&self, dist: usize) -> bool {
740 self.is_keyword_ahead(dist, &[kw::Const])
487cf647
FG
741 && self.look_ahead(dist + 1, |t| match &t.kind {
742 token::Interpolated(nt) => matches!(**nt, token::NtBlock(..)),
04454e1e 743 token::OpenDelim(Delimiter::Brace) => true,
29967ef6
XL
744 _ => false,
745 })
746 }
747
e74abb32
XL
748 /// Checks to see if the next token is either `+` or `+=`.
749 /// Otherwise returns `false`.
750 fn check_plus(&mut self) -> bool {
751 self.check_or_expected(
752 self.token.is_like_plus(),
753 TokenType::Token(token::BinOp(token::Plus)),
754 )
9fa01778
XL
755 }
756
74b04a01
XL
757 /// Eats the expected token if it's present possibly breaking
758 /// compound tokens like multi-character operators in process.
759 /// Returns `true` if the token was eaten.
760 fn break_and_eat(&mut self, expected: TokenKind) -> bool {
761 if self.token.kind == expected {
762 self.bump();
763 return true;
764 }
765 match self.token.kind.break_two_token_op() {
766 Some((first, second)) if first == expected => {
767 let first_span = self.sess.source_map().start_point(self.token.span);
768 let second_span = self.token.span.with_lo(first_span.hi());
769 self.token = Token::new(first, first_span);
fc512014
XL
770 // Keep track of this token - if we end token capturing now,
771 // we'll want to append this token to the captured stream.
772 //
773 // If we consume any additional tokens, then this token
774 // is not needed (we'll capture the entire 'glued' token),
04454e1e 775 // and `bump` will set this field to `None`
cdc7bbd5 776 self.token_cursor.break_last_token = true;
29967ef6
XL
777 // Use the spacing of the glued token as the spacing
778 // of the unglued second token.
779 self.bump_with((Token::new(second, second_span), self.token_spacing));
94b46f34
XL
780 true
781 }
74b04a01
XL
782 _ => {
783 self.expected_tokens.push(TokenType::Token(expected));
784 false
94b46f34 785 }
94b46f34
XL
786 }
787 }
788
74b04a01
XL
789 /// Eats `+` possibly breaking tokens like `+=` in process.
790 fn eat_plus(&mut self) -> bool {
791 self.break_and_eat(token::BinOp(token::Plus))
792 }
793
794 /// Eats `&` possibly breaking tokens like `&&` in process.
795 /// Signals an error if `&` is not eaten.
9cc50fc6 796 fn expect_and(&mut self) -> PResult<'a, ()> {
74b04a01 797 if self.break_and_eat(token::BinOp(token::And)) { Ok(()) } else { self.unexpected() }
ea8adc8c
XL
798 }
799
74b04a01
XL
800 /// Eats `|` possibly breaking tokens like `||` in process.
801 /// Signals an error if `|` was not eaten.
ea8adc8c 802 fn expect_or(&mut self) -> PResult<'a, ()> {
74b04a01 803 if self.break_and_eat(token::BinOp(token::Or)) { Ok(()) } else { self.unexpected() }
970d7e83
LB
804 }
805
74b04a01 806 /// Eats `<` possibly breaking tokens like `<<` in process.
9cc50fc6 807 fn eat_lt(&mut self) -> bool {
74b04a01 808 let ate = self.break_and_eat(token::Lt);
9fa01778
XL
809 if ate {
810 // See doc comment for `unmatched_angle_bracket_count`.
811 self.unmatched_angle_bracket_count += 1;
812 self.max_angle_bracket_count += 1;
813 debug!("eat_lt: (increment) count={:?}", self.unmatched_angle_bracket_count);
1a4d82fc 814 }
9fa01778 815 ate
1a4d82fc
JJ
816 }
817
74b04a01
XL
818 /// Eats `<` possibly breaking tokens like `<<` in process.
819 /// Signals an error if `<` was not eaten.
9cc50fc6 820 fn expect_lt(&mut self) -> PResult<'a, ()> {
74b04a01 821 if self.eat_lt() { Ok(()) } else { self.unexpected() }
1a4d82fc
JJ
822 }
823
74b04a01
XL
824 /// Eats `>` possibly breaking tokens like `>>` in process.
825 /// Signals an error if `>` was not eaten.
94b46f34 826 fn expect_gt(&mut self) -> PResult<'a, ()> {
74b04a01
XL
827 if self.break_and_eat(token::Gt) {
828 // See doc comment for `unmatched_angle_bracket_count`.
829 if self.unmatched_angle_bracket_count > 0 {
830 self.unmatched_angle_bracket_count -= 1;
831 debug!("expect_gt: (decrement) count={:?}", self.unmatched_angle_bracket_count);
dfeec247 832 }
74b04a01
XL
833 Ok(())
834 } else {
835 self.unexpected()
1a4d82fc
JJ
836 }
837 }
838
416331ca 839 fn expect_any_with_type(&mut self, kets: &[&TokenKind], expect: TokenExpectType) -> bool {
dfeec247
XL
840 kets.iter().any(|k| match expect {
841 TokenExpectType::Expect => self.check(k),
842 TokenExpectType::NoExpect => self.token == **k,
416331ca
XL
843 })
844 }
845
e74abb32 846 fn parse_seq_to_before_tokens<T>(
b7449926 847 &mut self,
dc9dc135 848 kets: &[&TokenKind],
b7449926
XL
849 sep: SeqSep,
850 expect: TokenExpectType,
416331ca 851 mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
9ffffee4 852 ) -> PResult<'a, (ThinVec<T>, bool /* trailing */, bool /* recovered */)> {
9fa01778
XL
853 let mut first = true;
854 let mut recovered = false;
416331ca 855 let mut trailing = false;
9ffffee4 856 let mut v = ThinVec::new();
cdc7bbd5 857
416331ca
XL
858 while !self.expect_any_with_type(kets, expect) {
859 if let token::CloseDelim(..) | token::Eof = self.token.kind {
dfeec247 860 break;
416331ca 861 }
487cf647 862 if let Some(t) = &sep.sep {
7cac9316
XL
863 if first {
864 first = false;
865 } else {
9fa01778 866 match self.expect(t) {
c295e0f8
XL
867 Ok(false) => {
868 self.current_closure.take();
869 }
9fa01778 870 Ok(true) => {
c295e0f8 871 self.current_closure.take();
9fa01778
XL
872 recovered = true;
873 break;
abe05a73 874 }
60c5eb7d 875 Err(mut expect_err) => {
74b04a01 876 let sp = self.prev_token.span.shrink_to_hi();
60c5eb7d
XL
877 let token_str = pprust::token_kind_to_string(t);
878
c295e0f8
XL
879 match self.current_closure.take() {
880 Some(closure_spans) if self.token.kind == TokenKind::Semi => {
881 // Finding a semicolon instead of a comma
882 // after a closure body indicates that the
883 // closure body may be a block but the user
884 // forgot to put braces around its
885 // statements.
886
887 self.recover_missing_braces_around_closure_body(
888 closure_spans,
889 expect_err,
890 )?;
891
892 continue;
893 }
894
895 _ => {
896 // Attempt to keep parsing if it was a similar separator.
487cf647 897 if let Some(tokens) = t.similar_tokens() {
353b0b11 898 if tokens.contains(&self.token.kind) {
c295e0f8
XL
899 self.bump();
900 }
901 }
9fa01778
XL
902 }
903 }
60c5eb7d 904
f9f354fc
XL
905 // If this was a missing `@` in a binding pattern
906 // bail with a suggestion
907 // https://github.com/rust-lang/rust/issues/72373
f035d41b 908 if self.prev_token.is_ident() && self.token.kind == token::DotDot {
f9f354fc
XL
909 let msg = format!(
910 "if you meant to bind the contents of \
911 the rest of the array pattern into `{}`, use `@`",
912 pprust::token_to_string(&self.prev_token)
913 );
914 expect_err
915 .span_suggestion_verbose(
916 self.prev_token.span.shrink_to_hi().until(self.token.span),
917 &msg,
04454e1e 918 " @ ",
f9f354fc
XL
919 Applicability::MaybeIncorrect,
920 )
921 .emit();
922 break;
923 }
924
e1599b0c 925 // Attempt to keep parsing if it was an omitted separator.
9fa01778
XL
926 match f(self) {
927 Ok(t) => {
60c5eb7d
XL
928 // Parsed successfully, therefore most probably the code only
929 // misses a separator.
930 expect_err
931 .span_suggestion_short(
5869c6ff 932 sp,
60c5eb7d 933 &format!("missing `{}`", token_str),
04454e1e 934 token_str,
60c5eb7d
XL
935 Applicability::MaybeIncorrect,
936 )
937 .emit();
938
9fa01778
XL
939 v.push(t);
940 continue;
dfeec247 941 }
5e7ed085 942 Err(e) => {
60c5eb7d
XL
943 // Parsing failed, therefore it must be something more serious
944 // than just a missing separator.
487cf647
FG
945 for xx in &e.children {
946 // propagate the help message from sub error 'e' to main error 'expect_err;
947 expect_err.children.push(xx.clone());
948 }
60c5eb7d
XL
949 expect_err.emit();
950
9fa01778
XL
951 e.cancel();
952 break;
953 }
abe05a73
XL
954 }
955 }
7453a54e
SL
956 }
957 }
7453a54e 958 }
416331ca
XL
959 if sep.trailing_sep_allowed && self.expect_any_with_type(kets, expect) {
960 trailing = true;
7453a54e
SL
961 break;
962 }
963
abe05a73
XL
964 let t = f(self)?;
965 v.push(t);
970d7e83 966 }
7453a54e 967
416331ca 968 Ok((v, trailing, recovered))
970d7e83
LB
969 }
970
c295e0f8
XL
971 fn recover_missing_braces_around_closure_body(
972 &mut self,
973 closure_spans: ClosureSpans,
5e7ed085 974 mut expect_err: DiagnosticBuilder<'_, ErrorGuaranteed>,
c295e0f8
XL
975 ) -> PResult<'a, ()> {
976 let initial_semicolon = self.token.span;
977
978 while self.eat(&TokenKind::Semi) {
9ffffee4
FG
979 let _ =
980 self.parse_stmt_without_recovery(false, ForceCollect::Yes).unwrap_or_else(|e| {
981 e.cancel();
982 None
983 });
c295e0f8
XL
984 }
985
986 expect_err.set_primary_message(
987 "closure bodies that contain statements must be surrounded by braces",
988 );
989
990 let preceding_pipe_span = closure_spans.closing_pipe;
991 let following_token_span = self.token.span;
992
993 let mut first_note = MultiSpan::from(vec![initial_semicolon]);
994 first_note.push_span_label(
995 initial_semicolon,
064997fb 996 "this `;` turns the preceding closure into a statement",
c295e0f8
XL
997 );
998 first_note.push_span_label(
999 closure_spans.body,
064997fb 1000 "this expression is a statement because of the trailing semicolon",
c295e0f8
XL
1001 );
1002 expect_err.span_note(first_note, "statement found outside of a block");
1003
1004 let mut second_note = MultiSpan::from(vec![closure_spans.whole_closure]);
064997fb 1005 second_note.push_span_label(closure_spans.whole_closure, "this is the parsed closure...");
c295e0f8
XL
1006 second_note.push_span_label(
1007 following_token_span,
064997fb 1008 "...but likely you meant the closure to end here",
c295e0f8
XL
1009 );
1010 expect_err.span_note(second_note, "the closure body may be incorrectly delimited");
1011
1012 expect_err.set_span(vec![preceding_pipe_span, following_token_span]);
1013
1014 let opening_suggestion_str = " {".to_string();
1015 let closing_suggestion_str = "}".to_string();
1016
1017 expect_err.multipart_suggestion(
1018 "try adding braces",
1019 vec![
1020 (preceding_pipe_span.shrink_to_hi(), opening_suggestion_str),
1021 (following_token_span.shrink_to_lo(), closing_suggestion_str),
1022 ],
1023 Applicability::MaybeIncorrect,
1024 );
1025
1026 expect_err.emit();
1027
1028 Ok(())
1029 }
1030
dfeec247
XL
1031 /// Parses a sequence, not including the closing delimiter. The function
1032 /// `f` must consume tokens until reaching the next separator or
1033 /// closing bracket.
1034 fn parse_seq_to_before_end<T>(
1035 &mut self,
1036 ket: &TokenKind,
1037 sep: SeqSep,
1038 f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
9ffffee4 1039 ) -> PResult<'a, (ThinVec<T>, bool, bool)> {
dfeec247
XL
1040 self.parse_seq_to_before_tokens(&[ket], sep, TokenExpectType::Expect, f)
1041 }
1042
1043 /// Parses a sequence, including the closing delimiter. The function
1044 /// `f` must consume tokens until reaching the next separator or
1045 /// closing bracket.
1046 fn parse_seq_to_end<T>(
1047 &mut self,
1048 ket: &TokenKind,
1049 sep: SeqSep,
1050 f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
9ffffee4 1051 ) -> PResult<'a, (ThinVec<T>, bool /* trailing */)> {
dfeec247
XL
1052 let (val, trailing, recovered) = self.parse_seq_to_before_end(ket, sep, f)?;
1053 if !recovered {
1054 self.eat(ket);
1055 }
1056 Ok((val, trailing))
1057 }
1058
9fa01778
XL
1059 /// Parses a sequence, including the closing delimiter. The function
1060 /// `f` must consume tokens until reaching the next separator or
1a4d82fc 1061 /// closing bracket.
416331ca 1062 fn parse_unspanned_seq<T>(
9fa01778 1063 &mut self,
dc9dc135
XL
1064 bra: &TokenKind,
1065 ket: &TokenKind,
9fa01778 1066 sep: SeqSep,
416331ca 1067 f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
9ffffee4 1068 ) -> PResult<'a, (ThinVec<T>, bool)> {
54a0048b 1069 self.expect(bra)?;
dfeec247 1070 self.parse_seq_to_end(ket, sep, f)
416331ca
XL
1071 }
1072
1073 fn parse_delim_comma_seq<T>(
1074 &mut self,
04454e1e 1075 delim: Delimiter,
416331ca 1076 f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
9ffffee4 1077 ) -> PResult<'a, (ThinVec<T>, bool)> {
416331ca
XL
1078 self.parse_unspanned_seq(
1079 &token::OpenDelim(delim),
1080 &token::CloseDelim(delim),
1081 SeqSep::trailing_allowed(token::Comma),
1082 f,
1083 )
1084 }
1085
1086 fn parse_paren_comma_seq<T>(
1087 &mut self,
1088 f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>,
9ffffee4 1089 ) -> PResult<'a, (ThinVec<T>, bool)> {
04454e1e 1090 self.parse_delim_comma_seq(Delimiter::Parenthesis, f)
970d7e83
LB
1091 }
1092
74b04a01 1093 /// Advance the parser by one token using provided token as the next one.
5e7ed085
FG
1094 fn bump_with(&mut self, next: (Token, Spacing)) {
1095 self.inlined_bump_with(next)
1096 }
1097
1098 /// This always-inlined version should only be used on hot code paths.
1099 #[inline(always)]
1100 fn inlined_bump_with(&mut self, (next_token, next_spacing): (Token, Spacing)) {
74b04a01
XL
1101 // Update the current and previous tokens.
1102 self.prev_token = mem::replace(&mut self.token, next_token);
29967ef6 1103 self.token_spacing = next_spacing;
9e0c209e 1104
74b04a01 1105 // Diagnostics.
7453a54e 1106 self.expected_tokens.clear();
223e47cc 1107 }
7453a54e 1108
74b04a01
XL
1109 /// Advance the parser by one token.
1110 pub fn bump(&mut self) {
04454e1e
FG
1111 // Note: destructuring here would give nicer code, but it was found in #96210 to be slower
1112 // than `.0`/`.1` access.
1113 let mut next = self.token_cursor.inlined_next(self.desugar_doc_comments);
1114 self.token_cursor.num_next_calls += 1;
1115 // We've retrieved an token from the underlying
1116 // cursor, so we no longer need to worry about
1117 // an unglued token. See `break_and_eat` for more details
1118 self.token_cursor.break_last_token = false;
1119 if next.0.span.is_dummy() {
1120 // Tweak the location for better diagnostics, but keep syntactic context intact.
1121 let fallback_span = self.token.span;
1122 next.0.span = fallback_span.with_ctxt(next.0.span.ctxt());
1123 }
1124 debug_assert!(!matches!(
1125 next.0.kind,
1126 token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
1127 ));
1128 self.inlined_bump_with(next)
74b04a01
XL
1129 }
1130
e74abb32
XL
1131 /// Look-ahead `dist` tokens of `self.token` and get access to that token there.
1132 /// When `dist == 0` then the current token is looked at.
1133 pub fn look_ahead<R>(&self, dist: usize, looker: impl FnOnce(&Token) -> R) -> R {
c30ab7b3 1134 if dist == 0 {
e74abb32 1135 return looker(&self.token);
223e47cc 1136 }
8bb4bdeb 1137
9ffffee4
FG
1138 let tree_cursor = &self.token_cursor.tree_cursor;
1139 if let Some(&(_, delim, span)) = self.token_cursor.stack.last()
1140 && delim != Delimiter::Invisible
1141 {
cdc7bbd5 1142 let all_normal = (0..dist).all(|i| {
9ffffee4 1143 let token = tree_cursor.look_ahead(i);
04454e1e 1144 !matches!(token, Some(TokenTree::Delimited(_, Delimiter::Invisible, _)))
cdc7bbd5
XL
1145 });
1146 if all_normal {
9ffffee4 1147 return match tree_cursor.look_ahead(dist - 1) {
cdc7bbd5 1148 Some(tree) => match tree {
064997fb 1149 TokenTree::Token(token, _) => looker(token),
cdc7bbd5
XL
1150 TokenTree::Delimited(dspan, delim, _) => {
1151 looker(&Token::new(token::OpenDelim(*delim), dspan.open))
1152 }
1153 },
04454e1e 1154 None => looker(&Token::new(token::CloseDelim(delim), span.close)),
cdc7bbd5
XL
1155 };
1156 }
29967ef6 1157 }
cdc7bbd5
XL
1158
1159 let mut cursor = self.token_cursor.clone();
1160 let mut i = 0;
1161 let mut token = Token::dummy();
1162 while i < dist {
04454e1e 1163 token = cursor.next(/* desugar_doc_comments */ false).0;
cdc7bbd5
XL
1164 if matches!(
1165 token.kind,
04454e1e 1166 token::OpenDelim(Delimiter::Invisible) | token::CloseDelim(Delimiter::Invisible)
cdc7bbd5
XL
1167 ) {
1168 continue;
1169 }
1170 i += 1;
1171 }
1172 return looker(&token);
223e47cc 1173 }
ff7c6d11 1174
dc9dc135
XL
1175 /// Returns whether any of the given keywords are `dist` tokens ahead of the current one.
1176 fn is_keyword_ahead(&self, dist: usize, kws: &[Symbol]) -> bool {
1177 self.look_ahead(dist, |t| kws.iter().any(|&kw| t.is_keyword(kw)))
223e47cc 1178 }
223e47cc 1179
9fa01778 1180 /// Parses asyncness: `async` or nothing.
487cf647
FG
1181 fn parse_asyncness(&mut self, case: Case) -> Async {
1182 if self.eat_keyword_case(kw::Async, case) {
74b04a01
XL
1183 let span = self.prev_token.uninterpolated_span();
1184 Async::Yes { span, closure_id: DUMMY_NODE_ID, return_impl_trait_id: DUMMY_NODE_ID }
8faf50e0 1185 } else {
74b04a01 1186 Async::No
8faf50e0
XL
1187 }
1188 }
1189
9fa01778 1190 /// Parses unsafety: `unsafe` or nothing.
487cf647
FG
1191 fn parse_unsafety(&mut self, case: Case) -> Unsafe {
1192 if self.eat_keyword_case(kw::Unsafe, case) {
74b04a01
XL
1193 Unsafe::Yes(self.prev_token.uninterpolated_span())
1194 } else {
1195 Unsafe::No
1196 }
1197 }
1198
1199 /// Parses constness: `const` or nothing.
487cf647 1200 fn parse_constness(&mut self, case: Case) -> Const {
9ffffee4
FG
1201 self.parse_constness_(case, false)
1202 }
1203
353b0b11
FG
1204 /// Parses constness for closures (case sensitive, feature-gated)
1205 fn parse_closure_constness(&mut self) -> Const {
1206 let constness = self.parse_constness_(Case::Sensitive, true);
1207 if let Const::Yes(span) = constness {
1208 self.sess.gated_spans.gate(sym::const_closures, span);
1209 }
1210 constness
9ffffee4
FG
1211 }
1212
1213 fn parse_constness_(&mut self, case: Case, is_closure: bool) -> Const {
1214 // Avoid const blocks and const closures to be parsed as const items
1215 if (self.check_const_closure() == is_closure)
1216 && self.look_ahead(1, |t| t != &token::OpenDelim(Delimiter::Brace))
487cf647 1217 && self.eat_keyword_case(kw::Const, case)
29967ef6 1218 {
74b04a01
XL
1219 Const::Yes(self.prev_token.uninterpolated_span())
1220 } else {
1221 Const::No
1222 }
1a4d82fc
JJ
1223 }
1224
29967ef6 1225 /// Parses inline const expressions.
3c0e092e
XL
1226 fn parse_const_block(&mut self, span: Span, pat: bool) -> PResult<'a, P<Expr>> {
1227 if pat {
1228 self.sess.gated_spans.gate(sym::inline_const_pat, span);
1229 } else {
1230 self.sess.gated_spans.gate(sym::inline_const, span);
1231 }
29967ef6 1232 self.eat_keyword(kw::Const);
04454e1e 1233 let (attrs, blk) = self.parse_inner_attrs_and_block()?;
29967ef6
XL
1234 let anon_const = AnonConst {
1235 id: DUMMY_NODE_ID,
f2b60f7d 1236 value: self.mk_expr(blk.span, ExprKind::Block(blk, None)),
29967ef6 1237 };
fc512014 1238 let blk_span = anon_const.value.span;
9c376795 1239 Ok(self.mk_expr_with_attrs(span.to(blk_span), ExprKind::ConstBlock(anon_const), attrs))
29967ef6
XL
1240 }
1241
416331ca
XL
1242 /// Parses mutability (`mut` or nothing).
1243 fn parse_mutability(&mut self) -> Mutability {
dfeec247 1244 if self.eat_keyword(kw::Mut) { Mutability::Mut } else { Mutability::Not }
223e47cc
LB
1245 }
1246
e74abb32
XL
1247 /// Possibly parses mutability (`const` or `mut`).
1248 fn parse_const_or_mut(&mut self) -> Option<Mutability> {
1249 if self.eat_keyword(kw::Mut) {
dfeec247 1250 Some(Mutability::Mut)
e74abb32 1251 } else if self.eat_keyword(kw::Const) {
dfeec247 1252 Some(Mutability::Not)
e74abb32
XL
1253 } else {
1254 None
1255 }
1256 }
1257
416331ca 1258 fn parse_field_name(&mut self) -> PResult<'a, Ident> {
dfeec247
XL
1259 if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = self.token.kind
1260 {
2b03887a
FG
1261 if let Some(suffix) = suffix {
1262 self.expect_no_tuple_index_suffix(self.token.span, suffix);
1263 }
416331ca 1264 self.bump();
74b04a01 1265 Ok(Ident::new(symbol, self.prev_token.span))
223e47cc 1266 } else {
6a06907d 1267 self.parse_ident_common(true)
223e47cc
LB
1268 }
1269 }
1270
487cf647
FG
1271 fn parse_delim_args(&mut self) -> PResult<'a, P<DelimArgs>> {
1272 if let Some(args) = self.parse_delim_args_inner() { Ok(P(args)) } else { self.unexpected() }
60c5eb7d
XL
1273 }
1274
487cf647
FG
1275 fn parse_attr_args(&mut self) -> PResult<'a, AttrArgs> {
1276 Ok(if let Some(args) = self.parse_delim_args_inner() {
1277 AttrArgs::Delimited(args)
1278 } else {
1279 if self.eat(&token::Eq) {
1280 let eq_span = self.prev_token.span;
1281 AttrArgs::Eq(eq_span, AttrArgsEq::Ast(self.parse_expr_force_collect()?))
1282 } else {
1283 AttrArgs::Empty
1284 }
1285 })
60c5eb7d
XL
1286 }
1287
487cf647 1288 fn parse_delim_args_inner(&mut self) -> Option<DelimArgs> {
9ffffee4 1289 let delimited = self.check(&token::OpenDelim(Delimiter::Parenthesis))
487cf647 1290 || self.check(&token::OpenDelim(Delimiter::Bracket))
9ffffee4
FG
1291 || self.check(&token::OpenDelim(Delimiter::Brace));
1292
1293 delimited.then(|| {
1294 // We've confirmed above that there is a delimiter so unwrapping is OK.
1295 let TokenTree::Delimited(dspan, delim, tokens) = self.parse_token_tree() else { unreachable!() };
1296
1297 DelimArgs { dspan, delim: MacDelimiter::from_token(delim).unwrap(), tokens }
1298 })
416331ca 1299 }
041b39d2 1300
e74abb32
XL
1301 fn parse_or_use_outer_attributes(
1302 &mut self,
6a06907d
XL
1303 already_parsed_attrs: Option<AttrWrapper>,
1304 ) -> PResult<'a, AttrWrapper> {
416331ca
XL
1305 if let Some(attrs) = already_parsed_attrs {
1306 Ok(attrs)
970d7e83 1307 } else {
6a06907d 1308 self.parse_outer_attributes()
970d7e83
LB
1309 }
1310 }
1311
416331ca 1312 /// Parses a single token tree from the input.
3dfed10e 1313 pub(crate) fn parse_token_tree(&mut self) -> TokenTree {
dc9dc135 1314 match self.token.kind {
416331ca 1315 token::OpenDelim(..) => {
9ffffee4
FG
1316 // Grab the tokens within the delimiters.
1317 let tree_cursor = &self.token_cursor.tree_cursor;
1318 let stream = tree_cursor.stream.clone();
1319 let (_, delim, span) = *self.token_cursor.stack.last().unwrap();
04454e1e
FG
1320
1321 // Advance the token cursor through the entire delimited
1322 // sequence. After getting the `OpenDelim` we are *within* the
1323 // delimited sequence, i.e. at depth `d`. After getting the
1324 // matching `CloseDelim` we are *after* the delimited sequence,
1325 // i.e. at depth `d - 1`.
1326 let target_depth = self.token_cursor.stack.len() - 1;
1327 loop {
29967ef6
XL
1328 // Advance one token at a time, so `TokenCursor::next()`
1329 // can capture these tokens if necessary.
1330 self.bump();
04454e1e
FG
1331 if self.token_cursor.stack.len() == target_depth {
1332 debug_assert!(matches!(self.token.kind, token::CloseDelim(_)));
1333 break;
1334 }
29967ef6 1335 }
04454e1e 1336
29967ef6 1337 // Consume close delimiter
0731742a 1338 self.bump();
29967ef6 1339 TokenTree::Delimited(span, delim, stream)
dfeec247 1340 }
416331ca
XL
1341 token::CloseDelim(_) | token::Eof => unreachable!(),
1342 _ => {
416331ca 1343 self.bump();
064997fb 1344 TokenTree::Token(self.prev_token.clone(), Spacing::Alone)
0731742a 1345 }
0731742a
XL
1346 }
1347 }
1348
416331ca
XL
1349 /// Parses a stream of tokens into a list of `TokenTree`s, up to EOF.
1350 pub fn parse_all_token_trees(&mut self) -> PResult<'a, Vec<TokenTree>> {
1351 let mut tts = Vec::new();
1352 while self.token != token::Eof {
1353 tts.push(self.parse_token_tree());
54a0048b 1354 }
416331ca 1355 Ok(tts)
54a0048b
SL
1356 }
1357
416331ca
XL
1358 pub fn parse_tokens(&mut self) -> TokenStream {
1359 let mut result = Vec::new();
1360 loop {
1361 match self.token.kind {
1362 token::Eof | token::CloseDelim(..) => break,
064997fb 1363 _ => result.push(self.parse_token_tree()),
ff7c6d11 1364 }
223e47cc 1365 }
416331ca 1366 TokenStream::new(result)
223e47cc
LB
1367 }
1368
416331ca
XL
1369 /// Evaluates the closure with restrictions in place.
1370 ///
1371 /// Afters the closure is evaluated, restrictions are reset.
e74abb32 1372 fn with_res<T>(&mut self, res: Restrictions, f: impl FnOnce(&mut Self) -> T) -> T {
416331ca 1373 let old = self.restrictions;
e74abb32
XL
1374 self.restrictions = res;
1375 let res = f(self);
416331ca 1376 self.restrictions = old;
e74abb32 1377 res
8faf50e0
XL
1378 }
1379
923072b8
FG
1380 /// Parses `pub` and `pub(in path)` plus shortcuts `pub(crate)` for `pub(in crate)`, `pub(self)`
1381 /// for `pub(in self)` and `pub(super)` for `pub(in super)`.
416331ca 1382 /// If the following element can't be a tuple (i.e., it's a function definition), then
3c0e092e 1383 /// it's not a tuple struct field), and the contents within the parentheses aren't valid,
416331ca 1384 /// so emit a proper diagnostic.
1b1a35ee
XL
1385 // Public for rustfmt usage.
1386 pub fn parse_visibility(&mut self, fbt: FollowedByType) -> PResult<'a, Visibility> {
04454e1e 1387 maybe_whole!(self, NtVis, |x| x.into_inner());
dc9dc135 1388
416331ca
XL
1389 if !self.eat_keyword(kw::Pub) {
1390 // We need a span for our `Spanned<VisibilityKind>`, but there's inherently no
1391 // keyword to grab a span from for inherited visibility; an empty span at the
1392 // beginning of the current token would seem to be the "Schelling span".
1b1a35ee
XL
1393 return Ok(Visibility {
1394 span: self.token.span.shrink_to_lo(),
1395 kind: VisibilityKind::Inherited,
1396 tokens: None,
1397 });
416331ca 1398 }
74b04a01 1399 let lo = self.prev_token.span;
223e47cc 1400
04454e1e 1401 if self.check(&token::OpenDelim(Delimiter::Parenthesis)) {
416331ca
XL
1402 // We don't `self.bump()` the `(` yet because this might be a struct definition where
1403 // `()` or a tuple might be allowed. For example, `struct Struct(pub (), pub (usize));`.
1404 // Because of this, we only `bump` the `(` if we're assured it is appropriate to do so
1405 // by the following tokens.
923072b8 1406 if self.is_keyword_ahead(1, &[kw::In]) {
e74abb32 1407 // Parse `pub(in path)`.
416331ca
XL
1408 self.bump(); // `(`
1409 self.bump(); // `in`
1410 let path = self.parse_path(PathStyle::Mod)?; // `path`
04454e1e 1411 self.expect(&token::CloseDelim(Delimiter::Parenthesis))?; // `)`
f2b60f7d
FG
1412 let vis = VisibilityKind::Restricted {
1413 path: P(path),
1414 id: ast::DUMMY_NODE_ID,
1415 shorthand: false,
1416 };
1b1a35ee
XL
1417 return Ok(Visibility {
1418 span: lo.to(self.prev_token.span),
1419 kind: vis,
1420 tokens: None,
1421 });
04454e1e 1422 } else if self.look_ahead(2, |t| t == &token::CloseDelim(Delimiter::Parenthesis))
923072b8 1423 && self.is_keyword_ahead(1, &[kw::Crate, kw::Super, kw::SelfLower])
416331ca 1424 {
923072b8 1425 // Parse `pub(crate)`, `pub(self)`, or `pub(super)`.
416331ca 1426 self.bump(); // `(`
923072b8 1427 let path = self.parse_path(PathStyle::Mod)?; // `crate`/`super`/`self`
04454e1e 1428 self.expect(&token::CloseDelim(Delimiter::Parenthesis))?; // `)`
f2b60f7d
FG
1429 let vis = VisibilityKind::Restricted {
1430 path: P(path),
1431 id: ast::DUMMY_NODE_ID,
1432 shorthand: true,
1433 };
1b1a35ee
XL
1434 return Ok(Visibility {
1435 span: lo.to(self.prev_token.span),
1436 kind: vis,
1437 tokens: None,
1438 });
60c5eb7d
XL
1439 } else if let FollowedByType::No = fbt {
1440 // Provide this diagnostic if a type cannot follow;
1441 // in particular, if this is not a tuple struct.
e74abb32
XL
1442 self.recover_incorrect_vis_restriction()?;
1443 // Emit diagnostic, but continue with public visibility.
532ac7d7 1444 }
223e47cc 1445 }
223e47cc 1446
1b1a35ee 1447 Ok(Visibility { span: lo, kind: VisibilityKind::Public, tokens: None })
223e47cc
LB
1448 }
1449
e74abb32
XL
1450 /// Recovery for e.g. `pub(something) fn ...` or `struct X { pub(something) y: Z }`
1451 fn recover_incorrect_vis_restriction(&mut self) -> PResult<'a, ()> {
1452 self.bump(); // `(`
1453 let path = self.parse_path(PathStyle::Mod)?;
04454e1e 1454 self.expect(&token::CloseDelim(Delimiter::Parenthesis))?; // `)`
e74abb32 1455
e74abb32 1456 let path_str = pprust::path_to_string(&path);
2b03887a 1457 self.sess.emit_err(IncorrectVisibilityRestriction { span: path.span, inner_str: path_str });
e74abb32
XL
1458
1459 Ok(())
1460 }
1461
60c5eb7d 1462 /// Parses `extern string_literal?`.
487cf647
FG
1463 fn parse_extern(&mut self, case: Case) -> Extern {
1464 if self.eat_keyword_case(kw::Extern, case) {
064997fb
FG
1465 let mut extern_span = self.prev_token.span;
1466 let abi = self.parse_abi();
1467 if let Some(abi) = abi {
1468 extern_span = extern_span.to(abi.span);
1469 }
1470 Extern::from_abi(abi, extern_span)
1471 } else {
1472 Extern::None
1473 }
e74abb32
XL
1474 }
1475
60c5eb7d
XL
1476 /// Parses a string literal as an ABI spec.
1477 fn parse_abi(&mut self) -> Option<StrLit> {
1478 match self.parse_str_lit() {
1479 Ok(str_lit) => Some(str_lit),
1480 Err(Some(lit)) => match lit.kind {
f2b60f7d 1481 ast::LitKind::Err => None,
60c5eb7d 1482 _ => {
2b03887a 1483 self.sess.emit_err(NonStringAbiLiteral { span: lit.span });
60c5eb7d 1484 None
223e47cc 1485 }
dfeec247 1486 },
60c5eb7d 1487 Err(None) => None,
223e47cc
LB
1488 }
1489 }
1490
04454e1e 1491 pub fn collect_tokens_no_attrs<R: HasAttrs + HasTokens>(
5869c6ff
XL
1492 &mut self,
1493 f: impl FnOnce(&mut Self) -> PResult<'a, R>,
1494 ) -> PResult<'a, R> {
6a06907d
XL
1495 // The only reason to call `collect_tokens_no_attrs` is if you want tokens, so use
1496 // `ForceCollect::Yes`
1497 self.collect_tokens_trailing_token(
1498 AttrWrapper::empty(),
1499 ForceCollect::Yes,
1500 |this, _attrs| Ok((f(this)?, TrailingToken::None)),
1501 )
3b2f2976
XL
1502 }
1503
0531ce1d
XL
1504 /// `::{` or `::*`
1505 fn is_import_coupler(&mut self) -> bool {
dfeec247
XL
1506 self.check(&token::ModSep)
1507 && self.look_ahead(1, |t| {
04454e1e 1508 *t == token::OpenDelim(Delimiter::Brace) || *t == token::BinOp(token::Star)
dfeec247 1509 })
a7813a04 1510 }
1b1a35ee
XL
1511
1512 pub fn clear_expected_tokens(&mut self) {
1513 self.expected_tokens.clear();
1514 }
9c376795
FG
1515
1516 pub fn approx_token_stream_pos(&self) -> usize {
1517 self.token_cursor.num_next_calls
1518 }
e74abb32 1519}
532ac7d7 1520
923072b8 1521pub(crate) fn make_unclosed_delims_error(
9ffffee4 1522 unmatched: UnmatchedDelim,
e74abb32 1523 sess: &ParseSess,
5e7ed085 1524) -> Option<DiagnosticBuilder<'_, ErrorGuaranteed>> {
e74abb32 1525 // `None` here means an `Eof` was found. We already emit those errors elsewhere, we add them to
9ffffee4 1526 // `unmatched_delims` only for error recovery in the `Parser`.
e74abb32 1527 let found_delim = unmatched.found_delim?;
2b03887a 1528 let mut spans = vec![unmatched.found_span];
e74abb32 1529 if let Some(sp) = unmatched.unclosed_span {
2b03887a
FG
1530 spans.push(sp);
1531 };
1532 let err = MismatchedClosingDelimiter {
1533 spans,
1534 delimiter: pprust::token_kind_to_string(&token::CloseDelim(found_delim)).to_string(),
1535 unmatched: unmatched.found_span,
1536 opening_candidate: unmatched.candidate_span,
1537 unclosed: unmatched.unclosed_span,
1538 }
1539 .into_diagnostic(&sess.span_diagnostic);
e74abb32 1540 Some(err)
223e47cc 1541}
9fa01778 1542
9ffffee4 1543pub fn emit_unclosed_delims(unclosed_delims: &mut Vec<UnmatchedDelim>, sess: &ParseSess) {
353b0b11
FG
1544 let _ = sess.reached_eof.fetch_or(
1545 unclosed_delims.iter().any(|unmatched_delim| unmatched_delim.found_delim.is_none()),
1546 Ordering::Relaxed,
1547 );
e74abb32 1548 for unmatched in unclosed_delims.drain(..) {
f9f354fc 1549 if let Some(mut e) = make_unclosed_delims_error(unmatched, sess) {
74b04a01 1550 e.emit();
f9f354fc 1551 }
9fa01778 1552 }
9fa01778 1553}
cdc7bbd5 1554
f2b60f7d
FG
1555/// A helper struct used when building an `AttrTokenStream` from
1556/// a `LazyAttrTokenStream`. Both delimiter and non-delimited tokens
cdc7bbd5 1557/// are stored as `FlatToken::Token`. A vector of `FlatToken`s
f2b60f7d
FG
1558/// is then 'parsed' to build up an `AttrTokenStream` with nested
1559/// `AttrTokenTree::Delimited` tokens.
cdc7bbd5
XL
1560#[derive(Debug, Clone)]
1561pub enum FlatToken {
1562 /// A token - this holds both delimiter (e.g. '{' and '}')
1563 /// and non-delimiter tokens
1564 Token(Token),
1565 /// Holds the `AttributesData` for an AST node. The
1566 /// `AttributesData` is inserted directly into the
f2b60f7d
FG
1567 /// constructed `AttrTokenStream` as
1568 /// an `AttrTokenTree::Attributes`.
cdc7bbd5
XL
1569 AttrTarget(AttributesData),
1570 /// A special 'empty' token that is ignored during the conversion
f2b60f7d 1571 /// to an `AttrTokenStream`. This is used to simplify the
cdc7bbd5
XL
1572 /// handling of replace ranges.
1573 Empty,
1574}
5e7ed085
FG
1575
1576#[derive(Debug)]
1577pub enum NtOrTt {
1578 Nt(Nonterminal),
1579 Tt(TokenTree),
1580}