1 //! This is an NFA-based parser, which calls out to the main rust parser for named non-terminals
2 //! (which it commits to fully when it hits one in a grammar). There's a set of current NFA threads
3 //! and a set of next ones. Instead of NTs, we have a special case for Kleene star. The big-O, in
4 //! pathological cases, is worse than traditional use of NFA or Earley parsing, but it's an easier
5 //! fit for Macro-by-Example-style rules.
7 //! (In order to prevent the pathological case, we'd need to lazily construct the resulting
8 //! `NamedMatch`es at the very end. It'd be a pain, and require more memory to keep around old
9 //! items, but it would also save overhead)
11 //! We don't say this parser uses the Earley algorithm, because it's unnecessarily inaccurate.
12 //! The macro parser restricts itself to the features of finite state automata. Earley parsers
13 //! can be described as an extension of NFAs with completion rules, prediction rules, and recursion.
15 //! Quick intro to how the parser works:
17 //! A 'position' is a dot in the middle of a matcher, usually represented as a
18 //! dot. For example `· a $( a )* a b` is a position, as is `a $( · a )* a b`.
20 //! The parser walks through the input a character at a time, maintaining a list
21 //! of threads consistent with the current position in the input string: `cur_items`.
23 //! As it processes them, it fills up `eof_items` with threads that would be valid if
24 //! the macro invocation is now over, `bb_items` with threads that are waiting on
25 //! a Rust non-terminal like `$e:expr`, and `next_items` with threads that are waiting
26 //! on a particular token. Most of the logic concerns moving the · through the
27 //! repetitions indicated by Kleene stars. The rules for moving the · without
28 //! consuming any input are called epsilon transitions. It only advances or calls
29 //! out to the real Rust parser when no `cur_items` threads remain.
34 //! Start parsing a a a a b against [· a $( a )* a b].
36 //! Remaining input: a a a a b
37 //! next: [· a $( a )* a b]
39 //! - - - Advance over an a. - - -
41 //! Remaining input: a a a b
42 //! cur: [a · $( a )* a b]
43 //! Descend/Skip (first item).
44 //! next: [a $( · a )* a b] [a $( a )* · a b].
46 //! - - - Advance over an a. - - -
48 //! Remaining input: a a b
49 //! cur: [a $( a · )* a b] [a $( a )* a · b]
50 //! Follow epsilon transition: Finish/Repeat (first item)
51 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
53 //! - - - Advance over an a. - - - (this looks exactly like the last step)
55 //! Remaining input: a b
56 //! cur: [a $( a · )* a b] [a $( a )* a · b]
57 //! Follow epsilon transition: Finish/Repeat (first item)
58 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
60 //! - - - Advance over an a. - - - (this looks exactly like the last step)
62 //! Remaining input: b
63 //! cur: [a $( a · )* a b] [a $( a )* a · b]
64 //! Follow epsilon transition: Finish/Repeat (first item)
65 //! next: [a $( a )* · a b] [a $( · a )* a b] [a $( a )* a · b]
67 //! - - - Advance over a b. - - -
69 //! Remaining input: ''
70 //! eof: [a $( a )* a b ·]
73 crate use NamedMatch
::*;
74 crate use ParseResult
::*;
75 use TokenTreeOrTokenTreeSlice
::*;
77 use crate::mbe
::{self, TokenTree}
;
79 use rustc_ast
::ast
::{Ident, Name}
;
80 use rustc_ast
::ptr
::P
;
81 use rustc_ast
::token
::{self, DocComment, Nonterminal, Token}
;
82 use rustc_ast_pretty
::pprust
;
83 use rustc_parse
::parser
::{FollowedByType, Parser, PathStyle}
;
84 use rustc_session
::parse
::ParseSess
;
85 use rustc_span
::symbol
::{kw, sym, Symbol}
;
87 use rustc_errors
::{FatalError, PResult}
;
89 use smallvec
::{smallvec, SmallVec}
;
91 use rustc_data_structures
::fx
::FxHashMap
;
92 use rustc_data_structures
::sync
::Lrc
;
94 use std
::collections
::hash_map
::Entry
::{Occupied, Vacant}
;
96 use std
::ops
::{Deref, DerefMut}
;
98 // To avoid costly uniqueness checks, we require that `MatchSeq` always has a nonempty body.
100 /// Either a sequence of token trees or a single one. This is used as the representation of the
101 /// sequence of tokens that make up a matcher.
103 enum TokenTreeOrTokenTreeSlice
<'tt
> {
105 TtSeq(&'tt
[TokenTree
]),
108 impl<'tt
> TokenTreeOrTokenTreeSlice
<'tt
> {
109 /// Returns the number of constituent top-level token trees of `self` (top-level in that it
110 /// will not recursively descend into subtrees).
111 fn len(&self) -> usize {
113 TtSeq(ref v
) => v
.len(),
114 Tt(ref tt
) => tt
.len(),
118 /// The `index`-th token tree of `self`.
119 fn get_tt(&self, index
: usize) -> TokenTree
{
121 TtSeq(ref v
) => v
[index
].clone(),
122 Tt(ref tt
) => tt
.get_tt(index
),
127 /// An unzipping of `TokenTree`s... see the `stack` field of `MatcherPos`.
129 /// This is used by `inner_parse_loop` to keep track of delimited submatchers that we have
132 struct MatcherTtFrame
<'tt
> {
133 /// The "parent" matcher that we are descending into.
134 elts
: TokenTreeOrTokenTreeSlice
<'tt
>,
135 /// The position of the "dot" in `elts` at the time we descended.
139 type NamedMatchVec
= SmallVec
<[NamedMatch
; 4]>;
141 /// Represents a single "position" (aka "matcher position", aka "item"), as
142 /// described in the module documentation.
146 /// - `'root` represents the lifetime of the stack slot that holds the root
147 /// `MatcherPos`. As described in `MatcherPosHandle`, the root `MatcherPos`
148 /// structure is stored on the stack, but subsequent instances are put into
150 /// - `'tt` represents the lifetime of the token trees that this matcher
151 /// position refers to.
153 /// It is important to distinguish these two lifetimes because we have a
154 /// `SmallVec<TokenTreeOrTokenTreeSlice<'tt>>` below, and the destructor of
155 /// that is considered to possibly access the data from its elements (it lacks
156 /// a `#[may_dangle]` attribute). As a result, the compiler needs to know that
157 /// all the elements in that `SmallVec` strictly outlive the root stack slot
158 /// lifetime. By separating `'tt` from `'root`, we can show that.
160 struct MatcherPos
<'root
, 'tt
> {
161 /// The token or sequence of tokens that make up the matcher
162 top_elts
: TokenTreeOrTokenTreeSlice
<'tt
>,
164 /// The position of the "dot" in this matcher
167 /// For each named metavar in the matcher, we keep track of token trees matched against the
168 /// metavar by the black box parser. In particular, there may be more than one match per
169 /// metavar if we are in a repetition (each repetition matches each of the variables).
170 /// Moreover, matchers and repetitions can be nested; the `matches` field is shared (hence the
171 /// `Rc`) among all "nested" matchers. `match_lo`, `match_cur`, and `match_hi` keep track of
172 /// the current position of the `self` matcher position in the shared `matches` list.
174 /// Also, note that while we are descending into a sequence, matchers are given their own
175 /// `matches` vector. Only once we reach the end of a full repetition of the sequence do we add
176 /// all bound matches from the submatcher into the shared top-level `matches` vector. If `sep`
177 /// and `up` are `Some`, then `matches` is _not_ the shared top-level list. Instead, if one
178 /// wants the shared `matches`, one should use `up.matches`.
179 matches
: Box
<[Lrc
<NamedMatchVec
>]>,
180 /// The position in `matches` corresponding to the first metavar in this matcher's sequence of
181 /// token trees. In other words, the first metavar in the first token of `top_elts` corresponds
182 /// to `matches[match_lo]`.
184 /// The position in `matches` corresponding to the metavar we are currently trying to match
185 /// against the source token stream. `match_lo <= match_cur <= match_hi`.
187 /// Similar to `match_lo` except `match_hi` is the position in `matches` of the _last_ metavar
191 // The following fields are used if we are matching a repetition. If we aren't, they should be
193 /// The KleeneOp of this sequence if we are in a repetition.
194 seq_op
: Option
<mbe
::KleeneOp
>,
196 /// The separator if we are in a repetition.
199 /// The "parent" matcher position if we are in a repetition. That is, the matcher position just
200 /// before we enter the sequence.
201 up
: Option
<MatcherPosHandle
<'root
, 'tt
>>,
203 /// Specifically used to "unzip" token trees. By "unzip", we mean to unwrap the delimiters from
204 /// a delimited token tree (e.g., something wrapped in `(` `)`) or to get the contents of a doc
207 /// When matching against matchers with nested delimited submatchers (e.g., `pat ( pat ( .. )
208 /// pat ) pat`), we need to keep track of the matchers we are descending into. This stack does
209 /// that where the bottom of the stack is the outermost matcher.
210 /// Also, throughout the comments, this "descent" is often referred to as "unzipping"...
211 stack
: SmallVec
<[MatcherTtFrame
<'tt
>; 1]>,
214 impl<'root
, 'tt
> MatcherPos
<'root
, 'tt
> {
215 /// Adds `m` as a named match for the `idx`-th metavar.
216 fn push_match(&mut self, idx
: usize, m
: NamedMatch
) {
217 let matches
= Lrc
::make_mut(&mut self.matches
[idx
]);
222 // Lots of MatcherPos instances are created at runtime. Allocating them on the
223 // heap is slow. Furthermore, using SmallVec<MatcherPos> to allocate them all
224 // on the stack is also slow, because MatcherPos is quite a large type and
225 // instances get moved around a lot between vectors, which requires lots of
226 // slow memcpy calls.
228 // Therefore, the initial MatcherPos is always allocated on the stack,
229 // subsequent ones (of which there aren't that many) are allocated on the heap,
230 // and this type is used to encapsulate both cases.
231 enum MatcherPosHandle
<'root
, 'tt
> {
232 Ref(&'root
mut MatcherPos
<'root
, 'tt
>),
233 Box(Box
<MatcherPos
<'root
, 'tt
>>),
236 impl<'root
, 'tt
> Clone
for MatcherPosHandle
<'root
, 'tt
> {
237 // This always produces a new Box.
238 fn clone(&self) -> Self {
239 MatcherPosHandle
::Box(match *self {
240 MatcherPosHandle
::Ref(ref r
) => Box
::new((**r
).clone()),
241 MatcherPosHandle
::Box(ref b
) => b
.clone(),
246 impl<'root
, 'tt
> Deref
for MatcherPosHandle
<'root
, 'tt
> {
247 type Target
= MatcherPos
<'root
, 'tt
>;
248 fn deref(&self) -> &Self::Target
{
250 MatcherPosHandle
::Ref(ref r
) => r
,
251 MatcherPosHandle
::Box(ref b
) => b
,
256 impl<'root
, 'tt
> DerefMut
for MatcherPosHandle
<'root
, 'tt
> {
257 fn deref_mut(&mut self) -> &mut MatcherPos
<'root
, 'tt
> {
259 MatcherPosHandle
::Ref(ref mut r
) => r
,
260 MatcherPosHandle
::Box(ref mut b
) => b
,
265 /// Represents the possible results of an attempted parse.
266 crate enum ParseResult
<T
> {
267 /// Parsed successfully.
269 /// Arm failed to match. If the second parameter is `token::Eof`, it indicates an unexpected
270 /// end of macro invocation. Otherwise, it indicates that no rules expected the given token.
271 Failure(Token
, &'
static str),
272 /// Fatal error (malformed macro?). Abort compilation.
273 Error(rustc_span
::Span
, String
),
276 /// A `ParseResult` where the `Success` variant contains a mapping of `Ident`s to `NamedMatch`es.
277 /// This represents the mapping of metavars to the token trees they bind to.
278 crate type NamedParseResult
= ParseResult
<FxHashMap
<Ident
, NamedMatch
>>;
280 /// Count how many metavars are named in the given matcher `ms`.
281 pub(super) fn count_names(ms
: &[TokenTree
]) -> usize {
282 ms
.iter().fold(0, |count
, elt
| {
285 TokenTree
::Sequence(_
, ref seq
) => seq
.num_captures
,
286 TokenTree
::Delimited(_
, ref delim
) => count_names(&delim
.tts
),
287 TokenTree
::MetaVar(..) => 0,
288 TokenTree
::MetaVarDecl(..) => 1,
289 TokenTree
::Token(..) => 0,
294 /// `len` `Vec`s (initially shared and empty) that will store matches of metavars.
295 fn create_matches(len
: usize) -> Box
<[Lrc
<NamedMatchVec
>]> {
299 let empty_matches
= Lrc
::new(SmallVec
::new());
300 vec
![empty_matches
; len
]
305 /// Generates the top-level matcher position in which the "dot" is before the first token of the
307 fn initial_matcher_pos
<'root
, 'tt
>(ms
: &'tt
[TokenTree
]) -> MatcherPos
<'root
, 'tt
> {
308 let match_idx_hi
= count_names(ms
);
309 let matches
= create_matches(match_idx_hi
);
311 // Start with the top level matcher given to us
312 top_elts
: TtSeq(ms
), // "elts" is an abbr. for "elements"
313 // The "dot" is before the first token of the matcher
316 // Initialize `matches` to a bunch of empty `Vec`s -- one for each metavar in `top_elts`.
317 // `match_lo` for `top_elts` is 0 and `match_hi` is `matches.len()`. `match_cur` is 0 since
318 // we haven't actually matched anything yet.
322 match_hi
: match_idx_hi
,
324 // Haven't descended into any delimiters, so empty stack
327 // Haven't descended into any sequences, so both of these are `None`.
334 /// `NamedMatch` is a pattern-match result for a single `token::MATCH_NONTERMINAL`:
335 /// so it is associated with a single ident in a parse, and all
336 /// `MatchedNonterminal`s in the `NamedMatch` have the same non-terminal type
337 /// (expr, item, etc). Each leaf in a single `NamedMatch` corresponds to a
338 /// single `token::MATCH_NONTERMINAL` in the `TokenTree` that produced it.
340 /// The in-memory structure of a particular `NamedMatch` represents the match
341 /// that occurred when a particular subset of a matcher was applied to a
342 /// particular token tree.
344 /// The width of each `MatchedSeq` in the `NamedMatch`, and the identity of
345 /// the `MatchedNonterminal`s, will depend on the token tree it was applied
346 /// to: each `MatchedSeq` corresponds to a single `TTSeq` in the originating
347 /// token tree. The depth of the `NamedMatch` structure will therefore depend
348 /// only on the nesting depth of `ast::TTSeq`s in the originating
349 /// token tree it was derived from.
350 #[derive(Debug, Clone)]
351 crate enum NamedMatch
{
352 MatchedSeq(Lrc
<NamedMatchVec
>),
353 MatchedNonterminal(Lrc
<Nonterminal
>),
356 /// Takes a sequence of token trees `ms` representing a matcher which successfully matched input
357 /// and an iterator of items that matched input and produces a `NamedParseResult`.
358 fn nameize
<I
: Iterator
<Item
= NamedMatch
>>(
362 ) -> NamedParseResult
{
363 // Recursively descend into each type of matcher (e.g., sequences, delimited, metavars) and make
364 // sure that each metavar has _exactly one_ binding. If a metavar does not have exactly one
365 // binding, then there is an error. If it does, then we insert the binding into the
366 // `NamedParseResult`.
367 fn n_rec
<I
: Iterator
<Item
= NamedMatch
>>(
371 ret_val
: &mut FxHashMap
<Ident
, NamedMatch
>,
372 ) -> Result
<(), (rustc_span
::Span
, String
)> {
374 TokenTree
::Sequence(_
, ref seq
) => {
375 for next_m
in &seq
.tts
{
376 n_rec(sess
, next_m
, res
.by_ref(), ret_val
)?
379 TokenTree
::Delimited(_
, ref delim
) => {
380 for next_m
in &delim
.tts
{
381 n_rec(sess
, next_m
, res
.by_ref(), ret_val
)?
;
384 TokenTree
::MetaVarDecl(span
, _
, id
) if id
.name
== kw
::Invalid
=> {
385 if sess
.missing_fragment_specifiers
.borrow_mut().remove(&span
) {
386 return Err((span
, "missing fragment specifier".to_string()));
389 TokenTree
::MetaVarDecl(sp
, bind_name
, _
) => match ret_val
.entry(bind_name
) {
391 spot
.insert(res
.next().unwrap());
393 Occupied(..) => return Err((sp
, format
!("duplicated bind name: {}", bind_name
))),
395 TokenTree
::MetaVar(..) | TokenTree
::Token(..) => (),
401 let mut ret_val
= FxHashMap
::default();
403 match n_rec(sess
, m
, res
.by_ref(), &mut ret_val
) {
405 Err((sp
, msg
)) => return Error(sp
, msg
),
412 /// Performs a token equality check, ignoring syntax context (that is, an unhygienic comparison)
413 fn token_name_eq(t1
: &Token
, t2
: &Token
) -> bool
{
414 if let (Some((ident1
, is_raw1
)), Some((ident2
, is_raw2
))) = (t1
.ident(), t2
.ident()) {
415 ident1
.name
== ident2
.name
&& is_raw1
== is_raw2
416 } else if let (Some(ident1
), Some(ident2
)) = (t1
.lifetime(), t2
.lifetime()) {
417 ident1
.name
== ident2
.name
423 /// Process the matcher positions of `cur_items` until it is empty. In the process, this will
424 /// produce more items in `next_items`, `eof_items`, and `bb_items`.
426 /// For more info about the how this happens, see the module-level doc comments and the inline
427 /// comments of this function.
431 /// - `sess`: the parsing session into which errors are emitted.
432 /// - `cur_items`: the set of current items to be processed. This should be empty by the end of a
433 /// successful execution of this function.
434 /// - `next_items`: the set of newly generated items. These are used to replenish `cur_items` in
435 /// the function `parse`.
436 /// - `eof_items`: the set of items that would be valid if this was the EOF.
437 /// - `bb_items`: the set of items that are waiting for the black-box parser.
438 /// - `token`: the current token of the parser.
439 /// - `span`: the `Span` in the source code corresponding to the token trees we are trying to match
440 /// against the matcher positions in `cur_items`.
444 /// A `ParseResult`. Note that matches are kept track of through the items generated.
445 fn inner_parse_loop
<'root
, 'tt
>(
447 cur_items
: &mut SmallVec
<[MatcherPosHandle
<'root
, 'tt
>; 1]>,
448 next_items
: &mut Vec
<MatcherPosHandle
<'root
, 'tt
>>,
449 eof_items
: &mut SmallVec
<[MatcherPosHandle
<'root
, 'tt
>; 1]>,
450 bb_items
: &mut SmallVec
<[MatcherPosHandle
<'root
, 'tt
>; 1]>,
452 ) -> ParseResult
<()> {
453 // Pop items from `cur_items` until it is empty.
454 while let Some(mut item
) = cur_items
.pop() {
455 // When unzipped trees end, remove them. This corresponds to backtracking out of a
456 // delimited submatcher into which we already descended. In backtracking out again, we need
457 // to advance the "dot" past the delimiters in the outer matcher.
458 while item
.idx
>= item
.top_elts
.len() {
459 match item
.stack
.pop() {
460 Some(MatcherTtFrame { elts, idx }
) => {
461 item
.top_elts
= elts
;
468 // Get the current position of the "dot" (`idx`) in `item` and the number of token trees in
469 // the matcher (`len`).
471 let len
= item
.top_elts
.len();
473 // If `idx >= len`, then we are at or past the end of the matcher of `item`.
475 // We are repeating iff there is a parent. If the matcher is inside of a repetition,
476 // then we could be at the end of a sequence or at the beginning of the next
478 if item
.up
.is_some() {
479 // At this point, regardless of whether there is a separator, we should add all
480 // matches from the complete repetition of the sequence to the shared, top-level
481 // `matches` list (actually, `up.matches`, which could itself not be the top-level,
482 // but anyway...). Moreover, we add another item to `cur_items` in which the "dot"
483 // is at the end of the `up` matcher. This ensures that the "dot" in the `up`
484 // matcher is also advanced sufficiently.
486 // NOTE: removing the condition `idx == len` allows trailing separators.
488 // Get the `up` matcher
489 let mut new_pos
= item
.up
.clone().unwrap();
491 // Add matches from this repetition to the `matches` of `up`
492 for idx
in item
.match_lo
..item
.match_hi
{
493 let sub
= item
.matches
[idx
].clone();
494 new_pos
.push_match(idx
, MatchedSeq(sub
));
497 // Move the "dot" past the repetition in `up`
498 new_pos
.match_cur
= item
.match_hi
;
500 cur_items
.push(new_pos
);
503 // Check if we need a separator.
504 if idx
== len
&& item
.sep
.is_some() {
505 // We have a separator, and it is the current token. We can advance past the
507 if item
.sep
.as_ref().map(|sep
| token_name_eq(token
, sep
)).unwrap_or(false) {
509 next_items
.push(item
);
512 // We don't need a separator. Move the "dot" back to the beginning of the matcher
513 // and try to match again UNLESS we are only allowed to have _one_ repetition.
514 else if item
.seq_op
!= Some(mbe
::KleeneOp
::ZeroOrOne
) {
515 item
.match_cur
= item
.match_lo
;
517 cur_items
.push(item
);
520 // If we are not in a repetition, then being at the end of a matcher means that we have
521 // reached the potential end of the input.
523 eof_items
.push(item
);
526 // We are in the middle of a matcher.
528 // Look at what token in the matcher we are trying to match the current token (`token`)
529 // against. Depending on that, we may generate new items.
530 match item
.top_elts
.get_tt(idx
) {
531 // Need to descend into a sequence
532 TokenTree
::Sequence(sp
, seq
) => {
533 // Examine the case where there are 0 matches of this sequence. We are
534 // implicitly disallowing OneOrMore from having 0 matches here. Thus, that will
535 // result in a "no rules expected token" error by virtue of this matcher not
537 if seq
.kleene
.op
== mbe
::KleeneOp
::ZeroOrMore
538 || seq
.kleene
.op
== mbe
::KleeneOp
::ZeroOrOne
540 let mut new_item
= item
.clone();
541 new_item
.match_cur
+= seq
.num_captures
;
543 for idx
in item
.match_cur
..item
.match_cur
+ seq
.num_captures
{
544 new_item
.push_match(idx
, MatchedSeq(Lrc
::new(smallvec
![])));
546 cur_items
.push(new_item
);
549 let matches
= create_matches(item
.matches
.len());
550 cur_items
.push(MatcherPosHandle
::Box(Box
::new(MatcherPos
{
552 sep
: seq
.separator
.clone(),
553 seq_op
: Some(seq
.kleene
.op
),
556 match_lo
: item
.match_cur
,
557 match_cur
: item
.match_cur
,
558 match_hi
: item
.match_cur
+ seq
.num_captures
,
560 top_elts
: Tt(TokenTree
::Sequence(sp
, seq
)),
564 // We need to match a metavar (but the identifier is invalid)... this is an error
565 TokenTree
::MetaVarDecl(span
, _
, id
) if id
.name
== kw
::Invalid
=> {
566 if sess
.missing_fragment_specifiers
.borrow_mut().remove(&span
) {
567 return Error(span
, "missing fragment specifier".to_string());
571 // We need to match a metavar with a valid ident... call out to the black-box
572 // parser by adding an item to `bb_items`.
573 TokenTree
::MetaVarDecl(_
, _
, id
) => {
574 // Built-in nonterminals never start with these tokens,
575 // so we can eliminate them from consideration.
576 if may_begin_with(token
, id
.name
) {
581 // We need to descend into a delimited submatcher or a doc comment. To do this, we
582 // push the current matcher onto a stack and push a new item containing the
583 // submatcher onto `cur_items`.
585 // At the beginning of the loop, if we reach the end of the delimited submatcher,
586 // we pop the stack to backtrack out of the descent.
587 seq @ TokenTree
::Delimited(..)
588 | seq @ TokenTree
::Token(Token { kind: DocComment(..), .. }
) => {
589 let lower_elts
= mem
::replace(&mut item
.top_elts
, Tt(seq
));
591 item
.stack
.push(MatcherTtFrame { elts: lower_elts, idx }
);
593 cur_items
.push(item
);
596 // We just matched a normal token. We can just advance the parser.
597 TokenTree
::Token(t
) if token_name_eq(&t
, token
) => {
599 next_items
.push(item
);
602 // There was another token that was not `token`... This means we can't add any
603 // rules. NOTE that this is not necessarily an error unless _all_ items in
604 // `cur_items` end up doing this. There may still be some other matchers that do
605 // end up working out.
606 TokenTree
::Token(..) | TokenTree
::MetaVar(..) => {}
611 // Yay a successful parse (so far)!
615 /// Use the given sequence of token trees (`ms`) as a matcher. Match the token
616 /// stream from the given `parser` against it and return the match.
617 pub(super) fn parse_tt(parser
: &mut Cow
<'_
, Parser
<'_
>>, ms
: &[TokenTree
]) -> NamedParseResult
{
618 // A queue of possible matcher positions. We initialize it with the matcher position in which
619 // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then
620 // processes all of these possible matcher positions and produces possible next positions into
621 // `next_items`. After some post-processing, the contents of `next_items` replenish `cur_items`
622 // and we start over again.
624 // This MatcherPos instance is allocated on the stack. All others -- and
625 // there are frequently *no* others! -- are allocated on the heap.
626 let mut initial
= initial_matcher_pos(ms
);
627 let mut cur_items
= smallvec
![MatcherPosHandle
::Ref(&mut initial
)];
628 let mut next_items
= Vec
::new();
631 // Matcher positions black-box parsed by parser.rs (`parser`)
632 let mut bb_items
= SmallVec
::new();
634 // Matcher positions that would be valid if the macro invocation was over now
635 let mut eof_items
= SmallVec
::new();
636 assert
!(next_items
.is_empty());
638 // Process `cur_items` until either we have finished the input or we need to get some
639 // parsing from the black-box parser done. The result is that `next_items` will contain a
640 // bunch of possible next matcher positions in `next_items`.
641 match inner_parse_loop(
650 Failure(token
, msg
) => return Failure(token
, msg
),
651 Error(sp
, msg
) => return Error(sp
, msg
),
654 // inner parse loop handled all cur_items, so it's empty
655 assert
!(cur_items
.is_empty());
657 // We need to do some post processing after the `inner_parser_loop`.
659 // Error messages here could be improved with links to original rules.
661 // If we reached the EOF, check that there is EXACTLY ONE possible matcher. Otherwise,
662 // either the parse is ambiguous (which should never happen) or there is a syntax error.
663 if parser
.token
== token
::Eof
{
664 if eof_items
.len() == 1 {
666 eof_items
[0].matches
.iter_mut().map(|dv
| Lrc
::make_mut(dv
).pop().unwrap());
667 return nameize(parser
.sess
, ms
, matches
);
668 } else if eof_items
.len() > 1 {
671 "ambiguity: multiple successful parses".to_string(),
677 if parser
.token
.span
.is_dummy() {
680 parser
.token
.span
.shrink_to_hi()
683 "missing tokens in macro arguments",
687 // Performance hack: eof_items may share matchers via Rc with other things that we want
688 // to modify. Dropping eof_items now may drop these refcounts to 1, preventing an
689 // unnecessary implicit clone later in Rc::make_mut.
692 // If there are no possible next positions AND we aren't waiting for the black-box parser,
693 // then there is a syntax error.
694 if bb_items
.is_empty() && next_items
.is_empty() {
695 return Failure(parser
.token
.clone(), "no rules expected this token in macro call");
697 // Another possibility is that we need to call out to parse some rust nonterminal
698 // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong.
699 else if (!bb_items
.is_empty() && !next_items
.is_empty()) || bb_items
.len() > 1 {
702 .map(|item
| match item
.top_elts
.get_tt(item
.idx
) {
703 TokenTree
::MetaVarDecl(_
, bind
, name
) => format
!("{} ('{}')", name
, bind
),
706 .collect
::<Vec
<String
>>()
712 "local ambiguity: multiple parsing options: {}",
713 match next_items
.len() {
714 0 => format
!("built-in NTs {}.", nts
),
715 1 => format
!("built-in NTs {} or 1 other option.", nts
),
716 n
=> format
!("built-in NTs {} or {} other options.", nts
, n
),
721 // Dump all possible `next_items` into `cur_items` for the next iteration.
722 else if !next_items
.is_empty() {
723 // Now process the next token
724 cur_items
.extend(next_items
.drain(..));
725 parser
.to_mut().bump();
727 // Finally, we have the case where we need to call the black-box parser to get some
730 assert_eq
!(bb_items
.len(), 1);
732 let mut item
= bb_items
.pop().unwrap();
733 if let TokenTree
::MetaVarDecl(span
, _
, ident
) = item
.top_elts
.get_tt(item
.idx
) {
734 let match_cur
= item
.match_cur
;
737 MatchedNonterminal(Lrc
::new(parse_nt(parser
.to_mut(), span
, ident
.name
))),
744 cur_items
.push(item
);
747 assert
!(!cur_items
.is_empty());
751 /// The token is an identifier, but not `_`.
752 /// We prohibit passing `_` to macros expecting `ident` for now.
753 fn get_macro_ident(token
: &Token
) -> Option
<(Ident
, bool
)> {
754 token
.ident().filter(|(ident
, _
)| ident
.name
!= kw
::Underscore
)
757 /// Checks whether a non-terminal may begin with a particular token.
759 /// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with that
760 /// token. Be conservative (return true) if not sure.
761 fn may_begin_with(token
: &Token
, name
: Name
) -> bool
{
762 /// Checks whether the non-terminal may contain a single (non-keyword) identifier.
763 fn may_be_ident(nt
: &token
::Nonterminal
) -> bool
{
765 token
::NtItem(_
) | token
::NtBlock(_
) | token
::NtVis(_
) | token
::NtLifetime(_
) => false,
772 token
.can_begin_expr()
773 // This exception is here for backwards compatibility.
774 && !token
.is_keyword(kw
::Let
)
776 sym
::ty
=> token
.can_begin_type(),
777 sym
::ident
=> get_macro_ident(token
).is_some(),
778 sym
::literal
=> token
.can_begin_literal_maybe_minus(),
779 sym
::vis
=> match token
.kind
{
780 // The follow-set of :vis + "priv" keyword + interpolated
781 token
::Comma
| token
::Ident(..) | token
::Interpolated(_
) => true,
782 _
=> token
.can_begin_type(),
784 sym
::block
=> match token
.kind
{
785 token
::OpenDelim(token
::Brace
) => true,
786 token
::Interpolated(ref nt
) => match **nt
{
793 | token
::NtVis(_
) => false, // none of these may start with '{'.
798 sym
::path
| sym
::meta
=> match token
.kind
{
799 token
::ModSep
| token
::Ident(..) => true,
800 token
::Interpolated(ref nt
) => match **nt
{
801 token
::NtPath(_
) | token
::NtMeta(_
) => true,
802 _
=> may_be_ident(&nt
),
806 sym
::pat
=> match token
.kind
{
807 token
::Ident(..) | // box, ref, mut, and other identifiers (can stricten)
808 token
::OpenDelim(token
::Paren
) | // tuple pattern
809 token
::OpenDelim(token
::Bracket
) | // slice pattern
810 token
::BinOp(token
::And
) | // reference
811 token
::BinOp(token
::Minus
) | // negative literal
812 token
::AndAnd
| // double reference
813 token
::Literal(..) | // literal
814 token
::DotDot
| // range pattern (future compat)
815 token
::DotDotDot
| // range pattern (future compat)
816 token
::ModSep
| // path
817 token
::Lt
| // path (UFCS constant)
818 token
::BinOp(token
::Shl
) => true, // path (double UFCS)
819 token
::Interpolated(ref nt
) => may_be_ident(nt
),
822 sym
::lifetime
=> match token
.kind
{
823 token
::Lifetime(_
) => true,
824 token
::Interpolated(ref nt
) => match **nt
{
825 token
::NtLifetime(_
) | token
::NtTT(_
) => true,
830 _
=> match token
.kind
{
831 token
::CloseDelim(_
) => false,
837 /// A call to the "black-box" parser to parse some Rust non-terminal.
841 /// - `p`: the "black-box" parser to use
842 /// - `sp`: the `Span` we want to parse
843 /// - `name`: the name of the metavar _matcher_ we want to match (e.g., `tt`, `ident`, `block`,
848 /// The parsed non-terminal.
849 fn parse_nt(p
: &mut Parser
<'_
>, sp
: Span
, name
: Symbol
) -> Nonterminal
{
850 // FIXME(Centril): Consider moving this to `parser.rs` to make
851 // the visibilities of the methods used below `pub(super)` at most.
854 return token
::NtTT(p
.parse_token_tree());
856 match parse_nt_inner(p
, sp
, name
) {
865 fn parse_nt_inner
<'a
>(p
: &mut Parser
<'a
>, sp
: Span
, name
: Symbol
) -> PResult
<'a
, Nonterminal
> {
867 sym
::item
=> match p
.parse_item()?
{
868 Some(i
) => token
::NtItem(i
),
869 None
=> return Err(p
.struct_span_err(p
.token
.span
, "expected an item keyword")),
871 sym
::block
=> token
::NtBlock(p
.parse_block()?
),
872 sym
::stmt
=> match p
.parse_stmt()?
{
873 Some(s
) => token
::NtStmt(s
),
874 None
=> return Err(p
.struct_span_err(p
.token
.span
, "expected a statement")),
876 sym
::pat
=> token
::NtPat(p
.parse_pat(None
)?
),
877 sym
::expr
=> token
::NtExpr(p
.parse_expr()?
),
878 sym
::literal
=> token
::NtLiteral(p
.parse_literal_maybe_minus()?
),
879 sym
::ty
=> token
::NtTy(p
.parse_ty()?
),
880 // this could be handled like a token, since it is one
882 if let Some((ident
, is_raw
)) = get_macro_ident(&p
.token
) {
884 token
::NtIdent(ident
, is_raw
)
886 let token_str
= pprust
::token_to_string(&p
.token
);
887 let msg
= &format
!("expected ident, found {}", &token_str
);
888 return Err(p
.struct_span_err(p
.token
.span
, msg
));
891 sym
::path
=> token
::NtPath(p
.parse_path(PathStyle
::Type
)?
),
892 sym
::meta
=> token
::NtMeta(P(p
.parse_attr_item()?
)),
893 sym
::vis
=> token
::NtVis(p
.parse_visibility(FollowedByType
::Yes
)?
),
895 if p
.check_lifetime() {
896 token
::NtLifetime(p
.expect_lifetime().ident
)
898 let token_str
= pprust
::token_to_string(&p
.token
);
899 let msg
= &format
!("expected a lifetime, found `{}`", &token_str
);
900 return Err(p
.struct_span_err(p
.token
.span
, msg
));
903 // this is not supposed to happen, since it has been checked
904 // when compiling the macro.
905 _
=> p
.span_bug(sp
, "invalid fragment specifier"),