]>
Commit | Line | Data |
---|---|---|
e74abb32 XL |
1 | use crate::mbe::macro_parser; |
2 | use crate::mbe::{TokenTree, KleeneOp, KleeneToken, SequenceRepetition, Delimited}; | |
9fa01778 | 3 | |
e74abb32 | 4 | use syntax::ast; |
e74abb32 XL |
5 | use syntax::print::pprust; |
6 | use syntax::sess::ParseSess; | |
7 | use syntax::symbol::kw; | |
60c5eb7d | 8 | use syntax::token::{self, Token}; |
e74abb32 | 9 | use syntax::tokenstream; |
8bb4bdeb | 10 | |
e74abb32 | 11 | use syntax_pos::Span; |
416331ca | 12 | |
e74abb32 | 13 | use rustc_data_structures::sync::Lrc; |
8bb4bdeb | 14 | |
2c00a5a8 XL |
15 | /// Takes a `tokenstream::TokenStream` and returns a `Vec<self::TokenTree>`. Specifically, this |
16 | /// takes a generic `TokenStream`, such as is used in the rest of the compiler, and returns a | |
17 | /// collection of `TokenTree` for use in parsing a macro. | |
18 | /// | |
19 | /// # Parameters | |
20 | /// | |
21 | /// - `input`: a token stream to read from, the contents of which we are parsing. | |
22 | /// - `expect_matchers`: `parse` can be used to parse either the "patterns" or the "body" of a | |
23 | /// macro. Both take roughly the same form _except_ that in a pattern, metavars are declared with | |
24 | /// their "matcher" type. For example `$var:expr` or `$id:ident`. In this example, `expr` and | |
25 | /// `ident` are "matchers". They are not present in the body of a macro rule -- just in the | |
26 | /// pattern, so we pass a parameter to indicate whether to expect them or not. | |
27 | /// - `sess`: the parsing session. Any errors will be emitted to this session. | |
28 | /// - `features`, `attrs`: language feature flags and attributes so that we know whether to use | |
29 | /// unstable features or not. | |
8faf50e0 XL |
30 | /// - `edition`: which edition are we in. |
31 | /// - `macro_node_id`: the NodeId of the macro we are parsing. | |
2c00a5a8 XL |
32 | /// |
33 | /// # Returns | |
34 | /// | |
35 | /// A collection of `self::TokenTree`. There may also be some errors emitted to `sess`. | |
e74abb32 | 36 | pub(super) fn parse( |
2c00a5a8 XL |
37 | input: tokenstream::TokenStream, |
38 | expect_matchers: bool, | |
39 | sess: &ParseSess, | |
2c00a5a8 XL |
40 | ) -> Vec<TokenTree> { |
41 | // Will contain the final collection of `self::TokenTree` | |
8bb4bdeb | 42 | let mut result = Vec::new(); |
2c00a5a8 XL |
43 | |
44 | // For each token tree in `input`, parse the token into a `self::TokenTree`, consuming | |
45 | // additional trees if need be. | |
e74abb32 | 46 | let mut trees = input.trees(); |
8bb4bdeb | 47 | while let Some(tree) = trees.next() { |
2c00a5a8 | 48 | // Given the parsed tree, if there is a metavar and we are expecting matchers, actually |
0731742a | 49 | // parse out the matcher (i.e., in `$id:ident` this would parse the `:` and `ident`). |
8faf50e0 XL |
50 | let tree = parse_tree( |
51 | tree, | |
52 | &mut trees, | |
53 | expect_matchers, | |
54 | sess, | |
8faf50e0 | 55 | ); |
8bb4bdeb | 56 | match tree { |
041b39d2 | 57 | TokenTree::MetaVar(start_sp, ident) if expect_matchers => { |
8bb4bdeb | 58 | let span = match trees.next() { |
dc9dc135 XL |
59 | Some(tokenstream::TokenTree::Token(Token { kind: token::Colon, span })) => { |
60 | match trees.next() { | |
61 | Some(tokenstream::TokenTree::Token(token)) => match token.ident() { | |
62 | Some((kind, _)) => { | |
63 | let span = token.span.with_lo(start_sp.lo()); | |
64 | result.push(TokenTree::MetaVarDecl(span, ident, kind)); | |
65 | continue; | |
66 | } | |
67 | _ => token.span, | |
68 | }, | |
69 | tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span), | |
70 | } | |
71 | } | |
72 | tree => tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(start_sp), | |
8bb4bdeb XL |
73 | }; |
74 | sess.missing_fragment_specifiers.borrow_mut().insert(span); | |
dc9dc135 | 75 | result.push(TokenTree::MetaVarDecl(span, ident, ast::Ident::invalid())); |
8bb4bdeb | 76 | } |
2c00a5a8 XL |
77 | |
78 | // Not a metavar or no matchers allowed, so just return the tree | |
8bb4bdeb XL |
79 | _ => result.push(tree), |
80 | } | |
81 | } | |
82 | result | |
83 | } | |
84 | ||
2c00a5a8 XL |
85 | /// Takes a `tokenstream::TokenTree` and returns a `self::TokenTree`. Specifically, this takes a |
86 | /// generic `TokenTree`, such as is used in the rest of the compiler, and returns a `TokenTree` | |
87 | /// for use in parsing a macro. | |
88 | /// | |
89 | /// Converting the given tree may involve reading more tokens. | |
90 | /// | |
91 | /// # Parameters | |
92 | /// | |
93 | /// - `tree`: the tree we wish to convert. | |
94 | /// - `trees`: an iterator over trees. We may need to read more tokens from it in order to finish | |
95 | /// converting `tree` | |
96 | /// - `expect_matchers`: same as for `parse` (see above). | |
97 | /// - `sess`: the parsing session. Any errors will be emitted to this session. | |
98 | /// - `features`, `attrs`: language feature flags and attributes so that we know whether to use | |
99 | /// unstable features or not. | |
dc9dc135 | 100 | fn parse_tree( |
2c00a5a8 | 101 | tree: tokenstream::TokenTree, |
e74abb32 | 102 | trees: &mut impl Iterator<Item = tokenstream::TokenTree>, |
2c00a5a8 XL |
103 | expect_matchers: bool, |
104 | sess: &ParseSess, | |
dc9dc135 | 105 | ) -> TokenTree { |
2c00a5a8 | 106 | // Depending on what `tree` is, we could be parsing different parts of a macro |
8bb4bdeb | 107 | match tree { |
2c00a5a8 | 108 | // `tree` is a `$` token. Look at the next token in `trees` |
dc9dc135 | 109 | tokenstream::TokenTree::Token(Token { kind: token::Dollar, span }) => match trees.next() { |
2c00a5a8 XL |
110 | // `tree` is followed by a delimited set of token trees. This indicates the beginning |
111 | // of a repetition sequence in the macro (e.g. `$(pat)*`). | |
0731742a | 112 | Some(tokenstream::TokenTree::Delimited(span, delim, tts)) => { |
2c00a5a8 | 113 | // Must have `(` not `{` or `[` |
0731742a | 114 | if delim != token::Paren { |
dc9dc135 | 115 | let tok = pprust::token_kind_to_string(&token::OpenDelim(delim)); |
8bb4bdeb | 116 | let msg = format!("expected `(`, found `{}`", tok); |
b7449926 | 117 | sess.span_diagnostic.span_err(span.entire(), &msg); |
8bb4bdeb | 118 | } |
2c00a5a8 | 119 | // Parse the contents of the sequence itself |
8faf50e0 | 120 | let sequence = parse( |
0731742a | 121 | tts.into(), |
8faf50e0 XL |
122 | expect_matchers, |
123 | sess, | |
8faf50e0 | 124 | ); |
2c00a5a8 | 125 | // Get the Kleene operator and optional separator |
416331ca | 126 | let (separator, kleene) = parse_sep_and_kleene_op(trees, span.entire(), sess); |
0731742a | 127 | // Count the number of captured "names" (i.e., named metavars) |
8bb4bdeb | 128 | let name_captures = macro_parser::count_names(&sequence); |
2c00a5a8 XL |
129 | TokenTree::Sequence( |
130 | span, | |
0531ce1d | 131 | Lrc::new(SequenceRepetition { |
2c00a5a8 XL |
132 | tts: sequence, |
133 | separator, | |
416331ca | 134 | kleene, |
2c00a5a8 XL |
135 | num_captures: name_captures, |
136 | }), | |
137 | ) | |
8bb4bdeb | 138 | } |
2c00a5a8 XL |
139 | |
140 | // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special | |
b7449926 | 141 | // metavariable that names the crate of the invocation. |
dc9dc135 | 142 | Some(tokenstream::TokenTree::Token(token)) if token.is_ident() => { |
83c7162d | 143 | let (ident, is_raw) = token.ident().unwrap(); |
dc9dc135 XL |
144 | let span = ident.span.with_lo(span.lo()); |
145 | if ident.name == kw::Crate && !is_raw { | |
146 | TokenTree::token(token::Ident(kw::DollarCrate, is_raw), span) | |
8bb4bdeb | 147 | } else { |
041b39d2 | 148 | TokenTree::MetaVar(span, ident) |
8bb4bdeb XL |
149 | } |
150 | } | |
2c00a5a8 XL |
151 | |
152 | // `tree` is followed by a random token. This is an error. | |
dc9dc135 XL |
153 | Some(tokenstream::TokenTree::Token(token)) => { |
154 | let msg = | |
155 | format!("expected identifier, found `{}`", pprust::token_to_string(&token),); | |
156 | sess.span_diagnostic.span_err(token.span, &msg); | |
157 | TokenTree::MetaVar(token.span, ast::Ident::invalid()) | |
8bb4bdeb | 158 | } |
2c00a5a8 XL |
159 | |
160 | // There are no more tokens. Just return the `$` we already have. | |
dc9dc135 | 161 | None => TokenTree::token(token::Dollar, span), |
8bb4bdeb | 162 | }, |
2c00a5a8 XL |
163 | |
164 | // `tree` is an arbitrary token. Keep it. | |
dc9dc135 | 165 | tokenstream::TokenTree::Token(token) => TokenTree::Token(token), |
2c00a5a8 | 166 | |
0731742a | 167 | // `tree` is the beginning of a delimited set of tokens (e.g., `(` or `{`). We need to |
2c00a5a8 | 168 | // descend into the delimited set and further parse it. |
0731742a | 169 | tokenstream::TokenTree::Delimited(span, delim, tts) => TokenTree::Delimited( |
2c00a5a8 | 170 | span, |
0531ce1d | 171 | Lrc::new(Delimited { |
dc9dc135 | 172 | delim, |
8faf50e0 | 173 | tts: parse( |
0731742a | 174 | tts.into(), |
8faf50e0 XL |
175 | expect_matchers, |
176 | sess, | |
8faf50e0 | 177 | ), |
2c00a5a8 XL |
178 | }), |
179 | ), | |
8bb4bdeb XL |
180 | } |
181 | } | |
182 | ||
2c00a5a8 XL |
183 | /// Takes a token and returns `Some(KleeneOp)` if the token is `+` `*` or `?`. Otherwise, return |
184 | /// `None`. | |
dc9dc135 XL |
185 | fn kleene_op(token: &Token) -> Option<KleeneOp> { |
186 | match token.kind { | |
2c00a5a8 XL |
187 | token::BinOp(token::Star) => Some(KleeneOp::ZeroOrMore), |
188 | token::BinOp(token::Plus) => Some(KleeneOp::OneOrMore), | |
189 | token::Question => Some(KleeneOp::ZeroOrOne), | |
190 | _ => None, | |
8bb4bdeb | 191 | } |
2c00a5a8 | 192 | } |
8bb4bdeb | 193 | |
2c00a5a8 XL |
194 | /// Parse the next token tree of the input looking for a KleeneOp. Returns |
195 | /// | |
8faf50e0 | 196 | /// - Ok(Ok((op, span))) if the next token tree is a KleeneOp |
2c00a5a8 XL |
197 | /// - Ok(Err(tok, span)) if the next token tree is a token but not a KleeneOp |
198 | /// - Err(span) if the next token tree is not a token | |
dc9dc135 XL |
199 | fn parse_kleene_op( |
200 | input: &mut impl Iterator<Item = tokenstream::TokenTree>, | |
2c00a5a8 | 201 | span: Span, |
dc9dc135 | 202 | ) -> Result<Result<(KleeneOp, Span), Token>, Span> { |
2c00a5a8 | 203 | match input.next() { |
dc9dc135 XL |
204 | Some(tokenstream::TokenTree::Token(token)) => match kleene_op(&token) { |
205 | Some(op) => Ok(Ok((op, token.span))), | |
206 | None => Ok(Err(token)), | |
2c00a5a8 | 207 | }, |
dc9dc135 | 208 | tree => Err(tree.as_ref().map(tokenstream::TokenTree::span).unwrap_or(span)), |
2c00a5a8 XL |
209 | } |
210 | } | |
211 | ||
212 | /// Attempt to parse a single Kleene star, possibly with a separator. | |
213 | /// | |
214 | /// For example, in a pattern such as `$(a),*`, `a` is the pattern to be repeated, `,` is the | |
215 | /// separator, and `*` is the Kleene operator. This function is specifically concerned with parsing | |
216 | /// the last two tokens of such a pattern: namely, the optional separator and the Kleene operator | |
217 | /// itself. Note that here we are parsing the _macro_ itself, rather than trying to match some | |
218 | /// stream of tokens in an invocation of a macro. | |
219 | /// | |
220 | /// This function will take some input iterator `input` corresponding to `span` and a parsing | |
221 | /// session `sess`. If the next one (or possibly two) tokens in `input` correspond to a Kleene | |
222 | /// operator and separator, then a tuple with `(separator, KleeneOp)` is returned. Otherwise, an | |
223 | /// error with the appropriate span is emitted to `sess` and a dummy value is returned. | |
dc9dc135 | 224 | fn parse_sep_and_kleene_op( |
e74abb32 | 225 | input: &mut impl Iterator<Item = tokenstream::TokenTree>, |
8faf50e0 XL |
226 | span: Span, |
227 | sess: &ParseSess, | |
416331ca | 228 | ) -> (Option<Token>, KleeneToken) { |
8faf50e0 XL |
229 | // We basically look at two token trees here, denoted as #1 and #2 below |
230 | let span = match parse_kleene_op(input, span) { | |
dc9dc135 | 231 | // #1 is a `?`, `+`, or `*` KleeneOp |
416331ca | 232 | Ok(Ok((op, span))) => return (None, KleeneToken::new(op, span)), |
8faf50e0 | 233 | |
2c00a5a8 | 234 | // #1 is a separator followed by #2, a KleeneOp |
dc9dc135 | 235 | Ok(Err(token)) => match parse_kleene_op(input, token.span) { |
8faf50e0 | 236 | // #2 is the `?` Kleene op, which does not take a separator (error) |
416331ca | 237 | Ok(Ok((KleeneOp::ZeroOrOne, span))) => { |
8faf50e0 | 238 | // Error! |
a1dfa0c6 | 239 | sess.span_diagnostic.span_err( |
dc9dc135 | 240 | token.span, |
a1dfa0c6 XL |
241 | "the `?` macro repetition operator does not take a separator", |
242 | ); | |
8faf50e0 XL |
243 | |
244 | // Return a dummy | |
416331ca | 245 | return (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)); |
2c00a5a8 | 246 | } |
8faf50e0 XL |
247 | |
248 | // #2 is a KleeneOp :D | |
416331ca | 249 | Ok(Ok((op, span))) => return (Some(token), KleeneToken::new(op, span)), |
2c00a5a8 | 250 | |
dc9dc135 XL |
251 | // #2 is a random token or not a token at all :( |
252 | Ok(Err(Token { span, .. })) | Err(span) => span, | |
8bb4bdeb | 253 | }, |
2c00a5a8 XL |
254 | |
255 | // #1 is not a token | |
256 | Err(span) => span, | |
8bb4bdeb XL |
257 | }; |
258 | ||
8faf50e0 | 259 | // If we ever get to this point, we have experienced an "unexpected token" error |
dc9dc135 | 260 | sess.span_diagnostic.span_err(span, "expected one of: `*`, `+`, or `?`"); |
8faf50e0 XL |
261 | |
262 | // Return a dummy | |
416331ca | 263 | (None, KleeneToken::new(KleeneOp::ZeroOrMore, span)) |
8bb4bdeb | 264 | } |