]>
Commit | Line | Data |
---|---|---|
9fa01778 XL |
1 | //! The main parser interface. |
2 | ||
cdc7bbd5 | 3 | #![feature(array_windows)] |
60c5eb7d | 4 | #![feature(crate_visibility_modifier)] |
ba9703b0 | 5 | #![feature(bindings_after_at)] |
5869c6ff XL |
6 | #![feature(box_syntax)] |
7 | #![feature(box_patterns)] | |
6a06907d | 8 | #![recursion_limit = "256"] |
60c5eb7d | 9 | |
3dfed10e | 10 | use rustc_ast as ast; |
cdc7bbd5 XL |
11 | use rustc_ast::token::{self, Nonterminal, Token, TokenKind}; |
12 | use rustc_ast::tokenstream::{self, AttributesData, CanSynthesizeMissingTokens, LazyTokenStream}; | |
13 | use rustc_ast::tokenstream::{AttrAnnotatedTokenStream, AttrAnnotatedTokenTree}; | |
14 | use rustc_ast::tokenstream::{Spacing, TokenStream}; | |
6a06907d | 15 | use rustc_ast::AstLike; |
cdc7bbd5 | 16 | use rustc_ast::Attribute; |
74b04a01 | 17 | use rustc_ast_pretty::pprust; |
e74abb32 | 18 | use rustc_data_structures::sync::Lrc; |
dfeec247 | 19 | use rustc_errors::{Diagnostic, FatalError, Level, PResult}; |
74b04a01 | 20 | use rustc_session::parse::ParseSess; |
5869c6ff | 21 | use rustc_span::{FileName, SourceFile, Span}; |
9346a6ac | 22 | |
ba9703b0 | 23 | use std::path::Path; |
1a4d82fc | 24 | use std::str; |
223e47cc | 25 | |
29967ef6 | 26 | pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments"); |
9346a6ac | 27 | |
1a4d82fc | 28 | #[macro_use] |
223e47cc | 29 | pub mod parser; |
dfeec247 | 30 | use parser::{emit_unclosed_delims, make_unclosed_delims_error, Parser}; |
1a4d82fc | 31 | pub mod lexer; |
60c5eb7d | 32 | pub mod validate_attr; |
476ff2be | 33 | |
e1599b0c | 34 | // A bunch of utility functions of the form `parse_<thing>_from_<source>` |
223e47cc LB |
35 | // where <thing> includes crate, expr, item, stmt, tts, and one that |
36 | // uses a HOF to parse anything, and <source> includes file and | |
e1599b0c | 37 | // `source_str`. |
223e47cc | 38 | |
e74abb32 XL |
39 | /// A variant of 'panictry!' that works on a Vec<Diagnostic> instead of a single DiagnosticBuilder. |
40 | macro_rules! panictry_buffer { | |
dfeec247 | 41 | ($handler:expr, $e:expr) => {{ |
60c5eb7d | 42 | use rustc_errors::FatalError; |
dfeec247 | 43 | use std::result::Result::{Err, Ok}; |
e74abb32 XL |
44 | match $e { |
45 | Ok(e) => e, | |
46 | Err(errs) => { | |
47 | for e in errs { | |
48 | $handler.emit_diagnostic(&e); | |
49 | } | |
50 | FatalError.raise() | |
51 | } | |
52 | } | |
dfeec247 | 53 | }}; |
e74abb32 XL |
54 | } |
55 | ||
c30ab7b3 | 56 | pub fn parse_crate_from_file<'a>(input: &Path, sess: &'a ParseSess) -> PResult<'a, ast::Crate> { |
ba9703b0 | 57 | let mut parser = new_parser_from_file(sess, input, None); |
54a0048b | 58 | parser.parse_crate_mod() |
223e47cc LB |
59 | } |
60 | ||
dfeec247 XL |
61 | pub fn parse_crate_attrs_from_file<'a>( |
62 | input: &Path, | |
63 | sess: &'a ParseSess, | |
64 | ) -> PResult<'a, Vec<ast::Attribute>> { | |
ba9703b0 | 65 | let mut parser = new_parser_from_file(sess, input, None); |
54a0048b | 66 | parser.parse_inner_attributes() |
1a4d82fc JJ |
67 | } |
68 | ||
dfeec247 XL |
69 | pub fn parse_crate_from_source_str( |
70 | name: FileName, | |
71 | source: String, | |
72 | sess: &ParseSess, | |
73 | ) -> PResult<'_, ast::Crate> { | |
c30ab7b3 | 74 | new_parser_from_source_str(sess, name, source).parse_crate_mod() |
223e47cc LB |
75 | } |
76 | ||
dfeec247 XL |
77 | pub fn parse_crate_attrs_from_source_str( |
78 | name: FileName, | |
79 | source: String, | |
80 | sess: &ParseSess, | |
81 | ) -> PResult<'_, Vec<ast::Attribute>> { | |
c30ab7b3 | 82 | new_parser_from_source_str(sess, name, source).parse_inner_attributes() |
1a4d82fc JJ |
83 | } |
84 | ||
9fa01778 XL |
85 | pub fn parse_stream_from_source_str( |
86 | name: FileName, | |
87 | source: String, | |
88 | sess: &ParseSess, | |
89 | override_span: Option<Span>, | |
90 | ) -> TokenStream { | |
dfeec247 XL |
91 | let (stream, mut errors) = |
92 | source_file_to_stream(sess, sess.source_map().new_source_file(name, source), override_span); | |
e74abb32 | 93 | emit_unclosed_delims(&mut errors, &sess); |
9fa01778 | 94 | stream |
223e47cc LB |
95 | } |
96 | ||
9fa01778 XL |
97 | /// Creates a new parser from a source string. |
98 | pub fn new_parser_from_source_str(sess: &ParseSess, name: FileName, source: String) -> Parser<'_> { | |
a1dfa0c6 XL |
99 | panictry_buffer!(&sess.span_diagnostic, maybe_new_parser_from_source_str(sess, name, source)) |
100 | } | |
101 | ||
9fa01778 | 102 | /// Creates a new parser from a source string. Returns any buffered errors from lexing the initial |
a1dfa0c6 | 103 | /// token stream. |
dfeec247 XL |
104 | pub fn maybe_new_parser_from_source_str( |
105 | sess: &ParseSess, | |
106 | name: FileName, | |
107 | source: String, | |
108 | ) -> Result<Parser<'_>, Vec<Diagnostic>> { | |
ba9703b0 | 109 | maybe_source_file_to_parser(sess, sess.source_map().new_source_file(name, source)) |
223e47cc LB |
110 | } |
111 | ||
e1599b0c | 112 | /// Creates a new parser, handling errors as appropriate if the file doesn't exist. |
1b1a35ee | 113 | /// If a span is given, that is used on an error as the source of the problem. |
ba9703b0 XL |
114 | pub fn new_parser_from_file<'a>(sess: &'a ParseSess, path: &Path, sp: Option<Span>) -> Parser<'a> { |
115 | source_file_to_parser(sess, file_to_source_file(sess, path, sp)) | |
223e47cc LB |
116 | } |
117 | ||
e1599b0c | 118 | /// Given a `source_file` and config, returns a parser. |
9fa01778 | 119 | fn source_file_to_parser(sess: &ParseSess, source_file: Lrc<SourceFile>) -> Parser<'_> { |
dfeec247 | 120 | panictry_buffer!(&sess.span_diagnostic, maybe_source_file_to_parser(sess, source_file)) |
a1dfa0c6 XL |
121 | } |
122 | ||
e1599b0c | 123 | /// Given a `source_file` and config, return a parser. Returns any buffered errors from lexing the |
a1dfa0c6 | 124 | /// initial token stream. |
9fa01778 XL |
125 | fn maybe_source_file_to_parser( |
126 | sess: &ParseSess, | |
127 | source_file: Lrc<SourceFile>, | |
128 | ) -> Result<Parser<'_>, Vec<Diagnostic>> { | |
b7449926 | 129 | let end_pos = source_file.end_pos; |
9fa01778 | 130 | let (stream, unclosed_delims) = maybe_file_to_stream(sess, source_file, None)?; |
dc9dc135 | 131 | let mut parser = stream_to_parser(sess, stream, None); |
9fa01778 | 132 | parser.unclosed_delims = unclosed_delims; |
74b04a01 | 133 | if parser.token == token::Eof { |
dc9dc135 | 134 | parser.token.span = Span::new(end_pos, end_pos, parser.token.span.ctxt()); |
c1a9b12d SL |
135 | } |
136 | ||
a1dfa0c6 | 137 | Ok(parser) |
970d7e83 LB |
138 | } |
139 | ||
e1599b0c | 140 | // Base abstractions |
970d7e83 | 141 | |
0731742a XL |
142 | /// Given a session and a path and an optional span (for error reporting), |
143 | /// add the path to the session's source_map and return the new source_file or | |
144 | /// error when a file can't be read. | |
dfeec247 XL |
145 | fn try_file_to_source_file( |
146 | sess: &ParseSess, | |
147 | path: &Path, | |
148 | spanopt: Option<Span>, | |
149 | ) -> Result<Lrc<SourceFile>, Diagnostic> { | |
150 | sess.source_map().load_file(path).map_err(|e| { | |
0731742a XL |
151 | let msg = format!("couldn't read {}: {}", path.display(), e); |
152 | let mut diag = Diagnostic::new(Level::Fatal, &msg); | |
153 | if let Some(sp) = spanopt { | |
154 | diag.set_span(sp); | |
155 | } | |
156 | diag | |
157 | }) | |
158 | } | |
159 | ||
970d7e83 | 160 | /// Given a session and a path and an optional span (for error reporting), |
e1599b0c | 161 | /// adds the path to the session's `source_map` and returns the new `source_file`. |
dfeec247 | 162 | fn file_to_source_file(sess: &ParseSess, path: &Path, spanopt: Option<Span>) -> Lrc<SourceFile> { |
0731742a | 163 | match try_file_to_source_file(sess, path, spanopt) { |
b7449926 | 164 | Ok(source_file) => source_file, |
0731742a | 165 | Err(d) => { |
e1599b0c | 166 | sess.span_diagnostic.emit_diagnostic(&d); |
0731742a | 167 | FatalError.raise(); |
223e47cc LB |
168 | } |
169 | } | |
170 | } | |
171 | ||
e1599b0c | 172 | /// Given a `source_file`, produces a sequence of token trees. |
9fa01778 XL |
173 | pub fn source_file_to_stream( |
174 | sess: &ParseSess, | |
175 | source_file: Lrc<SourceFile>, | |
176 | override_span: Option<Span>, | |
177 | ) -> (TokenStream, Vec<lexer::UnmatchedBrace>) { | |
a1dfa0c6 XL |
178 | panictry_buffer!(&sess.span_diagnostic, maybe_file_to_stream(sess, source_file, override_span)) |
179 | } | |
180 | ||
9fa01778 | 181 | /// Given a source file, produces a sequence of token trees. Returns any buffered errors from |
48663c56 | 182 | /// parsing the token stream. |
9fa01778 XL |
183 | pub fn maybe_file_to_stream( |
184 | sess: &ParseSess, | |
185 | source_file: Lrc<SourceFile>, | |
186 | override_span: Option<Span>, | |
187 | ) -> Result<(TokenStream, Vec<lexer::UnmatchedBrace>), Vec<Diagnostic>> { | |
1b1a35ee | 188 | let src = source_file.src.as_ref().unwrap_or_else(|| { |
17df50a5 XL |
189 | sess.span_diagnostic.bug(&format!( |
190 | "cannot lex `source_file` without source: {}", | |
191 | source_file.name.prefer_local() | |
192 | )); | |
1b1a35ee XL |
193 | }); |
194 | ||
195 | let (token_trees, unmatched_braces) = | |
196 | lexer::parse_token_trees(sess, src.as_str(), source_file.start_pos, override_span); | |
a1dfa0c6 | 197 | |
48663c56 XL |
198 | match token_trees { |
199 | Ok(stream) => Ok((stream, unmatched_braces)), | |
a1dfa0c6 XL |
200 | Err(err) => { |
201 | let mut buffer = Vec::with_capacity(1); | |
202 | err.buffer(&mut buffer); | |
9fa01778 | 203 | // Not using `emit_unclosed_delims` to use `db.buffer` |
48663c56 | 204 | for unmatched in unmatched_braces { |
e74abb32 XL |
205 | if let Some(err) = make_unclosed_delims_error(unmatched, &sess) { |
206 | err.buffer(&mut buffer); | |
9fa01778 | 207 | } |
9fa01778 | 208 | } |
a1dfa0c6 XL |
209 | Err(buffer) |
210 | } | |
211 | } | |
970d7e83 LB |
212 | } |
213 | ||
e1599b0c | 214 | /// Given a stream and the `ParseSess`, produces a parser. |
dc9dc135 XL |
215 | pub fn stream_to_parser<'a>( |
216 | sess: &'a ParseSess, | |
217 | stream: TokenStream, | |
218 | subparser_name: Option<&'static str>, | |
219 | ) -> Parser<'a> { | |
ba9703b0 | 220 | Parser::new(sess, stream, false, subparser_name) |
1a4d82fc JJ |
221 | } |
222 | ||
e74abb32 | 223 | /// Runs the given subparser `f` on the tokens of the given `attr`'s item. |
60c5eb7d | 224 | pub fn parse_in<'a, T>( |
e74abb32 | 225 | sess: &'a ParseSess, |
60c5eb7d XL |
226 | tts: TokenStream, |
227 | name: &'static str, | |
e74abb32 XL |
228 | mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, |
229 | ) -> PResult<'a, T> { | |
ba9703b0 | 230 | let mut parser = Parser::new(sess, tts, false, Some(name)); |
e74abb32 XL |
231 | let result = f(&mut parser)?; |
232 | if parser.token != token::Eof { | |
233 | parser.unexpected()?; | |
234 | } | |
235 | Ok(result) | |
8faf50e0 XL |
236 | } |
237 | ||
e74abb32 XL |
238 | // NOTE(Centril): The following probably shouldn't be here but it acknowledges the |
239 | // fact that architecturally, we are using parsing (read on below to understand why). | |
240 | ||
5869c6ff XL |
241 | pub fn nt_to_tokenstream( |
242 | nt: &Nonterminal, | |
243 | sess: &ParseSess, | |
244 | synthesize_tokens: CanSynthesizeMissingTokens, | |
245 | ) -> TokenStream { | |
e74abb32 XL |
246 | // A `Nonterminal` is often a parsed AST item. At this point we now |
247 | // need to convert the parsed AST to an actual token stream, e.g. | |
248 | // un-parse it basically. | |
249 | // | |
250 | // Unfortunately there's not really a great way to do that in a | |
251 | // guaranteed lossless fashion right now. The fallback here is to just | |
252 | // stringify the AST node and reparse it, but this loses all span | |
253 | // information. | |
254 | // | |
255 | // As a result, some AST nodes are annotated with the token stream they | |
256 | // came from. Here we attempt to extract these lossless token streams | |
257 | // before we fall back to the stringification. | |
29967ef6 XL |
258 | |
259 | let convert_tokens = | |
cdc7bbd5 | 260 | |tokens: Option<&LazyTokenStream>| Some(tokens?.create_token_stream().to_tokenstream()); |
29967ef6 | 261 | |
e74abb32 | 262 | let tokens = match *nt { |
cdc7bbd5 | 263 | Nonterminal::NtItem(ref item) => prepend_attrs(&item.attrs, item.tokens.as_ref()), |
fc512014 | 264 | Nonterminal::NtBlock(ref block) => convert_tokens(block.tokens.as_ref()), |
5869c6ff | 265 | Nonterminal::NtStmt(ref stmt) => { |
5869c6ff | 266 | if let ast::StmtKind::Empty = stmt.kind { |
cdc7bbd5 XL |
267 | let tokens = AttrAnnotatedTokenStream::new(vec![( |
268 | tokenstream::AttrAnnotatedTokenTree::Token(Token::new( | |
269 | TokenKind::Semi, | |
270 | stmt.span, | |
271 | )), | |
272 | Spacing::Alone, | |
273 | )]); | |
274 | prepend_attrs(&stmt.attrs(), Some(&LazyTokenStream::new(tokens))) | |
5869c6ff | 275 | } else { |
cdc7bbd5 | 276 | prepend_attrs(&stmt.attrs(), stmt.tokens()) |
5869c6ff XL |
277 | } |
278 | } | |
fc512014 XL |
279 | Nonterminal::NtPat(ref pat) => convert_tokens(pat.tokens.as_ref()), |
280 | Nonterminal::NtTy(ref ty) => convert_tokens(ty.tokens.as_ref()), | |
e74abb32 XL |
281 | Nonterminal::NtIdent(ident, is_raw) => { |
282 | Some(tokenstream::TokenTree::token(token::Ident(ident.name, is_raw), ident.span).into()) | |
283 | } | |
284 | Nonterminal::NtLifetime(ident) => { | |
285 | Some(tokenstream::TokenTree::token(token::Lifetime(ident.name), ident.span).into()) | |
8faf50e0 | 286 | } |
fc512014 XL |
287 | Nonterminal::NtMeta(ref attr) => convert_tokens(attr.tokens.as_ref()), |
288 | Nonterminal::NtPath(ref path) => convert_tokens(path.tokens.as_ref()), | |
289 | Nonterminal::NtVis(ref vis) => convert_tokens(vis.tokens.as_ref()), | |
dfeec247 | 290 | Nonterminal::NtTT(ref tt) => Some(tt.clone().into()), |
1b1a35ee | 291 | Nonterminal::NtExpr(ref expr) | Nonterminal::NtLiteral(ref expr) => { |
cdc7bbd5 | 292 | prepend_attrs(&expr.attrs, expr.tokens.as_ref()) |
f9f354fc | 293 | } |
e74abb32 XL |
294 | }; |
295 | ||
e74abb32 | 296 | if let Some(tokens) = tokens { |
5869c6ff XL |
297 | return tokens; |
298 | } else if matches!(synthesize_tokens, CanSynthesizeMissingTokens::Yes) { | |
299 | return fake_token_stream(sess, nt); | |
300 | } else { | |
cdc7bbd5 XL |
301 | panic!( |
302 | "Missing tokens for nt {:?} at {:?}: {:?}", | |
303 | nt, | |
304 | nt.span(), | |
305 | pprust::nonterminal_to_string(nt) | |
306 | ); | |
8faf50e0 | 307 | } |
e74abb32 | 308 | } |
8faf50e0 | 309 | |
cdc7bbd5 XL |
310 | fn prepend_attrs(attrs: &[Attribute], tokens: Option<&LazyTokenStream>) -> Option<TokenStream> { |
311 | let tokens = tokens?; | |
312 | if attrs.is_empty() { | |
313 | return Some(tokens.create_token_stream().to_tokenstream()); | |
314 | } | |
315 | let attr_data = AttributesData { attrs: attrs.to_vec().into(), tokens: tokens.clone() }; | |
316 | let wrapped = AttrAnnotatedTokenStream::new(vec![( | |
317 | AttrAnnotatedTokenTree::Attributes(attr_data), | |
318 | Spacing::Alone, | |
319 | )]); | |
320 | Some(wrapped.to_tokenstream()) | |
321 | } | |
322 | ||
5869c6ff XL |
323 | pub fn fake_token_stream(sess: &ParseSess, nt: &Nonterminal) -> TokenStream { |
324 | let source = pprust::nonterminal_to_string(nt); | |
325 | let filename = FileName::macro_expansion_source_code(&source); | |
326 | parse_stream_from_source_str(filename, source, sess, Some(nt.span())) | |
3dfed10e | 327 | } |