1 //! The main parser interface.
3 #![feature(bool_to_option)]
4 #![feature(crate_visibility_modifier)]
5 #![feature(bindings_after_at)]
6 #![feature(try_blocks)]
7 #![feature(or_patterns)]
10 use rustc_ast
::token
::{self, Nonterminal}
;
11 use rustc_ast
::tokenstream
::{self, TokenStream, TokenTree}
;
12 use rustc_ast_pretty
::pprust
;
13 use rustc_data_structures
::sync
::Lrc
;
14 use rustc_errors
::{Diagnostic, FatalError, Level, PResult}
;
15 use rustc_session
::parse
::ParseSess
;
16 use rustc_span
::{FileName, SourceFile, Span}
;
21 use log
::{debug, info}
;
23 pub const MACRO_ARGUMENTS
: Option
<&'
static str> = Some("macro arguments");
27 use parser
::{emit_unclosed_delims, make_unclosed_delims_error, Parser}
;
29 pub mod validate_attr
;
31 // A bunch of utility functions of the form `parse_<thing>_from_<source>`
32 // where <thing> includes crate, expr, item, stmt, tts, and one that
33 // uses a HOF to parse anything, and <source> includes file and
36 /// A variant of 'panictry!' that works on a Vec<Diagnostic> instead of a single DiagnosticBuilder.
37 macro_rules
! panictry_buffer
{
38 ($handler
:expr
, $e
:expr
) => {{
39 use rustc_errors
::FatalError
;
40 use std
::result
::Result
::{Err, Ok}
;
45 $handler
.emit_diagnostic(&e
);
53 pub fn parse_crate_from_file
<'a
>(input
: &Path
, sess
: &'a ParseSess
) -> PResult
<'a
, ast
::Crate
> {
54 let mut parser
= new_parser_from_file(sess
, input
, None
);
55 parser
.parse_crate_mod()
58 pub fn parse_crate_attrs_from_file
<'a
>(
61 ) -> PResult
<'a
, Vec
<ast
::Attribute
>> {
62 let mut parser
= new_parser_from_file(sess
, input
, None
);
63 parser
.parse_inner_attributes()
66 pub fn parse_crate_from_source_str(
70 ) -> PResult
<'_
, ast
::Crate
> {
71 new_parser_from_source_str(sess
, name
, source
).parse_crate_mod()
74 pub fn parse_crate_attrs_from_source_str(
78 ) -> PResult
<'_
, Vec
<ast
::Attribute
>> {
79 new_parser_from_source_str(sess
, name
, source
).parse_inner_attributes()
82 pub fn parse_stream_from_source_str(
86 override_span
: Option
<Span
>,
88 let (stream
, mut errors
) =
89 source_file_to_stream(sess
, sess
.source_map().new_source_file(name
, source
), override_span
);
90 emit_unclosed_delims(&mut errors
, &sess
);
94 /// Creates a new parser from a source string.
95 pub fn new_parser_from_source_str(sess
: &ParseSess
, name
: FileName
, source
: String
) -> Parser
<'_
> {
96 panictry_buffer
!(&sess
.span_diagnostic
, maybe_new_parser_from_source_str(sess
, name
, source
))
99 /// Creates a new parser from a source string. Returns any buffered errors from lexing the initial
101 pub fn maybe_new_parser_from_source_str(
105 ) -> Result
<Parser
<'_
>, Vec
<Diagnostic
>> {
106 maybe_source_file_to_parser(sess
, sess
.source_map().new_source_file(name
, source
))
109 /// Creates a new parser, handling errors as appropriate if the file doesn't exist.
110 /// If a span is given, that is used on an error as the as the source of the problem.
111 pub fn new_parser_from_file
<'a
>(sess
: &'a ParseSess
, path
: &Path
, sp
: Option
<Span
>) -> Parser
<'a
> {
112 source_file_to_parser(sess
, file_to_source_file(sess
, path
, sp
))
115 /// Creates a new parser, returning buffered diagnostics if the file doesn't exist,
116 /// or from lexing the initial token stream.
117 pub fn maybe_new_parser_from_file
<'a
>(
120 ) -> Result
<Parser
<'a
>, Vec
<Diagnostic
>> {
121 let file
= try_file_to_source_file(sess
, path
, None
).map_err(|db
| vec
![db
])?
;
122 maybe_source_file_to_parser(sess
, file
)
125 /// Given a `source_file` and config, returns a parser.
126 fn source_file_to_parser(sess
: &ParseSess
, source_file
: Lrc
<SourceFile
>) -> Parser
<'_
> {
127 panictry_buffer
!(&sess
.span_diagnostic
, maybe_source_file_to_parser(sess
, source_file
))
130 /// Given a `source_file` and config, return a parser. Returns any buffered errors from lexing the
131 /// initial token stream.
132 fn maybe_source_file_to_parser(
134 source_file
: Lrc
<SourceFile
>,
135 ) -> Result
<Parser
<'_
>, Vec
<Diagnostic
>> {
136 let end_pos
= source_file
.end_pos
;
137 let (stream
, unclosed_delims
) = maybe_file_to_stream(sess
, source_file
, None
)?
;
138 let mut parser
= stream_to_parser(sess
, stream
, None
);
139 parser
.unclosed_delims
= unclosed_delims
;
140 if parser
.token
== token
::Eof
{
141 parser
.token
.span
= Span
::new(end_pos
, end_pos
, parser
.token
.span
.ctxt());
147 // Must preserve old name for now, because `quote!` from the *existing*
148 // compiler expands into it.
149 pub fn new_parser_from_tts(sess
: &ParseSess
, tts
: Vec
<TokenTree
>) -> Parser
<'_
> {
150 stream_to_parser(sess
, tts
.into_iter().collect(), crate::MACRO_ARGUMENTS
)
155 /// Given a session and a path and an optional span (for error reporting),
156 /// add the path to the session's source_map and return the new source_file or
157 /// error when a file can't be read.
158 fn try_file_to_source_file(
161 spanopt
: Option
<Span
>,
162 ) -> Result
<Lrc
<SourceFile
>, Diagnostic
> {
163 sess
.source_map().load_file(path
).map_err(|e
| {
164 let msg
= format
!("couldn't read {}: {}", path
.display(), e
);
165 let mut diag
= Diagnostic
::new(Level
::Fatal
, &msg
);
166 if let Some(sp
) = spanopt
{
173 /// Given a session and a path and an optional span (for error reporting),
174 /// adds the path to the session's `source_map` and returns the new `source_file`.
175 fn file_to_source_file(sess
: &ParseSess
, path
: &Path
, spanopt
: Option
<Span
>) -> Lrc
<SourceFile
> {
176 match try_file_to_source_file(sess
, path
, spanopt
) {
177 Ok(source_file
) => source_file
,
179 sess
.span_diagnostic
.emit_diagnostic(&d
);
185 /// Given a `source_file`, produces a sequence of token trees.
186 pub fn source_file_to_stream(
188 source_file
: Lrc
<SourceFile
>,
189 override_span
: Option
<Span
>,
190 ) -> (TokenStream
, Vec
<lexer
::UnmatchedBrace
>) {
191 panictry_buffer
!(&sess
.span_diagnostic
, maybe_file_to_stream(sess
, source_file
, override_span
))
194 /// Given a source file, produces a sequence of token trees. Returns any buffered errors from
195 /// parsing the token stream.
196 pub fn maybe_file_to_stream(
198 source_file
: Lrc
<SourceFile
>,
199 override_span
: Option
<Span
>,
200 ) -> Result
<(TokenStream
, Vec
<lexer
::UnmatchedBrace
>), Vec
<Diagnostic
>> {
201 let srdr
= lexer
::StringReader
::new(sess
, source_file
, override_span
);
202 let (token_trees
, unmatched_braces
) = srdr
.into_token_trees();
205 Ok(stream
) => Ok((stream
, unmatched_braces
)),
207 let mut buffer
= Vec
::with_capacity(1);
208 err
.buffer(&mut buffer
);
209 // Not using `emit_unclosed_delims` to use `db.buffer`
210 for unmatched
in unmatched_braces
{
211 if let Some(err
) = make_unclosed_delims_error(unmatched
, &sess
) {
212 err
.buffer(&mut buffer
);
220 /// Given a stream and the `ParseSess`, produces a parser.
221 pub fn stream_to_parser
<'a
>(
224 subparser_name
: Option
<&'
static str>,
226 Parser
::new(sess
, stream
, false, subparser_name
)
229 /// Runs the given subparser `f` on the tokens of the given `attr`'s item.
230 pub fn parse_in
<'a
, T
>(
234 mut f
: impl FnMut(&mut Parser
<'a
>) -> PResult
<'a
, T
>,
235 ) -> PResult
<'a
, T
> {
236 let mut parser
= Parser
::new(sess
, tts
, false, Some(name
));
237 let result
= f(&mut parser
)?
;
238 if parser
.token
!= token
::Eof
{
239 parser
.unexpected()?
;
244 // NOTE(Centril): The following probably shouldn't be here but it acknowledges the
245 // fact that architecturally, we are using parsing (read on below to understand why).
247 pub fn nt_to_tokenstream(nt
: &Nonterminal
, sess
: &ParseSess
, span
: Span
) -> TokenStream
{
248 // A `Nonterminal` is often a parsed AST item. At this point we now
249 // need to convert the parsed AST to an actual token stream, e.g.
250 // un-parse it basically.
252 // Unfortunately there's not really a great way to do that in a
253 // guaranteed lossless fashion right now. The fallback here is to just
254 // stringify the AST node and reparse it, but this loses all span
257 // As a result, some AST nodes are annotated with the token stream they
258 // came from. Here we attempt to extract these lossless token streams
259 // before we fall back to the stringification.
260 let tokens
= match *nt
{
261 Nonterminal
::NtItem(ref item
) => {
262 prepend_attrs(sess
, &item
.attrs
, item
.tokens
.as_ref(), span
)
264 Nonterminal
::NtIdent(ident
, is_raw
) => {
265 Some(tokenstream
::TokenTree
::token(token
::Ident(ident
.name
, is_raw
), ident
.span
).into())
267 Nonterminal
::NtLifetime(ident
) => {
268 Some(tokenstream
::TokenTree
::token(token
::Lifetime(ident
.name
), ident
.span
).into())
270 Nonterminal
::NtTT(ref tt
) => Some(tt
.clone().into()),
271 Nonterminal
::NtExpr(ref expr
) => {
272 if expr
.tokens
.is_none() {
273 debug
!("missing tokens for expr {:?}", expr
);
275 prepend_attrs(sess
, &expr
.attrs
, expr
.tokens
.as_ref(), span
)
280 // FIXME(#43081): Avoid this pretty-print + reparse hack
281 let source
= pprust
::nonterminal_to_string(nt
);
282 let filename
= FileName
::macro_expansion_source_code(&source
);
283 let tokens_for_real
= parse_stream_from_source_str(filename
, source
, sess
, Some(span
));
285 // During early phases of the compiler the AST could get modified
286 // directly (e.g., attributes added or removed) and the internal cache
287 // of tokens my not be invalidated or updated. Consequently if the
288 // "lossless" token stream disagrees with our actual stringification
289 // (which has historically been much more battle-tested) then we go
290 // with the lossy stream anyway (losing span information).
292 // Note that the comparison isn't `==` here to avoid comparing spans,
293 // but it *also* is a "probable" equality which is a pretty weird
294 // definition. We mostly want to catch actual changes to the AST
295 // like a `#[cfg]` being processed or some weird `macro_rules!`
298 // What we *don't* want to catch is the fact that a user-defined
299 // literal like `0xf` is stringified as `15`, causing the cached token
300 // stream to not be literal `==` token-wise (ignoring spans) to the
301 // token stream we got from stringification.
303 // Instead the "probably equal" check here is "does each token
304 // recursively have the same discriminant?" We basically don't look at
305 // the token values here and assume that such fine grained token stream
306 // modifications, including adding/removing typically non-semantic
307 // tokens such as extra braces and commas, don't happen.
308 if let Some(tokens
) = tokens
{
309 if tokens
.probably_equal_for_proc_macro(&tokens_for_real
) {
313 "cached tokens found, but they're not \"probably equal\", \
314 going with stringified version"
316 info
!("cached tokens: {:?}", tokens
);
317 info
!("reparsed tokens: {:?}", tokens_for_real
);
324 attrs
: &[ast
::Attribute
],
325 tokens
: Option
<&tokenstream
::TokenStream
>,
326 span
: rustc_span
::Span
,
327 ) -> Option
<tokenstream
::TokenStream
> {
328 let tokens
= tokens?
;
329 if attrs
.is_empty() {
330 return Some(tokens
.clone());
332 let mut builder
= tokenstream
::TokenStreamBuilder
::new();
336 ast
::AttrStyle
::Outer
,
337 "inner attributes should prevent cached tokens from existing"
340 let source
= pprust
::attribute_to_string(attr
);
341 let macro_filename
= FileName
::macro_expansion_source_code(&source
);
343 let item
= match attr
.kind
{
344 ast
::AttrKind
::Normal(ref item
) => item
,
345 ast
::AttrKind
::DocComment(_
) => {
346 let stream
= parse_stream_from_source_str(macro_filename
, source
, sess
, Some(span
));
347 builder
.push(stream
);
352 // synthesize # [ $path $tokens ] manually here
353 let mut brackets
= tokenstream
::TokenStreamBuilder
::new();
355 // For simple paths, push the identifier directly
356 if item
.path
.segments
.len() == 1 && item
.path
.segments
[0].args
.is_none() {
357 let ident
= item
.path
.segments
[0].ident
;
358 let token
= token
::Ident(ident
.name
, ident
.as_str().starts_with("r#"));
359 brackets
.push(tokenstream
::TokenTree
::token(token
, ident
.span
));
361 // ... and for more complicated paths, fall back to a reparse hack that
362 // should eventually be removed.
364 let stream
= parse_stream_from_source_str(macro_filename
, source
, sess
, Some(span
));
365 brackets
.push(stream
);
368 brackets
.push(item
.args
.outer_tokens());
370 // The span we list here for `#` and for `[ ... ]` are both wrong in
371 // that it encompasses more than each token, but it hopefully is "good
372 // enough" for now at least.
373 builder
.push(tokenstream
::TokenTree
::token(token
::Pound
, attr
.span
));
374 let delim_span
= tokenstream
::DelimSpan
::from_single(attr
.span
);
375 builder
.push(tokenstream
::TokenTree
::Delimited(
377 token
::DelimToken
::Bracket
,
381 builder
.push(tokens
.clone());
382 Some(builder
.build())