1 //! The main parser interface.
3 #![feature(bool_to_option)]
4 #![feature(crate_visibility_modifier)]
7 use syntax
::print
::pprust
;
8 use syntax
::sess
::ParseSess
;
9 use syntax
::token
::{self, Nonterminal}
;
10 use syntax
::tokenstream
::{self, TokenStream, TokenTree}
;
12 use rustc_errors
::{PResult, FatalError, Level, Diagnostic}
;
13 use rustc_data_structures
::sync
::Lrc
;
14 use syntax_pos
::{Span, SourceFile, FileName}
;
22 pub const MACRO_ARGUMENTS
: Option
<&'
static str> = Some("macro arguments");
26 use parser
::{Parser, emit_unclosed_delims, make_unclosed_delims_error}
;
28 pub mod validate_attr
;
33 pub struct Directory
<'a
> {
34 pub path
: Cow
<'a
, Path
>,
35 pub ownership
: DirectoryOwnership
,
38 #[derive(Copy, Clone)]
39 pub enum DirectoryOwnership
{
41 // None if `mod.rs`, `Some("foo")` if we're in `foo.rs`.
42 relative
: Option
<ast
::Ident
>,
48 // A bunch of utility functions of the form `parse_<thing>_from_<source>`
49 // where <thing> includes crate, expr, item, stmt, tts, and one that
50 // uses a HOF to parse anything, and <source> includes file and
53 /// A variant of 'panictry!' that works on a Vec<Diagnostic> instead of a single DiagnosticBuilder.
54 macro_rules
! panictry_buffer
{
55 ($handler
:expr
, $e
:expr
) => ({
56 use std
::result
::Result
::{Ok, Err}
;
57 use rustc_errors
::FatalError
;
62 $handler
.emit_diagnostic(&e
);
70 pub fn parse_crate_from_file
<'a
>(input
: &Path
, sess
: &'a ParseSess
) -> PResult
<'a
, ast
::Crate
> {
71 let mut parser
= new_parser_from_file(sess
, input
);
72 parser
.parse_crate_mod()
75 pub fn parse_crate_attrs_from_file
<'a
>(input
: &Path
, sess
: &'a ParseSess
)
76 -> PResult
<'a
, Vec
<ast
::Attribute
>> {
77 let mut parser
= new_parser_from_file(sess
, input
);
78 parser
.parse_inner_attributes()
81 pub fn parse_crate_from_source_str(name
: FileName
, source
: String
, sess
: &ParseSess
)
82 -> PResult
<'_
, ast
::Crate
> {
83 new_parser_from_source_str(sess
, name
, source
).parse_crate_mod()
86 pub fn parse_crate_attrs_from_source_str(name
: FileName
, source
: String
, sess
: &ParseSess
)
87 -> PResult
<'_
, Vec
<ast
::Attribute
>> {
88 new_parser_from_source_str(sess
, name
, source
).parse_inner_attributes()
91 pub fn parse_stream_from_source_str(
95 override_span
: Option
<Span
>,
97 let (stream
, mut errors
) = source_file_to_stream(
99 sess
.source_map().new_source_file(name
, source
),
102 emit_unclosed_delims(&mut errors
, &sess
);
106 /// Creates a new parser from a source string.
107 pub fn new_parser_from_source_str(sess
: &ParseSess
, name
: FileName
, source
: String
) -> Parser
<'_
> {
108 panictry_buffer
!(&sess
.span_diagnostic
, maybe_new_parser_from_source_str(sess
, name
, source
))
111 /// Creates a new parser from a source string. Returns any buffered errors from lexing the initial
113 pub fn maybe_new_parser_from_source_str(sess
: &ParseSess
, name
: FileName
, source
: String
)
114 -> Result
<Parser
<'_
>, Vec
<Diagnostic
>>
116 let mut parser
= maybe_source_file_to_parser(sess
,
117 sess
.source_map().new_source_file(name
, source
))?
;
118 parser
.recurse_into_file_modules
= false;
122 /// Creates a new parser, handling errors as appropriate if the file doesn't exist.
123 pub fn new_parser_from_file
<'a
>(sess
: &'a ParseSess
, path
: &Path
) -> Parser
<'a
> {
124 source_file_to_parser(sess
, file_to_source_file(sess
, path
, None
))
127 /// Creates a new parser, returning buffered diagnostics if the file doesn't exist,
128 /// or from lexing the initial token stream.
129 pub fn maybe_new_parser_from_file
<'a
>(sess
: &'a ParseSess
, path
: &Path
)
130 -> Result
<Parser
<'a
>, Vec
<Diagnostic
>> {
131 let file
= try_file_to_source_file(sess
, path
, None
).map_err(|db
| vec
![db
])?
;
132 maybe_source_file_to_parser(sess
, file
)
135 /// Given a session, a crate config, a path, and a span, add
136 /// the file at the given path to the `source_map`, and returns a parser.
137 /// On an error, uses the given span as the source of the problem.
138 pub fn new_sub_parser_from_file
<'a
>(sess
: &'a ParseSess
,
140 directory_ownership
: DirectoryOwnership
,
141 module_name
: Option
<String
>,
142 sp
: Span
) -> Parser
<'a
> {
143 let mut p
= source_file_to_parser(sess
, file_to_source_file(sess
, path
, Some(sp
)));
144 p
.directory
.ownership
= directory_ownership
;
145 p
.root_module_name
= module_name
;
149 /// Given a `source_file` and config, returns a parser.
150 fn source_file_to_parser(sess
: &ParseSess
, source_file
: Lrc
<SourceFile
>) -> Parser
<'_
> {
151 panictry_buffer
!(&sess
.span_diagnostic
,
152 maybe_source_file_to_parser(sess
, source_file
))
155 /// Given a `source_file` and config, return a parser. Returns any buffered errors from lexing the
156 /// initial token stream.
157 fn maybe_source_file_to_parser(
159 source_file
: Lrc
<SourceFile
>,
160 ) -> Result
<Parser
<'_
>, Vec
<Diagnostic
>> {
161 let end_pos
= source_file
.end_pos
;
162 let (stream
, unclosed_delims
) = maybe_file_to_stream(sess
, source_file
, None
)?
;
163 let mut parser
= stream_to_parser(sess
, stream
, None
);
164 parser
.unclosed_delims
= unclosed_delims
;
165 if parser
.token
== token
::Eof
&& parser
.token
.span
.is_dummy() {
166 parser
.token
.span
= Span
::new(end_pos
, end_pos
, parser
.token
.span
.ctxt());
172 // Must preserve old name for now, because `quote!` from the *existing*
173 // compiler expands into it.
174 pub fn new_parser_from_tts(sess
: &ParseSess
, tts
: Vec
<TokenTree
>) -> Parser
<'_
> {
175 stream_to_parser(sess
, tts
.into_iter().collect(), crate::MACRO_ARGUMENTS
)
181 /// Given a session and a path and an optional span (for error reporting),
182 /// add the path to the session's source_map and return the new source_file or
183 /// error when a file can't be read.
184 fn try_file_to_source_file(sess
: &ParseSess
, path
: &Path
, spanopt
: Option
<Span
>)
185 -> Result
<Lrc
<SourceFile
>, Diagnostic
> {
186 sess
.source_map().load_file(path
)
188 let msg
= format
!("couldn't read {}: {}", path
.display(), e
);
189 let mut diag
= Diagnostic
::new(Level
::Fatal
, &msg
);
190 if let Some(sp
) = spanopt
{
197 /// Given a session and a path and an optional span (for error reporting),
198 /// adds the path to the session's `source_map` and returns the new `source_file`.
199 fn file_to_source_file(sess
: &ParseSess
, path
: &Path
, spanopt
: Option
<Span
>)
201 match try_file_to_source_file(sess
, path
, spanopt
) {
202 Ok(source_file
) => source_file
,
204 sess
.span_diagnostic
.emit_diagnostic(&d
);
210 /// Given a `source_file`, produces a sequence of token trees.
211 pub fn source_file_to_stream(
213 source_file
: Lrc
<SourceFile
>,
214 override_span
: Option
<Span
>,
215 ) -> (TokenStream
, Vec
<lexer
::UnmatchedBrace
>) {
216 panictry_buffer
!(&sess
.span_diagnostic
, maybe_file_to_stream(sess
, source_file
, override_span
))
219 /// Given a source file, produces a sequence of token trees. Returns any buffered errors from
220 /// parsing the token stream.
221 pub fn maybe_file_to_stream(
223 source_file
: Lrc
<SourceFile
>,
224 override_span
: Option
<Span
>,
225 ) -> Result
<(TokenStream
, Vec
<lexer
::UnmatchedBrace
>), Vec
<Diagnostic
>> {
226 let srdr
= lexer
::StringReader
::new(sess
, source_file
, override_span
);
227 let (token_trees
, unmatched_braces
) = srdr
.into_token_trees();
230 Ok(stream
) => Ok((stream
, unmatched_braces
)),
232 let mut buffer
= Vec
::with_capacity(1);
233 err
.buffer(&mut buffer
);
234 // Not using `emit_unclosed_delims` to use `db.buffer`
235 for unmatched
in unmatched_braces
{
236 if let Some(err
) = make_unclosed_delims_error(unmatched
, &sess
) {
237 err
.buffer(&mut buffer
);
245 /// Given a stream and the `ParseSess`, produces a parser.
246 pub fn stream_to_parser
<'a
>(
249 subparser_name
: Option
<&'
static str>,
251 Parser
::new(sess
, stream
, None
, true, false, subparser_name
)
254 /// Given a stream, the `ParseSess` and the base directory, produces a parser.
256 /// Use this function when you are creating a parser from the token stream
257 /// and also care about the current working directory of the parser (e.g.,
258 /// you are trying to resolve modules defined inside a macro invocation).
262 /// The main usage of this function is outside of rustc, for those who uses
263 /// libsyntax as a library. Please do not remove this function while refactoring
264 /// just because it is not used in rustc codebase!
265 pub fn stream_to_parser_with_base_dir
<'a
>(
268 base_dir
: Directory
<'a
>,
270 Parser
::new(sess
, stream
, Some(base_dir
), true, false, None
)
273 /// Runs the given subparser `f` on the tokens of the given `attr`'s item.
274 pub fn parse_in
<'a
, T
>(
278 mut f
: impl FnMut(&mut Parser
<'a
>) -> PResult
<'a
, T
>,
279 ) -> PResult
<'a
, T
> {
280 let mut parser
= Parser
::new(sess
, tts
, None
, false, false, Some(name
));
281 let result
= f(&mut parser
)?
;
282 if parser
.token
!= token
::Eof
{
283 parser
.unexpected()?
;
288 // NOTE(Centril): The following probably shouldn't be here but it acknowledges the
289 // fact that architecturally, we are using parsing (read on below to understand why).
291 pub fn nt_to_tokenstream(nt
: &Nonterminal
, sess
: &ParseSess
, span
: Span
) -> TokenStream
{
292 // A `Nonterminal` is often a parsed AST item. At this point we now
293 // need to convert the parsed AST to an actual token stream, e.g.
294 // un-parse it basically.
296 // Unfortunately there's not really a great way to do that in a
297 // guaranteed lossless fashion right now. The fallback here is to just
298 // stringify the AST node and reparse it, but this loses all span
301 // As a result, some AST nodes are annotated with the token stream they
302 // came from. Here we attempt to extract these lossless token streams
303 // before we fall back to the stringification.
304 let tokens
= match *nt
{
305 Nonterminal
::NtItem(ref item
) => {
306 prepend_attrs(sess
, &item
.attrs
, item
.tokens
.as_ref(), span
)
308 Nonterminal
::NtTraitItem(ref item
) => {
309 prepend_attrs(sess
, &item
.attrs
, item
.tokens
.as_ref(), span
)
311 Nonterminal
::NtImplItem(ref item
) => {
312 prepend_attrs(sess
, &item
.attrs
, item
.tokens
.as_ref(), span
)
314 Nonterminal
::NtIdent(ident
, is_raw
) => {
315 Some(tokenstream
::TokenTree
::token(token
::Ident(ident
.name
, is_raw
), ident
.span
).into())
317 Nonterminal
::NtLifetime(ident
) => {
318 Some(tokenstream
::TokenTree
::token(token
::Lifetime(ident
.name
), ident
.span
).into())
320 Nonterminal
::NtTT(ref tt
) => {
321 Some(tt
.clone().into())
326 // FIXME(#43081): Avoid this pretty-print + reparse hack
327 let source
= pprust
::nonterminal_to_string(nt
);
328 let filename
= FileName
::macro_expansion_source_code(&source
);
329 let tokens_for_real
= parse_stream_from_source_str(filename
, source
, sess
, Some(span
));
331 // During early phases of the compiler the AST could get modified
332 // directly (e.g., attributes added or removed) and the internal cache
333 // of tokens my not be invalidated or updated. Consequently if the
334 // "lossless" token stream disagrees with our actual stringification
335 // (which has historically been much more battle-tested) then we go
336 // with the lossy stream anyway (losing span information).
338 // Note that the comparison isn't `==` here to avoid comparing spans,
339 // but it *also* is a "probable" equality which is a pretty weird
340 // definition. We mostly want to catch actual changes to the AST
341 // like a `#[cfg]` being processed or some weird `macro_rules!`
344 // What we *don't* want to catch is the fact that a user-defined
345 // literal like `0xf` is stringified as `15`, causing the cached token
346 // stream to not be literal `==` token-wise (ignoring spans) to the
347 // token stream we got from stringification.
349 // Instead the "probably equal" check here is "does each token
350 // recursively have the same discriminant?" We basically don't look at
351 // the token values here and assume that such fine grained token stream
352 // modifications, including adding/removing typically non-semantic
353 // tokens such as extra braces and commas, don't happen.
354 if let Some(tokens
) = tokens
{
355 if tokens
.probably_equal_for_proc_macro(&tokens_for_real
) {
358 info
!("cached tokens found, but they're not \"probably equal\", \
359 going with stringified version");
361 return tokens_for_real
366 attrs
: &[ast
::Attribute
],
367 tokens
: Option
<&tokenstream
::TokenStream
>,
368 span
: syntax_pos
::Span
369 ) -> Option
<tokenstream
::TokenStream
> {
370 let tokens
= tokens?
;
371 if attrs
.len() == 0 {
372 return Some(tokens
.clone())
374 let mut builder
= tokenstream
::TokenStreamBuilder
::new();
376 assert_eq
!(attr
.style
, ast
::AttrStyle
::Outer
,
377 "inner attributes should prevent cached tokens from existing");
379 let source
= pprust
::attribute_to_string(attr
);
380 let macro_filename
= FileName
::macro_expansion_source_code(&source
);
382 let item
= match attr
.kind
{
383 ast
::AttrKind
::Normal(ref item
) => item
,
384 ast
::AttrKind
::DocComment(_
) => {
385 let stream
= parse_stream_from_source_str(macro_filename
, source
, sess
, Some(span
));
386 builder
.push(stream
);
391 // synthesize # [ $path $tokens ] manually here
392 let mut brackets
= tokenstream
::TokenStreamBuilder
::new();
394 // For simple paths, push the identifier directly
395 if item
.path
.segments
.len() == 1 && item
.path
.segments
[0].args
.is_none() {
396 let ident
= item
.path
.segments
[0].ident
;
397 let token
= token
::Ident(ident
.name
, ident
.as_str().starts_with("r#"));
398 brackets
.push(tokenstream
::TokenTree
::token(token
, ident
.span
));
400 // ... and for more complicated paths, fall back to a reparse hack that
401 // should eventually be removed.
403 let stream
= parse_stream_from_source_str(macro_filename
, source
, sess
, Some(span
));
404 brackets
.push(stream
);
407 brackets
.push(item
.args
.outer_tokens());
409 // The span we list here for `#` and for `[ ... ]` are both wrong in
410 // that it encompasses more than each token, but it hopefully is "good
411 // enough" for now at least.
412 builder
.push(tokenstream
::TokenTree
::token(token
::Pound
, attr
.span
));
413 let delim_span
= tokenstream
::DelimSpan
::from_single(attr
.span
);
414 builder
.push(tokenstream
::TokenTree
::Delimited(
415 delim_span
, token
::DelimToken
::Bracket
, brackets
.build().into()));
417 builder
.push(tokens
.clone());
418 Some(builder
.build())