[rustc.git] / src / libsyntax / parse / lexer / tokentrees.rs

use crate::print::pprust::token_to_string;
use crate::parse::lexer::{StringReader, UnmatchedBrace};
use crate::parse::{token, PResult};
use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint};

impl<'a> StringReader<'a> {
    // Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
    crate fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
        let mut tts = Vec::new();

        while self.token != token::Eof {
            tts.push(self.parse_token_tree()?);
        }

        Ok(TokenStream::new(tts))
    }

    // Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
    fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
        let mut tts = vec![];
        loop {
            if let token::CloseDelim(..) = self.token {
                return TokenStream::new(tts);
            }

            match self.parse_token_tree() {
                Ok(tree) => tts.push(tree),
                Err(mut e) => {
                    e.emit();
                    return TokenStream::new(tts);
                }
            }
        }
    }

    fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
        let sm = self.sess.source_map();
        match self.token {
            token::Eof => {
                let msg = "this file contains an un-closed delimiter";
                let mut err = self.sess.span_diagnostic.struct_span_err(self.span, msg);
                for &(_, sp) in &self.open_braces {
                    err.span_label(sp, "un-closed delimiter");
                }

                if let Some((delim, _)) = self.open_braces.last() {
                    if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter()
                        .filter(|(d, open_sp, close_sp)| {

                        if let Some(close_padding) = sm.span_to_margin(*close_sp) {
                            if let Some(open_padding) = sm.span_to_margin(*open_sp) {
                                return delim == d && close_padding != open_padding;
                            }
                        }
                        false
                        }).next()  // these are in reverse order as they get inserted on close, but
                    {              // we want the last open/first close
                        err.span_label(
                            *open_sp,
                            "this delimiter might not be properly closed...",
                        );
                        err.span_label(
                            *close_sp,
                            "...as it matches this but it has different indentation",
                        );
                    }
                }
                Err(err)
            },
            token::OpenDelim(delim) => {
                // The span for beginning of the delimited section
                let pre_span = self.span;

                // Parse the open delimiter.
                self.open_braces.push((delim, self.span));
                self.real_token();

                // Parse the token trees within the delimiters.
                // We stop at any delimiter so we can try to recover if the user
                // uses an incorrect delimiter.
                let tts = self.parse_token_trees_until_close_delim();

                // Expand to cover the entire delimited token tree
                let delim_span = DelimSpan::from_pair(pre_span, self.span);

                match self.token {
                    // Correct delimiter.
                    token::CloseDelim(d) if d == delim => {
                        let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
                        if self.open_braces.len() == 0 {
                            // Clear up these spans to avoid suggesting them as we've found
                            // properly matched delimiters so far for an entire block.
                            self.matching_delim_spans.clear();
                        } else {
                            self.matching_delim_spans.push(
                                (open_brace, open_brace_span, self.span),
                            );
                        }
                        // Parse the close delimiter.
                        self.real_token();
                    }
                    // Incorrect delimiter.
                    token::CloseDelim(other) => {
                        let mut unclosed_delimiter = None;
                        let mut candidate = None;
                        if self.last_unclosed_found_span != Some(self.span) {
                            // do not complain about the same unclosed delimiter multiple times
                            self.last_unclosed_found_span = Some(self.span);
                            // This is a conservative error: only report the last unclosed
                            // delimiter. The previous unclosed delimiters could actually be
                            // closed! The parser just hasn't gotten to them yet.
                            if let Some(&(_, sp)) = self.open_braces.last() {
                                unclosed_delimiter = Some(sp);
                            };
                            if let Some(current_padding) = sm.span_to_margin(self.span) {
                                for (brace, brace_span) in &self.open_braces {
                                    if let Some(padding) = sm.span_to_margin(*brace_span) {
                                        // high likelihood of these two corresponding
                                        if current_padding == padding && brace == &other {
                                            candidate = Some(*brace_span);
                                        }
                                    }
                                }
                            }
                            let (tok, _) = self.open_braces.pop().unwrap();
                            self.unmatched_braces.push(UnmatchedBrace {
                                expected_delim: tok,
                                found_delim: other,
                                found_span: self.span,
                                unclosed_span: unclosed_delimiter,
                                candidate_span: candidate,
                            });
                        } else {
                            self.open_braces.pop();
                        }

                        // If the incorrect delimiter matches an earlier opening
                        // delimiter, then don't consume it (it can be used to
                        // close the earlier one). Otherwise, consume it.
                        // E.g., we try to recover from:
                        // fn foo() {
                        //     bar(baz(
                        // }  // Incorrect delimiter but matches the earlier `{`
                        if !self.open_braces.iter().any(|&(b, _)| b == other) {
                            self.real_token();
                        }
                    }
                    token::Eof => {
                        // Silently recover, the EOF token will be seen again
                        // and an error emitted then. Thus we don't pop from
                        // self.open_braces here.
                    },
                    _ => {}
                }

                Ok(TokenTree::Delimited(
                    delim_span,
                    delim,
                    tts.into()
                ).into())
            },
            token::CloseDelim(_) => {
                // An unexpected closing delimiter (i.e., there is no
                // matching opening delimiter).
                let token_str = token_to_string(&self.token);
                let msg = format!("unexpected close delimiter: `{}`", token_str);
                let mut err = self.sess.span_diagnostic.struct_span_err(self.span, &msg);
                err.span_label(self.span, "unexpected close delimiter");
                Err(err)
            },
            _ => {
                let tt = TokenTree::Token(self.span, self.token.clone());
                // Note that testing for joint-ness here is done via the raw
                // source span as the joint-ness is a property of the raw source
                // rather than wanting to take `override_span` into account.
                let raw = self.span_src_raw;
                self.real_token();
                let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token);
                Ok((tt, if is_joint { Joint } else { NonJoint }))
            }
        }
    }
}
Commit	Line	Data
9fa01778 XL	1	use crate::print::pprust::token_to_string;
	2	use crate::parse::lexer::{StringReader, UnmatchedBrace};
	3	use crate::parse::{token, PResult};
	4	use crate::tokenstream::{DelimSpan, IsJoint::*, TokenStream, TokenTree, TreeAndJoint};
32a655c1 SL	5
	6	impl<'a> StringReader<'a> {
	7	// Parse a stream of tokens into a list of `TokenTree`s, up to an `Eof`.
94b46f34	8	crate fn parse_all_token_trees(&mut self) -> PResult<'a, TokenStream> {
32a655c1	9	let mut tts = Vec::new();
b7449926	10
32a655c1	11	while self.token != token::Eof {
94b46f34	12	tts.push(self.parse_token_tree()?);
32a655c1	13	}
b7449926	14
0731742a	15	Ok(TokenStream::new(tts))
32a655c1 SL	16	}
	17
	18	// Parse a stream of tokens into a list of `TokenTree`s, up to a `CloseDelim`.
8bb4bdeb	19	fn parse_token_trees_until_close_delim(&mut self) -> TokenStream {
32a655c1 SL	20	let mut tts = vec![];
	21	loop {
	22	if let token::CloseDelim(..) = self.token {
0731742a	23	return TokenStream::new(tts);
32a655c1	24	}
b7449926	25
94b46f34 XL	26	match self.parse_token_tree() {
94b46f34 XL	27	Ok(tree) => tts.push(tree),
32a655c1 SL	28	Err(mut e) => {
32a655c1 SL	29	e.emit();
0731742a	30	return TokenStream::new(tts);
32a655c1	31	}
94b46f34	32	}
32a655c1 SL	33	}
	34	}
	35
0731742a	36	fn parse_token_tree(&mut self) -> PResult<'a, TreeAndJoint> {
b7449926	37	let sm = self.sess.source_map();
32a655c1 SL	38	match self.token {
	39	token::Eof => {
	40	let msg = "this file contains an un-closed delimiter";
	41	let mut err = self.sess.span_diagnostic.struct_span_err(self.span, msg);
	42	for &(_, sp) in &self.open_braces {
b7449926 XL	43	err.span_label(sp, "un-closed delimiter");
	44	}
	45
	46	if let Some((delim, _)) = self.open_braces.last() {
	47	if let Some((_, open_sp, close_sp)) = self.matching_delim_spans.iter()
	48	.filter(\|(d, open_sp, close_sp)\| {
	49
	50	if let Some(close_padding) = sm.span_to_margin(*close_sp) {
	51	if let Some(open_padding) = sm.span_to_margin(*open_sp) {
	52	return delim == d && close_padding != open_padding;
	53	}
	54	}
	55	false
	56	}).next() // these are in reverse order as they get inserted on close, but
	57	{ // we want the last open/first close
	58	err.span_label(
	59	*open_sp,
	60	"this delimiter might not be properly closed...",
	61	);
	62	err.span_label(
	63	*close_sp,
	64	"...as it matches this but it has different indentation",
	65	);
	66	}
32a655c1 SL	67	}
	68	Err(err)
	69	},
	70	token::OpenDelim(delim) => {
	71	// The span for beginning of the delimited section
	72	let pre_span = self.span;
	73
	74	// Parse the open delimiter.
	75	self.open_braces.push((delim, self.span));
	76	self.real_token();
	77
	78	// Parse the token trees within the delimiters.
	79	// We stop at any delimiter so we can try to recover if the user
	80	// uses an incorrect delimiter.
	81	let tts = self.parse_token_trees_until_close_delim();
	82
	83	// Expand to cover the entire delimited token tree
b7449926	84	let delim_span = DelimSpan::from_pair(pre_span, self.span);
32a655c1 SL	85
	86	match self.token {
	87	// Correct delimiter.
	88	token::CloseDelim(d) if d == delim => {
b7449926	89	let (open_brace, open_brace_span) = self.open_braces.pop().unwrap();
0731742a XL	90	if self.open_braces.len() == 0 {
	91	// Clear up these spans to avoid suggesting them as we've found
	92	// properly matched delimiters so far for an entire block.
	93	self.matching_delim_spans.clear();
	94	} else {
	95	self.matching_delim_spans.push(
	96	(open_brace, open_brace_span, self.span),
	97	);
	98	}
32a655c1 SL	99	// Parse the close delimiter.
	100	self.real_token();
	101	}
	102	// Incorrect delimiter.
	103	token::CloseDelim(other) => {
9fa01778 XL	104	let mut unclosed_delimiter = None;
9fa01778 XL	105	let mut candidate = None;
b7449926 XL	106	if self.last_unclosed_found_span != Some(self.span) {
	107	// do not complain about the same unclosed delimiter multiple times
	108	self.last_unclosed_found_span = Some(self.span);
b7449926 XL	109	// This is a conservative error: only report the last unclosed
	110	// delimiter. The previous unclosed delimiters could actually be
	111	// closed! The parser just hasn't gotten to them yet.
	112	if let Some(&(_, sp)) = self.open_braces.last() {
9fa01778	113	unclosed_delimiter = Some(sp);
b7449926 XL	114	};
	115	if let Some(current_padding) = sm.span_to_margin(self.span) {
	116	for (brace, brace_span) in &self.open_braces {
	117	if let Some(padding) = sm.span_to_margin(*brace_span) {
	118	// high likelihood of these two corresponding
	119	if current_padding == padding && brace == &other {
9fa01778	120	candidate = Some(*brace_span);
b7449926 XL	121	}
	122	}
	123	}
	124	}
9fa01778 XL	125	let (tok, _) = self.open_braces.pop().unwrap();
	126	self.unmatched_braces.push(UnmatchedBrace {
	127	expected_delim: tok,
	128	found_delim: other,
	129	found_span: self.span,
	130	unclosed_span: unclosed_delimiter,
	131	candidate_span: candidate,
	132	});
	133	} else {
	134	self.open_braces.pop();
b7449926	135	}
32a655c1 SL	136
	137	// If the incorrect delimiter matches an earlier opening
	138	// delimiter, then don't consume it (it can be used to
	139	// close the earlier one). Otherwise, consume it.
	140	// E.g., we try to recover from:
	141	// fn foo() {
	142	// bar(baz(
	143	// } // Incorrect delimiter but matches the earlier `{`
	144	if !self.open_braces.iter().any(\|&(b, _)\| b == other) {
	145	self.real_token();
	146	}
	147	}
	148	token::Eof => {
	149	// Silently recover, the EOF token will be seen again
	150	// and an error emitted then. Thus we don't pop from
	151	// self.open_braces here.
	152	},
	153	_ => {}
	154	}
	155
0731742a XL	156	Ok(TokenTree::Delimited(
0731742a XL	157	delim_span,
3b2f2976	158	delim,
0731742a XL	159	tts.into()
0731742a XL	160	).into())
32a655c1 SL	161	},
	162	token::CloseDelim(_) => {
	163	// An unexpected closing delimiter (i.e., there is no
	164	// matching opening delimiter).
	165	let token_str = token_to_string(&self.token);
	166	let msg = format!("unexpected close delimiter: `{}`", token_str);
b7449926 XL	167	let mut err = self.sess.span_diagnostic.struct_span_err(self.span, &msg);
b7449926 XL	168	err.span_label(self.span, "unexpected close delimiter");
32a655c1 SL	169	Err(err)
	170	},
	171	_ => {
	172	let tt = TokenTree::Token(self.span, self.token.clone());
94b46f34 XL	173	// Note that testing for joint-ness here is done via the raw
	174	// source span as the joint-ness is a property of the raw source
	175	// rather than wanting to take `override_span` into account.
	176	let raw = self.span_src_raw;
32a655c1	177	self.real_token();
94b46f34	178	let is_joint = raw.hi() == self.span_src_raw.lo() && token::is_op(&self.token);
0731742a	179	Ok((tt, if is_joint { Joint } else { NonJoint }))
32a655c1 SL	180	}
	181	}
	182	}
	183	}