src/libsyntax/ext/tt/macro_rules.rs

   1 // Copyright 2015 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 use ast::{self, TokenTree};
  12 use codemap::{Span, DUMMY_SP};
  13 use ext::base::{DummyResult, ExtCtxt, MacResult, SyntaxExtension};
  14 use ext::base::{NormalTT, TTMacroExpander};
  15 use ext::tt::macro_parser::{Success, Error, Failure};
  16 use ext::tt::macro_parser::{MatchedSeq, MatchedNonterminal};
  17 use ext::tt::macro_parser::parse;
  18 use parse::lexer::new_tt_reader;
  19 use parse::parser::Parser;
  20 use parse::token::{self, special_idents, gensym_ident, NtTT, Token};
  21 use parse::token::Token::*;
  22 use print;
  23 use ptr::P;
  24
  25 use util::small_vector::SmallVector;
  26
  27 use std::cell::RefCell;
  28 use std::rc::Rc;
  29 use std::iter::once;
  30
  31 struct ParserAnyMacro<'a> {
  32     parser: RefCell<Parser<'a>>,
  33
  34     /// Span of the expansion site of the macro this parser is for
  35     site_span: Span,
  36     /// The ident of the macro we're parsing
  37     macro_ident: ast::Ident
  38 }
  39
  40 impl<'a> ParserAnyMacro<'a> {
  41     /// Make sure we don't have any tokens left to parse, so we don't
  42     /// silently drop anything. `allow_semi` is so that "optional"
  43     /// semicolons at the end of normal expressions aren't complained
  44     /// about e.g. the semicolon in `macro_rules! kapow { () => {
  45     /// panic!(); } }` doesn't get picked up by .parse_expr(), but it's
  46     /// allowed to be there.
  47     fn ensure_complete_parse(&self, allow_semi: bool, context: &str) {
  48         let mut parser = self.parser.borrow_mut();
  49         if allow_semi && parser.token == token::Semi {
  50             panictry!(parser.bump())
  51         }
  52         if parser.token != token::Eof {
  53             let token_str = parser.this_token_to_string();
  54             let msg = format!("macro expansion ignores token `{}` and any \
  55                                following",
  56                               token_str);
  57             let span = parser.span;
  58             parser.span_err(span, &msg[..]);
  59
  60             let msg = format!("caused by the macro expansion here; the usage \
  61                                of `{}!` is likely invalid in {} context",
  62                                self.macro_ident, context);
  63             parser.span_note(self.site_span, &msg[..]);
  64         }
  65     }
  66 }
  67
  68 impl<'a> MacResult for ParserAnyMacro<'a> {
  69     fn make_expr(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Expr>> {
  70         let ret = panictry!(self.parser.borrow_mut().parse_expr());
  71         self.ensure_complete_parse(true, "expression");
  72         Some(ret)
  73     }
  74     fn make_pat(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Pat>> {
  75         let ret = panictry!(self.parser.borrow_mut().parse_pat());
  76         self.ensure_complete_parse(false, "pattern");
  77         Some(ret)
  78     }
  79     fn make_items(self: Box<ParserAnyMacro<'a>>) -> Option<SmallVector<P<ast::Item>>> {
  80         let mut ret = SmallVector::zero();
  81         while let Some(item) = panictry!(self.parser.borrow_mut().parse_item()) {
  82             ret.push(item);
  83         }
  84         self.ensure_complete_parse(false, "item");
  85         Some(ret)
  86     }
  87
  88     fn make_impl_items(self: Box<ParserAnyMacro<'a>>)
  89                        -> Option<SmallVector<P<ast::ImplItem>>> {
  90         let mut ret = SmallVector::zero();
  91         loop {
  92             let mut parser = self.parser.borrow_mut();
  93             match parser.token {
  94                 token::Eof => break,
  95                 _ => ret.push(panictry!(parser.parse_impl_item()))
  96             }
  97         }
  98         self.ensure_complete_parse(false, "item");
  99         Some(ret)
 100     }
 101
 102     fn make_stmts(self: Box<ParserAnyMacro<'a>>)
 103                  -> Option<SmallVector<P<ast::Stmt>>> {
 104         let mut ret = SmallVector::zero();
 105         loop {
 106             let mut parser = self.parser.borrow_mut();
 107             match parser.token {
 108                 token::Eof => break,
 109                 _ => match parser.parse_stmt() {
 110                     Ok(maybe_stmt) => match maybe_stmt {
 111                         Some(stmt) => ret.push(stmt),
 112                         None => (),
 113                     },
 114                     Err(_) => break,
 115                 }
 116             }
 117         }
 118         self.ensure_complete_parse(false, "statement");
 119         Some(ret)
 120     }
 121
 122     fn make_ty(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Ty>> {
 123         let ret = panictry!(self.parser.borrow_mut().parse_ty());
 124         self.ensure_complete_parse(false, "type");
 125         Some(ret)
 126     }
 127 }
 128
 129 struct MacroRulesMacroExpander {
 130     name: ast::Ident,
 131     imported_from: Option<ast::Ident>,
 132     lhses: Vec<TokenTree>,
 133     rhses: Vec<TokenTree>,
 134     valid: bool,
 135 }
 136
 137 impl TTMacroExpander for MacroRulesMacroExpander {
 138     fn expand<'cx>(&self,
 139                    cx: &'cx mut ExtCtxt,
 140                    sp: Span,
 141                    arg: &[TokenTree])
 142                    -> Box<MacResult+'cx> {
 143         if !self.valid {
 144             return DummyResult::any(sp);
 145         }
 146         generic_extension(cx,
 147                           sp,
 148                           self.name,
 149                           self.imported_from,
 150                           arg,
 151                           &self.lhses,
 152                           &self.rhses)
 153     }
 154 }
 155
 156 /// Given `lhses` and `rhses`, this is the new macro we create
 157 fn generic_extension<'cx>(cx: &'cx ExtCtxt,
 158                           sp: Span,
 159                           name: ast::Ident,
 160                           imported_from: Option<ast::Ident>,
 161                           arg: &[TokenTree],
 162                           lhses: &[TokenTree],
 163                           rhses: &[TokenTree])
 164                           -> Box<MacResult+'cx> {
 165     if cx.trace_macros() {
 166         println!("{}! {{ {} }}",
 167                  name,
 168                  print::pprust::tts_to_string(arg));
 169     }
 170
 171     // Which arm's failure should we report? (the one furthest along)
 172     let mut best_fail_spot = DUMMY_SP;
 173     let mut best_fail_msg = "internal error: ran no matchers".to_string();
 174
 175     for (i, lhs) in lhses.iter().enumerate() { // try each arm's matchers
 176         let lhs_tt = match *lhs {
 177             TokenTree::Delimited(_, ref delim) => &delim.tts[..],
 178             _ => cx.span_fatal(sp, "malformed macro lhs")
 179         };
 180
 181         match TokenTree::parse(cx, lhs_tt, arg) {
 182             Success(named_matches) => {
 183                 let rhs = match rhses[i] {
 184                     // ignore delimiters
 185                     TokenTree::Delimited(_, ref delimed) => delimed.tts.clone(),
 186                     _ => cx.span_fatal(sp, "malformed macro rhs"),
 187                 };
 188                 // rhs has holes ( `$id` and `$(...)` that need filled)
 189                 let trncbr = new_tt_reader(&cx.parse_sess().span_diagnostic,
 190                                            Some(named_matches),
 191                                            imported_from,
 192                                            rhs);
 193                 let mut p = Parser::new(cx.parse_sess(), cx.cfg(), Box::new(trncbr));
 194                 panictry!(p.check_unknown_macro_variable());
 195                 // Let the context choose how to interpret the result.
 196                 // Weird, but useful for X-macros.
 197                 return Box::new(ParserAnyMacro {
 198                     parser: RefCell::new(p),
 199
 200                     // Pass along the original expansion site and the name of the macro
 201                     // so we can print a useful error message if the parse of the expanded
 202                     // macro leaves unparsed tokens.
 203                     site_span: sp,
 204                     macro_ident: name
 205                 })
 206             }
 207             Failure(sp, ref msg) => if sp.lo >= best_fail_spot.lo {
 208                 best_fail_spot = sp;
 209                 best_fail_msg = (*msg).clone();
 210             },
 211             Error(err_sp, ref msg) => {
 212                 cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..])
 213             }
 214         }
 215     }
 216
 217      cx.span_fatal(best_fail_spot.substitute_dummy(sp), &best_fail_msg[..]);
 218 }
 219
 220 // Note that macro-by-example's input is also matched against a token tree:
 221 //                   $( $lhs:tt => $rhs:tt );+
 222 //
 223 // Holy self-referential!
 224
 225 /// Converts a `macro_rules!` invocation into a syntax extension.
 226 pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
 227                     def: &ast::MacroDef) -> SyntaxExtension {
 228
 229     let lhs_nm =  gensym_ident("lhs");
 230     let rhs_nm =  gensym_ident("rhs");
 231
 232     // The pattern that macro_rules matches.
 233     // The grammar for macro_rules! is:
 234     // $( $lhs:tt => $rhs:tt );+
 235     // ...quasiquoting this would be nice.
 236     // These spans won't matter, anyways
 237     let match_lhs_tok = MatchNt(lhs_nm, special_idents::tt, token::Plain, token::Plain);
 238     let match_rhs_tok = MatchNt(rhs_nm, special_idents::tt, token::Plain, token::Plain);
 239     let argument_gram = vec!(
 240         TokenTree::Sequence(DUMMY_SP,
 241                    Rc::new(ast::SequenceRepetition {
 242                        tts: vec![
 243                            TokenTree::Token(DUMMY_SP, match_lhs_tok),
 244                            TokenTree::Token(DUMMY_SP, token::FatArrow),
 245                            TokenTree::Token(DUMMY_SP, match_rhs_tok)],
 246                        separator: Some(token::Semi),
 247                        op: ast::OneOrMore,
 248                        num_captures: 2
 249                    })),
 250         //to phase into semicolon-termination instead of
 251         //semicolon-separation
 252         TokenTree::Sequence(DUMMY_SP,
 253                    Rc::new(ast::SequenceRepetition {
 254                        tts: vec![TokenTree::Token(DUMMY_SP, token::Semi)],
 255                        separator: None,
 256                        op: ast::ZeroOrMore,
 257                        num_captures: 0
 258                    })));
 259
 260
 261     // Parse the macro_rules! invocation (`none` is for no interpolations):
 262     let arg_reader = new_tt_reader(&cx.parse_sess().span_diagnostic,
 263                                    None,
 264                                    None,
 265                                    def.body.clone());
 266
 267     let argument_map = match parse(cx.parse_sess(),
 268                                    cx.cfg(),
 269                                    arg_reader,
 270                                    &argument_gram) {
 271         Success(m) => m,
 272         Failure(sp, str) | Error(sp, str) => {
 273             panic!(cx.parse_sess().span_diagnostic
 274                      .span_fatal(sp.substitute_dummy(def.span), &str[..]));
 275         }
 276     };
 277
 278     let mut valid = true;
 279
 280     // Extract the arguments:
 281     let lhses = match **argument_map.get(&lhs_nm.name).unwrap() {
 282         MatchedSeq(ref s, _) => {
 283             s.iter().map(|m| match **m {
 284                 MatchedNonterminal(NtTT(ref tt)) => (**tt).clone(),
 285                 _ => cx.span_bug(def.span, "wrong-structured lhs")
 286             }).collect()
 287         }
 288         _ => cx.span_bug(def.span, "wrong-structured lhs")
 289     };
 290
 291     for lhs in &lhses {
 292         check_lhs_nt_follows(cx, lhs, def.span);
 293     }
 294
 295     let rhses = match **argument_map.get(&rhs_nm.name).unwrap() {
 296         MatchedSeq(ref s, _) => {
 297             s.iter().map(|m| match **m {
 298                 MatchedNonterminal(NtTT(ref tt)) => (**tt).clone(),
 299                 _ => cx.span_bug(def.span, "wrong-structured rhs")
 300             }).collect()
 301         }
 302         _ => cx.span_bug(def.span, "wrong-structured rhs")
 303     };
 304
 305     for rhs in &rhses {
 306         valid &= check_rhs(cx, rhs);
 307     }
 308
 309     let exp: Box<_> = Box::new(MacroRulesMacroExpander {
 310         name: def.ident,
 311         imported_from: def.imported_from,
 312         lhses: lhses,
 313         rhses: rhses,
 314         valid: valid,
 315     });
 316
 317     NormalTT(exp, Some(def.span), def.allow_internal_unstable)
 318 }
 319
 320 fn check_lhs_nt_follows(cx: &mut ExtCtxt, lhs: &TokenTree, sp: Span) {
 321     // lhs is going to be like TokenTree::Delimited(...), where the
 322     // entire lhs is those tts. Or, it can be a "bare sequence", not wrapped in parens.
 323     match lhs {
 324         &TokenTree::Delimited(_, ref tts) => {
 325             check_matcher(cx, tts.tts.iter(), &Eof);
 326         },
 327         tt @ &TokenTree::Sequence(..) => {
 328             check_matcher(cx, Some(tt).into_iter(), &Eof);
 329         },
 330         _ => cx.span_err(sp, "invalid macro matcher; matchers must be contained \
 331                               in balanced delimiters or a repetition indicator")
 332     };
 333     // we don't abort on errors on rejection, the driver will do that for us
 334     // after parsing/expansion. we can report every error in every macro this way.
 335 }
 336
 337 fn check_rhs(cx: &mut ExtCtxt, rhs: &TokenTree) -> bool {
 338     match *rhs {
 339         TokenTree::Delimited(..) => return true,
 340         _ => cx.span_err(rhs.get_span(), "macro rhs must be delimited")
 341     }
 342     false
 343 }
 344
 345 // returns the last token that was checked, for TokenTree::Sequence. this gets used later on.
 346 fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
 347 -> Option<(Span, Token)> where I: Iterator<Item=&'a TokenTree> {
 348     use print::pprust::token_to_string;
 349
 350     let mut last = None;
 351
 352     // 2. For each token T in M:
 353     let mut tokens = matcher.peekable();
 354     while let Some(token) = tokens.next() {
 355         last = match *token {
 356             TokenTree::Token(sp, MatchNt(ref name, ref frag_spec, _, _)) => {
 357                 // ii. If T is a simple NT, look ahead to the next token T' in
 358                 // M. If T' is in the set FOLLOW(NT), continue. Else; reject.
 359                 if can_be_followed_by_any(&frag_spec.name.as_str()) {
 360                     continue
 361                 } else {
 362                     let next_token = match tokens.peek() {
 363                         // If T' closes a complex NT, replace T' with F
 364                         Some(&&TokenTree::Token(_, CloseDelim(_))) => follow.clone(),
 365                         Some(&&TokenTree::Token(_, ref tok)) => tok.clone(),
 366                         Some(&&TokenTree::Sequence(sp, _)) => {
 367                             // Be conservative around sequences: to be
 368                             // more specific, we would need to
 369                             // consider FIRST sets, but also the
 370                             // possibility that the sequence occurred
 371                             // zero times (in which case we need to
 372                             // look at the token that follows the
 373                             // sequence, which may itself be a sequence,
 374                             // and so on).
 375                             cx.span_err(sp,
 376                                         &format!("`${0}:{1}` is followed by a \
 377                                                   sequence repetition, which is not \
 378                                                   allowed for `{1}` fragments",
 379                                                  name, frag_spec)
 380                                         );
 381                             Eof
 382                         },
 383                         // die next iteration
 384                         Some(&&TokenTree::Delimited(_, ref delim)) => delim.close_token(),
 385                         // else, we're at the end of the macro or sequence
 386                         None => follow.clone()
 387                     };
 388
 389                     let tok = if let TokenTree::Token(_, ref tok) = *token {
 390                         tok
 391                     } else {
 392                         unreachable!()
 393                     };
 394
 395                     // If T' is in the set FOLLOW(NT), continue. Else, reject.
 396                     match (&next_token, is_in_follow(cx, &next_token, &frag_spec.name.as_str())) {
 397                         (_, Err(msg)) => {
 398                             cx.span_err(sp, &msg);
 399                             continue
 400                         }
 401                         (&Eof, _) => return Some((sp, tok.clone())),
 402                         (_, Ok(true)) => continue,
 403                         (next, Ok(false)) => {
 404                             cx.span_err(sp, &format!("`${0}:{1}` is followed by `{2}`, which \
 405                                                       is not allowed for `{1}` fragments",
 406                                                      name, frag_spec,
 407                                                      token_to_string(next)));
 408                             continue
 409                         },
 410                     }
 411                 }
 412             },
 413             TokenTree::Sequence(sp, ref seq) => {
 414                 // iii. Else, T is a complex NT.
 415                 match seq.separator {
 416                     // If T has the form $(...)U+ or $(...)U* for some token U,
 417                     // run the algorithm on the contents with F set to U. If it
 418                     // accepts, continue, else, reject.
 419                     Some(ref u) => {
 420                         let last = check_matcher(cx, seq.tts.iter(), u);
 421                         match last {
 422                             // Since the delimiter isn't required after the last
 423                             // repetition, make sure that the *next* token is
 424                             // sane. This doesn't actually compute the FIRST of
 425                             // the rest of the matcher yet, it only considers
 426                             // single tokens and simple NTs. This is imprecise,
 427                             // but conservatively correct.
 428                             Some((span, tok)) => {
 429                                 let fol = match tokens.peek() {
 430                                     Some(&&TokenTree::Token(_, ref tok)) => tok.clone(),
 431                                     Some(&&TokenTree::Delimited(_, ref delim)) =>
 432                                         delim.close_token(),
 433                                     Some(_) => {
 434                                         cx.span_err(sp, "sequence repetition followed by \
 435                                                 another sequence repetition, which is not allowed");
 436                                         Eof
 437                                     },
 438                                     None => Eof
 439                                 };
 440                                 check_matcher(cx, once(&TokenTree::Token(span, tok.clone())),
 441                                               &fol)
 442                             },
 443                             None => last,
 444                         }
 445                     },
 446                     // If T has the form $(...)+ or $(...)*, run the algorithm
 447                     // on the contents with F set to the token following the
 448                     // sequence. If it accepts, continue, else, reject.
 449                     None => {
 450                         let fol = match tokens.peek() {
 451                             Some(&&TokenTree::Token(_, ref tok)) => tok.clone(),
 452                             Some(&&TokenTree::Delimited(_, ref delim)) => delim.close_token(),
 453                             Some(_) => {
 454                                 cx.span_err(sp, "sequence repetition followed by another \
 455                                              sequence repetition, which is not allowed");
 456                                 Eof
 457                             },
 458                             None => Eof
 459                         };
 460                         check_matcher(cx, seq.tts.iter(), &fol)
 461                     }
 462                 }
 463             },
 464             TokenTree::Token(..) => {
 465                 // i. If T is not an NT, continue.
 466                 continue
 467             },
 468             TokenTree::Delimited(_, ref tts) => {
 469                 // if we don't pass in that close delimiter, we'll incorrectly consider the matcher
 470                 // `{ $foo:ty }` as having a follow that isn't `RBrace`
 471                 check_matcher(cx, tts.tts.iter(), &tts.close_token())
 472             }
 473         }
 474     }
 475     last
 476 }
 477
 478 /// True if a fragment of type `frag` can be followed by any sort of
 479 /// token.  We use this (among other things) as a useful approximation
 480 /// for when `frag` can be followed by a repetition like `$(...)*` or
 481 /// `$(...)+`. In general, these can be a bit tricky to reason about,
 482 /// so we adopt a conservative position that says that any fragment
 483 /// specifier which consumes at most one token tree can be followed by
 484 /// a fragment specifier (indeed, these fragments can be followed by
 485 /// ANYTHING without fear of future compatibility hazards).
 486 fn can_be_followed_by_any(frag: &str) -> bool {
 487     match frag {
 488         "item" |  // always terminated by `}` or `;`
 489         "block" | // exactly one token tree
 490         "ident" | // exactly one token tree
 491         "meta" |  // exactly one token tree
 492         "tt" =>    // exactly one token tree
 493             true,
 494
 495         _ =>
 496             false,
 497     }
 498 }
 499
 500 /// True if `frag` can legally be followed by the token `tok`. For
 501 /// fragments that can consume an unbounded numbe of tokens, `tok`
 502 /// must be within a well-defined follow set. This is intended to
 503 /// guarantee future compatibility: for example, without this rule, if
 504 /// we expanded `expr` to include a new binary operator, we might
 505 /// break macros that were relying on that binary operator as a
 506 /// separator.
 507 fn is_in_follow(_: &ExtCtxt, tok: &Token, frag: &str) -> Result<bool, String> {
 508     if let &CloseDelim(_) = tok {
 509         // closing a token tree can never be matched by any fragment;
 510         // iow, we always require that `(` and `)` match, etc.
 511         Ok(true)
 512     } else {
 513         match frag {
 514             "item" => {
 515                 // since items *must* be followed by either a `;` or a `}`, we can
 516                 // accept anything after them
 517                 Ok(true)
 518             },
 519             "block" => {
 520                 // anything can follow block, the braces provide an easy boundary to
 521                 // maintain
 522                 Ok(true)
 523             },
 524             "stmt" | "expr"  => {
 525                 match *tok {
 526                     FatArrow | Comma | Semi => Ok(true),
 527                     _ => Ok(false)
 528                 }
 529             },
 530             "pat" => {
 531                 match *tok {
 532                     FatArrow | Comma | Eq => Ok(true),
 533                     Ident(i, _) if i.name.as_str() == "if" || i.name.as_str() == "in" => Ok(true),
 534                     _ => Ok(false)
 535                 }
 536             },
 537             "path" | "ty" => {
 538                 match *tok {
 539                     Comma | FatArrow | Colon | Eq | Gt | Semi => Ok(true),
 540                     Ident(i, _) if i.name.as_str() == "as" => Ok(true),
 541                     _ => Ok(false)
 542                 }
 543             },
 544             "ident" => {
 545                 // being a single token, idents are harmless
 546                 Ok(true)
 547             },
 548             "meta" | "tt" => {
 549                 // being either a single token or a delimited sequence, tt is
 550                 // harmless
 551                 Ok(true)
 552             },
 553             _ => Err(format!("invalid fragment specifier `{}`", frag))
 554         }
 555     }
 556 }