src/libsyntax/ext/tt/macro_rules.rs

   1 // Copyright 2015 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 use ast::{self, TokenTree, TtDelimited, TtSequence, TtToken};
  12 use codemap::{Span, DUMMY_SP};
  13 use ext::base::{ExtCtxt, MacResult, SyntaxExtension};
  14 use ext::base::{NormalTT, TTMacroExpander};
  15 use ext::tt::macro_parser::{Success, Error, Failure};
  16 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
  17 use ext::tt::macro_parser::parse;
  18 use parse::lexer::new_tt_reader;
  19 use parse::parser::Parser;
  20 use parse::token::{self, special_idents, gensym_ident, NtTT, Token};
  21 use parse::token::Token::*;
  22 use print;
  23 use ptr::P;
  24
  25 use util::small_vector::SmallVector;
  26
  27 use std::cell::RefCell;
  28 use std::rc::Rc;
  29
  30 struct ParserAnyMacro<'a> {
  31     parser: RefCell<Parser<'a>>,
  32
  33     /// Span of the expansion site of the macro this parser is for
  34     site_span: Span,
  35     /// The ident of the macro we're parsing
  36     macro_ident: ast::Ident
  37 }
  38
  39 impl<'a> ParserAnyMacro<'a> {
  40     /// Make sure we don't have any tokens left to parse, so we don't
  41     /// silently drop anything. `allow_semi` is so that "optional"
  42     /// semicolons at the end of normal expressions aren't complained
  43     /// about e.g. the semicolon in `macro_rules! kapow { () => {
  44     /// panic!(); } }` doesn't get picked up by .parse_expr(), but it's
  45     /// allowed to be there.
  46     fn ensure_complete_parse(&self, allow_semi: bool) {
  47         let mut parser = self.parser.borrow_mut();
  48         if allow_semi && parser.token == token::Semi {
  49             panictry!(parser.bump())
  50         }
  51         if parser.token != token::Eof {
  52             let token_str = parser.this_token_to_string();
  53             let msg = format!("macro expansion ignores token `{}` and any \
  54                                following",
  55                               token_str);
  56             let span = parser.span;
  57             parser.span_err(span, &msg[..]);
  58
  59             let msg = format!("caused by the macro expansion here; the usage \
  60                                of `{}` is likely invalid in this context",
  61                                self.macro_ident);
  62             parser.span_note(self.site_span, &msg[..]);
  63         }
  64     }
  65 }
  66
  67 impl<'a> MacResult for ParserAnyMacro<'a> {
  68     fn make_expr(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Expr>> {
  69         let ret = self.parser.borrow_mut().parse_expr();
  70         self.ensure_complete_parse(true);
  71         Some(ret)
  72     }
  73     fn make_pat(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Pat>> {
  74         let ret = self.parser.borrow_mut().parse_pat();
  75         self.ensure_complete_parse(false);
  76         Some(ret)
  77     }
  78     fn make_items(self: Box<ParserAnyMacro<'a>>) -> Option<SmallVector<P<ast::Item>>> {
  79         let mut ret = SmallVector::zero();
  80         while let Some(item) = self.parser.borrow_mut().parse_item() {
  81             ret.push(item);
  82         }
  83         self.ensure_complete_parse(false);
  84         Some(ret)
  85     }
  86
  87     fn make_impl_items(self: Box<ParserAnyMacro<'a>>)
  88                        -> Option<SmallVector<P<ast::ImplItem>>> {
  89         let mut ret = SmallVector::zero();
  90         loop {
  91             let mut parser = self.parser.borrow_mut();
  92             match parser.token {
  93                 token::Eof => break,
  94                 _ => ret.push(panictry!(parser.parse_impl_item()))
  95             }
  96         }
  97         self.ensure_complete_parse(false);
  98         Some(ret)
  99     }
 100
 101     fn make_stmts(self: Box<ParserAnyMacro<'a>>)
 102                  -> Option<SmallVector<P<ast::Stmt>>> {
 103         let mut ret = SmallVector::zero();
 104         loop {
 105             let mut parser = self.parser.borrow_mut();
 106             match parser.token {
 107                 token::Eof => break,
 108                 _ => match parser.parse_stmt_nopanic() {
 109                     Ok(maybe_stmt) => match maybe_stmt {
 110                         Some(stmt) => ret.push(stmt),
 111                         None => (),
 112                     },
 113                     Err(_) => break,
 114                 }
 115             }
 116         }
 117         self.ensure_complete_parse(false);
 118         Some(ret)
 119     }
 120
 121     fn make_ty(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Ty>> {
 122         let ret = self.parser.borrow_mut().parse_ty();
 123         self.ensure_complete_parse(true);
 124         Some(ret)
 125     }
 126 }
 127
 128 struct MacroRulesMacroExpander {
 129     name: ast::Ident,
 130     imported_from: Option<ast::Ident>,
 131     lhses: Vec<Rc<NamedMatch>>,
 132     rhses: Vec<Rc<NamedMatch>>,
 133 }
 134
 135 impl TTMacroExpander for MacroRulesMacroExpander {
 136     fn expand<'cx>(&self,
 137                    cx: &'cx mut ExtCtxt,
 138                    sp: Span,
 139                    arg: &[ast::TokenTree])
 140                    -> Box<MacResult+'cx> {
 141         generic_extension(cx,
 142                           sp,
 143                           self.name,
 144                           self.imported_from,
 145                           arg,
 146                           &self.lhses,
 147                           &self.rhses)
 148     }
 149 }
 150
 151 /// Given `lhses` and `rhses`, this is the new macro we create
 152 fn generic_extension<'cx>(cx: &'cx ExtCtxt,
 153                           sp: Span,
 154                           name: ast::Ident,
 155                           imported_from: Option<ast::Ident>,
 156                           arg: &[ast::TokenTree],
 157                           lhses: &[Rc<NamedMatch>],
 158                           rhses: &[Rc<NamedMatch>])
 159                           -> Box<MacResult+'cx> {
 160     if cx.trace_macros() {
 161         println!("{}! {{ {} }}",
 162                  name,
 163                  print::pprust::tts_to_string(arg));
 164     }
 165
 166     // Which arm's failure should we report? (the one furthest along)
 167     let mut best_fail_spot = DUMMY_SP;
 168     let mut best_fail_msg = "internal error: ran no matchers".to_string();
 169
 170     for (i, lhs) in lhses.iter().enumerate() { // try each arm's matchers
 171         match **lhs {
 172           MatchedNonterminal(NtTT(ref lhs_tt)) => {
 173             let lhs_tt = match **lhs_tt {
 174                 TtDelimited(_, ref delim) => &delim.tts[..],
 175                 _ => panic!(cx.span_fatal(sp, "malformed macro lhs"))
 176             };
 177
 178             match TokenTree::parse(cx, lhs_tt, arg) {
 179               Success(named_matches) => {
 180                 let rhs = match *rhses[i] {
 181                     // okay, what's your transcriber?
 182                     MatchedNonterminal(NtTT(ref tt)) => {
 183                         match **tt {
 184                             // ignore delimiters
 185                             TtDelimited(_, ref delimed) => delimed.tts.clone(),
 186                             _ => panic!(cx.span_fatal(sp, "macro rhs must be delimited")),
 187                         }
 188                     },
 189                     _ => cx.span_bug(sp, "bad thing in rhs")
 190                 };
 191                 // rhs has holes ( `$id` and `$(...)` that need filled)
 192                 let trncbr = new_tt_reader(&cx.parse_sess().span_diagnostic,
 193                                            Some(named_matches),
 194                                            imported_from,
 195                                            rhs);
 196                 let mut p = Parser::new(cx.parse_sess(), cx.cfg(), Box::new(trncbr));
 197                 panictry!(p.check_unknown_macro_variable());
 198                 // Let the context choose how to interpret the result.
 199                 // Weird, but useful for X-macros.
 200                 return Box::new(ParserAnyMacro {
 201                     parser: RefCell::new(p),
 202
 203                     // Pass along the original expansion site and the name of the macro
 204                     // so we can print a useful error message if the parse of the expanded
 205                     // macro leaves unparsed tokens.
 206                     site_span: sp,
 207                     macro_ident: name
 208                 })
 209               }
 210               Failure(sp, ref msg) => if sp.lo >= best_fail_spot.lo {
 211                 best_fail_spot = sp;
 212                 best_fail_msg = (*msg).clone();
 213               },
 214               Error(err_sp, ref msg) => {
 215                 panic!(cx.span_fatal(err_sp.substitute_dummy(sp), &msg[..]))
 216               }
 217             }
 218           }
 219           _ => cx.bug("non-matcher found in parsed lhses")
 220         }
 221     }
 222
 223     panic!(cx.span_fatal(best_fail_spot.substitute_dummy(sp), &best_fail_msg[..]));
 224 }
 225
 226 // Note that macro-by-example's input is also matched against a token tree:
 227 //                   $( $lhs:tt => $rhs:tt );+
 228 //
 229 // Holy self-referential!
 230
 231 /// Converts a `macro_rules!` invocation into a syntax extension.
 232 pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
 233                     def: &ast::MacroDef) -> SyntaxExtension {
 234
 235     let lhs_nm =  gensym_ident("lhs");
 236     let rhs_nm =  gensym_ident("rhs");
 237
 238     // The pattern that macro_rules matches.
 239     // The grammar for macro_rules! is:
 240     // $( $lhs:tt => $rhs:tt );+
 241     // ...quasiquoting this would be nice.
 242     // These spans won't matter, anyways
 243     let match_lhs_tok = MatchNt(lhs_nm, special_idents::tt, token::Plain, token::Plain);
 244     let match_rhs_tok = MatchNt(rhs_nm, special_idents::tt, token::Plain, token::Plain);
 245     let argument_gram = vec!(
 246         TtSequence(DUMMY_SP,
 247                    Rc::new(ast::SequenceRepetition {
 248                        tts: vec![
 249                            TtToken(DUMMY_SP, match_lhs_tok),
 250                            TtToken(DUMMY_SP, token::FatArrow),
 251                            TtToken(DUMMY_SP, match_rhs_tok)],
 252                        separator: Some(token::Semi),
 253                        op: ast::OneOrMore,
 254                        num_captures: 2
 255                    })),
 256         //to phase into semicolon-termination instead of
 257         //semicolon-separation
 258         TtSequence(DUMMY_SP,
 259                    Rc::new(ast::SequenceRepetition {
 260                        tts: vec![TtToken(DUMMY_SP, token::Semi)],
 261                        separator: None,
 262                        op: ast::ZeroOrMore,
 263                        num_captures: 0
 264                    })));
 265
 266
 267     // Parse the macro_rules! invocation (`none` is for no interpolations):
 268     let arg_reader = new_tt_reader(&cx.parse_sess().span_diagnostic,
 269                                    None,
 270                                    None,
 271                                    def.body.clone());
 272
 273     let argument_map = match parse(cx.parse_sess(),
 274                                    cx.cfg(),
 275                                    arg_reader,
 276                                    &argument_gram) {
 277         Success(m) => m,
 278         Failure(sp, str) | Error(sp, str) => {
 279             panic!(cx.parse_sess().span_diagnostic
 280                      .span_fatal(sp.substitute_dummy(def.span), &str[..]));
 281         }
 282     };
 283
 284     // Extract the arguments:
 285     let lhses = match **argument_map.get(&lhs_nm.name).unwrap() {
 286         MatchedSeq(ref s, _) => /* FIXME (#2543) */ (*s).clone(),
 287         _ => cx.span_bug(def.span, "wrong-structured lhs")
 288     };
 289
 290     for lhs in &lhses {
 291         check_lhs_nt_follows(cx, &**lhs, def.span);
 292     }
 293
 294     let rhses = match **argument_map.get(&rhs_nm.name).unwrap() {
 295         MatchedSeq(ref s, _) => /* FIXME (#2543) */ (*s).clone(),
 296         _ => cx.span_bug(def.span, "wrong-structured rhs")
 297     };
 298
 299     let exp: Box<_> = Box::new(MacroRulesMacroExpander {
 300         name: def.ident,
 301         imported_from: def.imported_from,
 302         lhses: lhses,
 303         rhses: rhses,
 304     });
 305
 306     NormalTT(exp, Some(def.span), def.allow_internal_unstable)
 307 }
 308
 309 fn check_lhs_nt_follows(cx: &mut ExtCtxt, lhs: &NamedMatch, sp: Span) {
 310     // lhs is going to be like MatchedNonterminal(NtTT(TtDelimited(...))), where the entire lhs is
 311     // those tts. Or, it can be a "bare sequence", not wrapped in parens.
 312     match lhs {
 313         &MatchedNonterminal(NtTT(ref inner)) => match &**inner {
 314             &TtDelimited(_, ref tts) => {
 315                 check_matcher(cx, tts.tts.iter(), &Eof);
 316             },
 317             tt @ &TtSequence(..) => {
 318                 check_matcher(cx, Some(tt).into_iter(), &Eof);
 319             },
 320             _ => cx.span_err(sp, "Invalid macro matcher; matchers must be contained \
 321                in balanced delimiters or a repetition indicator")
 322         },
 323         _ => cx.span_bug(sp, "wrong-structured lhs for follow check (didn't find a \
 324            MatchedNonterminal)")
 325     };
 326     // we don't abort on errors on rejection, the driver will do that for us
 327     // after parsing/expansion. we can report every error in every macro this way.
 328 }
 329
 330 // returns the last token that was checked, for TtSequence. this gets used later on.
 331 fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
 332 -> Option<(Span, Token)> where I: Iterator<Item=&'a TokenTree> {
 333     use print::pprust::token_to_string;
 334
 335     let mut last = None;
 336
 337     // 2. For each token T in M:
 338     let mut tokens = matcher.peekable();
 339     while let Some(token) = tokens.next() {
 340         last = match *token {
 341             TtToken(sp, MatchNt(ref name, ref frag_spec, _, _)) => {
 342                 // ii. If T is a simple NT, look ahead to the next token T' in
 343                 // M. If T' is in the set FOLLOW(NT), continue. Else; reject.
 344                 if can_be_followed_by_any(&frag_spec.name.as_str()) {
 345                     continue
 346                 } else {
 347                     let next_token = match tokens.peek() {
 348                         // If T' closes a complex NT, replace T' with F
 349                         Some(&&TtToken(_, CloseDelim(_))) => follow.clone(),
 350                         Some(&&TtToken(_, ref tok)) => tok.clone(),
 351                         Some(&&TtSequence(sp, _)) => {
 352                             // Be conservative around sequences: to be
 353                             // more specific, we would need to
 354                             // consider FIRST sets, but also the
 355                             // possibility that the sequence occurred
 356                             // zero times (in which case we need to
 357                             // look at the token that follows the
 358                             // sequence, which may itself be a sequence,
 359                             // and so on).
 360                             cx.span_err(sp,
 361                                         &format!("`${0}:{1}` is followed by a \
 362                                                   sequence repetition, which is not \
 363                                                   allowed for `{1}` fragments",
 364                                                  name, frag_spec)
 365                                         );
 366                             Eof
 367                         },
 368                         // die next iteration
 369                         Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
 370                         // else, we're at the end of the macro or sequence
 371                         None => follow.clone()
 372                     };
 373
 374                     let tok = if let TtToken(_, ref tok) = *token { tok } else { unreachable!() };
 375
 376                     // If T' is in the set FOLLOW(NT), continue. Else, reject.
 377                     match (&next_token, is_in_follow(cx, &next_token, &frag_spec.name.as_str())) {
 378                         (_, Err(msg)) => {
 379                             cx.span_err(sp, &msg);
 380                             continue
 381                         }
 382                         (&Eof, _) => return Some((sp, tok.clone())),
 383                         (_, Ok(true)) => continue,
 384                         (next, Ok(false)) => {
 385                             cx.span_err(sp, &format!("`${0}:{1}` is followed by `{2}`, which \
 386                                                       is not allowed for `{1}` fragments",
 387                                                      name, frag_spec,
 388                                                      token_to_string(next)));
 389                             continue
 390                         },
 391                     }
 392                 }
 393             },
 394             TtSequence(sp, ref seq) => {
 395                 // iii. Else, T is a complex NT.
 396                 match seq.separator {
 397                     // If T has the form $(...)U+ or $(...)U* for some token U,
 398                     // run the algorithm on the contents with F set to U. If it
 399                     // accepts, continue, else, reject.
 400                     Some(ref u) => {
 401                         let last = check_matcher(cx, seq.tts.iter(), u);
 402                         match last {
 403                             // Since the delimiter isn't required after the last
 404                             // repetition, make sure that the *next* token is
 405                             // sane. This doesn't actually compute the FIRST of
 406                             // the rest of the matcher yet, it only considers
 407                             // single tokens and simple NTs. This is imprecise,
 408                             // but conservatively correct.
 409                             Some((span, tok)) => {
 410                                 let fol = match tokens.peek() {
 411                                     Some(&&TtToken(_, ref tok)) => tok.clone(),
 412                                     Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
 413                                     Some(_) => {
 414                                         cx.span_err(sp, "sequence repetition followed by \
 415                                                 another sequence repetition, which is not allowed");
 416                                         Eof
 417                                     },
 418                                     None => Eof
 419                                 };
 420                                 check_matcher(cx, Some(&TtToken(span, tok.clone())).into_iter(),
 421                                               &fol)
 422                             },
 423                             None => last,
 424                         }
 425                     },
 426                     // If T has the form $(...)+ or $(...)*, run the algorithm
 427                     // on the contents with F set to the token following the
 428                     // sequence. If it accepts, continue, else, reject.
 429                     None => {
 430                         let fol = match tokens.peek() {
 431                             Some(&&TtToken(_, ref tok)) => tok.clone(),
 432                             Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
 433                             Some(_) => {
 434                                 cx.span_err(sp, "sequence repetition followed by another \
 435                                              sequence repetition, which is not allowed");
 436                                 Eof
 437                             },
 438                             None => Eof
 439                         };
 440                         check_matcher(cx, seq.tts.iter(), &fol)
 441                     }
 442                 }
 443             },
 444             TtToken(..) => {
 445                 // i. If T is not an NT, continue.
 446                 continue
 447             },
 448             TtDelimited(_, ref tts) => {
 449                 // if we don't pass in that close delimiter, we'll incorrectly consider the matcher
 450                 // `{ $foo:ty }` as having a follow that isn't `RBrace`
 451                 check_matcher(cx, tts.tts.iter(), &tts.close_token())
 452             }
 453         }
 454     }
 455     last
 456 }
 457
 458 /// True if a fragment of type `frag` can be followed by any sort of
 459 /// token.  We use this (among other things) as a useful approximation
 460 /// for when `frag` can be followed by a repetition like `$(...)*` or
 461 /// `$(...)+`. In general, these can be a bit tricky to reason about,
 462 /// so we adopt a conservative position that says that any fragment
 463 /// specifier which consumes at most one token tree can be followed by
 464 /// a fragment specifier (indeed, these fragments can be followed by
 465 /// ANYTHING without fear of future compatibility hazards).
 466 fn can_be_followed_by_any(frag: &str) -> bool {
 467     match frag {
 468         "item" |  // always terminated by `}` or `;`
 469         "block" | // exactly one token tree
 470         "ident" | // exactly one token tree
 471         "meta" |  // exactly one token tree
 472         "tt" =>    // exactly one token tree
 473             true,
 474
 475         _ =>
 476             false,
 477     }
 478 }
 479
 480 /// True if `frag` can legally be followed by the token `tok`. For
 481 /// fragments that can consume an unbounded numbe of tokens, `tok`
 482 /// must be within a well-defined follow set. This is intended to
 483 /// guarantee future compatibility: for example, without this rule, if
 484 /// we expanded `expr` to include a new binary operator, we might
 485 /// break macros that were relying on that binary operator as a
 486 /// separator.
 487 fn is_in_follow(_: &ExtCtxt, tok: &Token, frag: &str) -> Result<bool, String> {
 488     if let &CloseDelim(_) = tok {
 489         // closing a token tree can never be matched by any fragment;
 490         // iow, we always require that `(` and `)` match, etc.
 491         Ok(true)
 492     } else {
 493         match frag {
 494             "item" => {
 495                 // since items *must* be followed by either a `;` or a `}`, we can
 496                 // accept anything after them
 497                 Ok(true)
 498             },
 499             "block" => {
 500                 // anything can follow block, the braces provide an easy boundary to
 501                 // maintain
 502                 Ok(true)
 503             },
 504             "stmt" | "expr"  => {
 505                 match *tok {
 506                     FatArrow | Comma | Semi => Ok(true),
 507                     _ => Ok(false)
 508                 }
 509             },
 510             "pat" => {
 511                 match *tok {
 512                     FatArrow | Comma | Eq => Ok(true),
 513                     Ident(i, _) if i.name.as_str() == "if" || i.name.as_str() == "in" => Ok(true),
 514                     _ => Ok(false)
 515                 }
 516             },
 517             "path" | "ty" => {
 518                 match *tok {
 519                     Comma | FatArrow | Colon | Eq | Gt | Semi => Ok(true),
 520                     Ident(i, _) if i.name.as_str() == "as" => Ok(true),
 521                     _ => Ok(false)
 522                 }
 523             },
 524             "ident" => {
 525                 // being a single token, idents are harmless
 526                 Ok(true)
 527             },
 528             "meta" | "tt" => {
 529                 // being either a single token or a delimited sequence, tt is
 530                 // harmless
 531                 Ok(true)
 532             },
 533             _ => Err(format!("invalid fragment specifier `{}`", frag))
 534         }
 535     }
 536 }