src/libsyntax/ext/tt/macro_rules.rs

   1 // Copyright 2015 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 use ast::{self, TokenTree, TtDelimited, TtSequence, TtToken};
  12 use codemap::{Span, DUMMY_SP};
  13 use ext::base::{ExtCtxt, MacResult, SyntaxExtension};
  14 use ext::base::{NormalTT, TTMacroExpander};
  15 use ext::tt::macro_parser::{Success, Error, Failure};
  16 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
  17 use ext::tt::macro_parser::{parse, parse_or_else};
  18 use parse::lexer::{new_tt_reader, new_tt_reader_with_doc_flag};
  19 use parse::parser::Parser;
  20 use parse::attr::ParserAttr;
  21 use parse::token::{self, special_idents, gensym_ident, NtTT, Token};
  22 use parse::token::Token::*;
  23 use print;
  24 use ptr::P;
  25
  26 use util::small_vector::SmallVector;
  27
  28 use std::cell::RefCell;
  29 use std::rc::Rc;
  30
  31 struct ParserAnyMacro<'a> {
  32     parser: RefCell<Parser<'a>>,
  33 }
  34
  35 impl<'a> ParserAnyMacro<'a> {
  36     /// Make sure we don't have any tokens left to parse, so we don't
  37     /// silently drop anything. `allow_semi` is so that "optional"
  38     /// semicolons at the end of normal expressions aren't complained
  39     /// about e.g. the semicolon in `macro_rules! kapow { () => {
  40     /// panic!(); } }` doesn't get picked up by .parse_expr(), but it's
  41     /// allowed to be there.
  42     fn ensure_complete_parse(&self, allow_semi: bool) {
  43         let mut parser = self.parser.borrow_mut();
  44         if allow_semi && parser.token == token::Semi {
  45             parser.bump()
  46         }
  47         if parser.token != token::Eof {
  48             let token_str = parser.this_token_to_string();
  49             let msg = format!("macro expansion ignores token `{}` and any \
  50                                following",
  51                               token_str);
  52             let span = parser.span;
  53             parser.span_err(span, &msg[..]);
  54         }
  55     }
  56 }
  57
  58 impl<'a> MacResult for ParserAnyMacro<'a> {
  59     fn make_expr(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Expr>> {
  60         let ret = self.parser.borrow_mut().parse_expr();
  61         self.ensure_complete_parse(true);
  62         Some(ret)
  63     }
  64     fn make_pat(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Pat>> {
  65         let ret = self.parser.borrow_mut().parse_pat();
  66         self.ensure_complete_parse(false);
  67         Some(ret)
  68     }
  69     fn make_items(self: Box<ParserAnyMacro<'a>>) -> Option<SmallVector<P<ast::Item>>> {
  70         let mut ret = SmallVector::zero();
  71         loop {
  72             let mut parser = self.parser.borrow_mut();
  73             // so... do outer attributes attached to the macro invocation
  74             // just disappear? This question applies to make_methods, as
  75             // well.
  76             match parser.parse_item_with_outer_attributes() {
  77                 Some(item) => ret.push(item),
  78                 None => break
  79             }
  80         }
  81         self.ensure_complete_parse(false);
  82         Some(ret)
  83     }
  84
  85     fn make_methods(self: Box<ParserAnyMacro<'a>>) -> Option<SmallVector<P<ast::Method>>> {
  86         let mut ret = SmallVector::zero();
  87         loop {
  88             let mut parser = self.parser.borrow_mut();
  89             match parser.token {
  90                 token::Eof => break,
  91                 _ => {
  92                     ret.push(parser.parse_method_with_outer_attributes());
  93                 }
  94             }
  95         }
  96         self.ensure_complete_parse(false);
  97         Some(ret)
  98     }
  99
 100     fn make_stmt(self: Box<ParserAnyMacro<'a>>) -> Option<P<ast::Stmt>> {
 101         let attrs = self.parser.borrow_mut().parse_outer_attributes();
 102         let ret = self.parser.borrow_mut().parse_stmt(attrs);
 103         self.ensure_complete_parse(true);
 104         Some(ret)
 105     }
 106 }
 107
 108 struct MacroRulesMacroExpander {
 109     name: ast::Ident,
 110     imported_from: Option<ast::Ident>,
 111     lhses: Vec<Rc<NamedMatch>>,
 112     rhses: Vec<Rc<NamedMatch>>,
 113 }
 114
 115 impl TTMacroExpander for MacroRulesMacroExpander {
 116     fn expand<'cx>(&self,
 117                    cx: &'cx mut ExtCtxt,
 118                    sp: Span,
 119                    arg: &[ast::TokenTree])
 120                    -> Box<MacResult+'cx> {
 121         generic_extension(cx,
 122                           sp,
 123                           self.name,
 124                           self.imported_from,
 125                           arg,
 126                           &self.lhses,
 127                           &self.rhses)
 128     }
 129 }
 130
 131 /// Given `lhses` and `rhses`, this is the new macro we create
 132 fn generic_extension<'cx>(cx: &'cx ExtCtxt,
 133                           sp: Span,
 134                           name: ast::Ident,
 135                           imported_from: Option<ast::Ident>,
 136                           arg: &[ast::TokenTree],
 137                           lhses: &[Rc<NamedMatch>],
 138                           rhses: &[Rc<NamedMatch>])
 139                           -> Box<MacResult+'cx> {
 140     if cx.trace_macros() {
 141         println!("{}! {{ {} }}",
 142                  token::get_ident(name),
 143                  print::pprust::tts_to_string(arg));
 144     }
 145
 146     // Which arm's failure should we report? (the one furthest along)
 147     let mut best_fail_spot = DUMMY_SP;
 148     let mut best_fail_msg = "internal error: ran no matchers".to_string();
 149
 150     for (i, lhs) in lhses.iter().enumerate() { // try each arm's matchers
 151         match **lhs {
 152           MatchedNonterminal(NtTT(ref lhs_tt)) => {
 153             let lhs_tt = match **lhs_tt {
 154                 TtDelimited(_, ref delim) => &delim.tts[..],
 155                 _ => cx.span_fatal(sp, "malformed macro lhs")
 156             };
 157             // `None` is because we're not interpolating
 158             let arg_rdr = new_tt_reader_with_doc_flag(&cx.parse_sess().span_diagnostic,
 159                                                       None,
 160                                                       None,
 161                                                       arg.iter()
 162                                                          .cloned()
 163                                                          .collect(),
 164                                                       true);
 165             match parse(cx.parse_sess(), cx.cfg(), arg_rdr, lhs_tt) {
 166               Success(named_matches) => {
 167                 let rhs = match *rhses[i] {
 168                     // okay, what's your transcriber?
 169                     MatchedNonterminal(NtTT(ref tt)) => {
 170                         match **tt {
 171                             // ignore delimiters
 172                             TtDelimited(_, ref delimed) => delimed.tts.clone(),
 173                             _ => cx.span_fatal(sp, "macro rhs must be delimited"),
 174                         }
 175                     },
 176                     _ => cx.span_bug(sp, "bad thing in rhs")
 177                 };
 178                 // rhs has holes ( `$id` and `$(...)` that need filled)
 179                 let trncbr = new_tt_reader(&cx.parse_sess().span_diagnostic,
 180                                            Some(named_matches),
 181                                            imported_from,
 182                                            rhs);
 183                 let mut p = Parser::new(cx.parse_sess(), cx.cfg(), box trncbr);
 184                 p.check_unknown_macro_variable();
 185                 // Let the context choose how to interpret the result.
 186                 // Weird, but useful for X-macros.
 187                 return box ParserAnyMacro {
 188                     parser: RefCell::new(p),
 189                 } as Box<MacResult+'cx>
 190               }
 191               Failure(sp, ref msg) => if sp.lo >= best_fail_spot.lo {
 192                 best_fail_spot = sp;
 193                 best_fail_msg = (*msg).clone();
 194               },
 195               Error(sp, ref msg) => cx.span_fatal(sp, &msg[..])
 196             }
 197           }
 198           _ => cx.bug("non-matcher found in parsed lhses")
 199         }
 200     }
 201     cx.span_fatal(best_fail_spot, &best_fail_msg[..]);
 202 }
 203
 204 // Note that macro-by-example's input is also matched against a token tree:
 205 //                   $( $lhs:tt => $rhs:tt );+
 206 //
 207 // Holy self-referential!
 208
 209 /// Converts a `macro_rules!` invocation into a syntax extension.
 210 pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
 211                     def: &ast::MacroDef) -> SyntaxExtension {
 212
 213     let lhs_nm =  gensym_ident("lhs");
 214     let rhs_nm =  gensym_ident("rhs");
 215
 216     // The pattern that macro_rules matches.
 217     // The grammar for macro_rules! is:
 218     // $( $lhs:tt => $rhs:tt );+
 219     // ...quasiquoting this would be nice.
 220     // These spans won't matter, anyways
 221     let match_lhs_tok = MatchNt(lhs_nm, special_idents::tt, token::Plain, token::Plain);
 222     let match_rhs_tok = MatchNt(rhs_nm, special_idents::tt, token::Plain, token::Plain);
 223     let argument_gram = vec!(
 224         TtSequence(DUMMY_SP,
 225                    Rc::new(ast::SequenceRepetition {
 226                        tts: vec![
 227                            TtToken(DUMMY_SP, match_lhs_tok),
 228                            TtToken(DUMMY_SP, token::FatArrow),
 229                            TtToken(DUMMY_SP, match_rhs_tok)],
 230                        separator: Some(token::Semi),
 231                        op: ast::OneOrMore,
 232                        num_captures: 2
 233                    })),
 234         //to phase into semicolon-termination instead of
 235         //semicolon-separation
 236         TtSequence(DUMMY_SP,
 237                    Rc::new(ast::SequenceRepetition {
 238                        tts: vec![TtToken(DUMMY_SP, token::Semi)],
 239                        separator: None,
 240                        op: ast::ZeroOrMore,
 241                        num_captures: 0
 242                    })));
 243
 244
 245     // Parse the macro_rules! invocation (`none` is for no interpolations):
 246     let arg_reader = new_tt_reader(&cx.parse_sess().span_diagnostic,
 247                                    None,
 248                                    None,
 249                                    def.body.clone());
 250     let argument_map = parse_or_else(cx.parse_sess(),
 251                                      cx.cfg(),
 252                                      arg_reader,
 253                                      argument_gram);
 254
 255     // Extract the arguments:
 256     let lhses = match *argument_map[lhs_nm] {
 257         MatchedSeq(ref s, _) => /* FIXME (#2543) */ (*s).clone(),
 258         _ => cx.span_bug(def.span, "wrong-structured lhs")
 259     };
 260
 261     for lhs in &lhses {
 262         check_lhs_nt_follows(cx, &**lhs, def.span);
 263     }
 264
 265     let rhses = match *argument_map[rhs_nm] {
 266         MatchedSeq(ref s, _) => /* FIXME (#2543) */ (*s).clone(),
 267         _ => cx.span_bug(def.span, "wrong-structured rhs")
 268     };
 269
 270     let exp = box MacroRulesMacroExpander {
 271         name: def.ident,
 272         imported_from: def.imported_from,
 273         lhses: lhses,
 274         rhses: rhses,
 275     };
 276
 277     NormalTT(exp, Some(def.span))
 278 }
 279
 280 fn check_lhs_nt_follows(cx: &mut ExtCtxt, lhs: &NamedMatch, sp: Span) {
 281     // lhs is going to be like MatchedNonterminal(NtTT(TtDelimited(...))), where the entire lhs is
 282     // those tts. Or, it can be a "bare sequence", not wrapped in parens.
 283     match lhs {
 284         &MatchedNonterminal(NtTT(ref inner)) => match &**inner {
 285             &TtDelimited(_, ref tts) => {
 286                 check_matcher(cx, tts.tts.iter(), &Eof);
 287             },
 288             tt @ &TtSequence(..) => {
 289                 check_matcher(cx, Some(tt).into_iter(), &Eof);
 290             },
 291             _ => cx.span_bug(sp, "wrong-structured lhs for follow check (didn't find \
 292             a TtDelimited or TtSequence)")
 293         },
 294         _ => cx.span_bug(sp, "wrong-structured lhs for follow check (didn't find a \
 295            MatchedNonterminal)")
 296     };
 297     // we don't abort on errors on rejection, the driver will do that for us
 298     // after parsing/expansion. we can report every error in every macro this way.
 299 }
 300
 301 // returns the last token that was checked, for TtSequence. this gets used later on.
 302 fn check_matcher<'a, I>(cx: &mut ExtCtxt, matcher: I, follow: &Token)
 303 -> Option<(Span, Token)> where I: Iterator<Item=&'a TokenTree> {
 304     use print::pprust::token_to_string;
 305
 306     let mut last = None;
 307
 308     // 2. For each token T in M:
 309     let mut tokens = matcher.peekable();
 310     while let Some(token) = tokens.next() {
 311         last = match *token {
 312             TtToken(sp, MatchNt(ref name, ref frag_spec, _, _)) => {
 313                 // ii. If T is a simple NT, look ahead to the next token T' in
 314                 // M.
 315                 let next_token = match tokens.peek() {
 316                     // If T' closes a complex NT, replace T' with F
 317                     Some(&&TtToken(_, CloseDelim(_))) => follow.clone(),
 318                     Some(&&TtToken(_, ref tok)) => tok.clone(),
 319                     Some(&&TtSequence(sp, _)) => {
 320                         cx.span_err(sp,
 321                                     &format!("`${0}:{1}` is followed by a \
 322                                               sequence repetition, which is not \
 323                                               allowed for `{1}` fragments",
 324                                              name.as_str(), frag_spec.as_str())
 325                                         );
 326                         Eof
 327                     },
 328                     // die next iteration
 329                     Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
 330                     // else, we're at the end of the macro or sequence
 331                     None => follow.clone()
 332                 };
 333
 334                 let tok = if let TtToken(_, ref tok) = *token { tok } else { unreachable!() };
 335                 // If T' is in the set FOLLOW(NT), continue. Else, reject.
 336                 match (&next_token, is_in_follow(cx, &next_token, frag_spec.as_str())) {
 337                     (&Eof, _) => return Some((sp, tok.clone())),
 338                     (_, Ok(true)) => continue,
 339                     (next, Ok(false)) => {
 340                         cx.span_err(sp, &format!("`${0}:{1}` is followed by `{2}`, which \
 341                                                   is not allowed for `{1}` fragments",
 342                                                  name.as_str(), frag_spec.as_str(),
 343                                                  token_to_string(next)));
 344                         continue
 345                     },
 346                     (_, Err(msg)) => {
 347                         cx.span_err(sp, &msg);
 348                         continue
 349                     }
 350                 }
 351             },
 352             TtSequence(sp, ref seq) => {
 353                 // iii. Else, T is a complex NT.
 354                 match seq.separator {
 355                     // If T has the form $(...)U+ or $(...)U* for some token U,
 356                     // run the algorithm on the contents with F set to U. If it
 357                     // accepts, continue, else, reject.
 358                     Some(ref u) => {
 359                         let last = check_matcher(cx, seq.tts.iter(), u);
 360                         match last {
 361                             // Since the delimiter isn't required after the last
 362                             // repetition, make sure that the *next* token is
 363                             // sane. This doesn't actually compute the FIRST of
 364                             // the rest of the matcher yet, it only considers
 365                             // single tokens and simple NTs. This is imprecise,
 366                             // but conservatively correct.
 367                             Some((span, tok)) => {
 368                                 let fol = match tokens.peek() {
 369                                     Some(&&TtToken(_, ref tok)) => tok.clone(),
 370                                     Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
 371                                     Some(_) => {
 372                                         cx.span_err(sp, "sequence repetition followed by \
 373                                                 another sequence repetition, which is not allowed");
 374                                         Eof
 375                                     },
 376                                     None => Eof
 377                                 };
 378                                 check_matcher(cx, Some(&TtToken(span, tok.clone())).into_iter(),
 379                                               &fol)
 380                             },
 381                             None => last,
 382                         }
 383                     },
 384                     // If T has the form $(...)+ or $(...)*, run the algorithm
 385                     // on the contents with F set to the token following the
 386                     // sequence. If it accepts, continue, else, reject.
 387                     None => {
 388                         let fol = match tokens.peek() {
 389                             Some(&&TtToken(_, ref tok)) => tok.clone(),
 390                             Some(&&TtDelimited(_, ref delim)) => delim.close_token(),
 391                             Some(_) => {
 392                                 cx.span_err(sp, "sequence repetition followed by another \
 393                                              sequence repetition, which is not allowed");
 394                                 Eof
 395                             },
 396                             None => Eof
 397                         };
 398                         check_matcher(cx, seq.tts.iter(), &fol)
 399                     }
 400                 }
 401             },
 402             TtToken(..) => {
 403                 // i. If T is not an NT, continue.
 404                 continue
 405             },
 406             TtDelimited(_, ref tts) => {
 407                 // if we don't pass in that close delimiter, we'll incorrectly consider the matcher
 408                 // `{ $foo:ty }` as having a follow that isn't `RBrace`
 409                 check_matcher(cx, tts.tts.iter(), &tts.close_token())
 410             }
 411         }
 412     }
 413     last
 414 }
 415
 416 fn is_in_follow(_: &ExtCtxt, tok: &Token, frag: &str) -> Result<bool, String> {
 417     if let &CloseDelim(_) = tok {
 418         Ok(true)
 419     } else {
 420         match frag {
 421             "item" => {
 422                 // since items *must* be followed by either a `;` or a `}`, we can
 423                 // accept anything after them
 424                 Ok(true)
 425             },
 426             "block" => {
 427                 // anything can follow block, the braces provide a easy boundary to
 428                 // maintain
 429                 Ok(true)
 430             },
 431             "stmt" | "expr"  => {
 432                 match *tok {
 433                     FatArrow | Comma | Semi => Ok(true),
 434                     _ => Ok(false)
 435                 }
 436             },
 437             "pat" => {
 438                 match *tok {
 439                     FatArrow | Comma | Eq => Ok(true),
 440                     _ => Ok(false)
 441                 }
 442             },
 443             "path" | "ty" => {
 444                 match *tok {
 445                     Comma | FatArrow | Colon | Eq | Gt => Ok(true),
 446                     Ident(i, _) if i.as_str() == "as" => Ok(true),
 447                     _ => Ok(false)
 448                 }
 449             },
 450             "ident" => {
 451                 // being a single token, idents are harmless
 452                 Ok(true)
 453             },
 454             "meta" | "tt" => {
 455                 // being either a single token or a delimited sequence, tt is
 456                 // harmless
 457                 Ok(true)
 458             },
 459             _ => Err(format!("invalid fragment specifier `{}`", frag))
 460         }
 461     }
 462 }