src/libsyntax/parse/mod.rs

   1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 //! The main parser interface
  12
  13 use rustc_data_structures::sync::{Lrc, Lock};
  14 use ast::{self, CrateConfig, NodeId};
  15 use early_buffered_lints::{BufferedEarlyLint, BufferedEarlyLintId};
  16 use codemap::{CodeMap, FilePathMapping};
  17 use syntax_pos::{Span, FileMap, FileName, MultiSpan};
  18 use errors::{Handler, ColorConfig, DiagnosticBuilder};
  19 use feature_gate::UnstableFeatures;
  20 use parse::parser::Parser;
  21 use ptr::P;
  22 use str::char_at;
  23 use symbol::Symbol;
  24 use tokenstream::{TokenStream, TokenTree};
  25 use diagnostics::plugin::ErrorMap;
  26
  27 use std::borrow::Cow;
  28 use std::collections::HashSet;
  29 use std::iter;
  30 use std::path::{Path, PathBuf};
  31 use std::str;
  32
  33 pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>;
  34
  35 #[macro_use]
  36 pub mod parser;
  37
  38 pub mod lexer;
  39 pub mod token;
  40 pub mod attr;
  41
  42 pub mod classify;
  43
  44 /// Info about a parsing session.
  45 pub struct ParseSess {
  46     pub span_diagnostic: Handler,
  47     pub unstable_features: UnstableFeatures,
  48     pub config: CrateConfig,
  49     pub missing_fragment_specifiers: Lock<HashSet<Span>>,
  50     /// Places where raw identifiers were used. This is used for feature gating
  51     /// raw identifiers
  52     pub raw_identifier_spans: Lock<Vec<Span>>,
  53     /// The registered diagnostics codes
  54     crate registered_diagnostics: Lock<ErrorMap>,
  55     // Spans where a `mod foo;` statement was included in a non-mod.rs file.
  56     // These are used to issue errors if the non_modrs_mods feature is not enabled.
  57     pub non_modrs_mods: Lock<Vec<(ast::Ident, Span)>>,
  58     /// Used to determine and report recursive mod inclusions
  59     included_mod_stack: Lock<Vec<PathBuf>>,
  60     code_map: Lrc<CodeMap>,
  61     pub buffered_lints: Lock<Vec<BufferedEarlyLint>>,
  62 }
  63
  64 impl ParseSess {
  65     pub fn new(file_path_mapping: FilePathMapping) -> Self {
  66         let cm = Lrc::new(CodeMap::new(file_path_mapping));
  67         let handler = Handler::with_tty_emitter(ColorConfig::Auto,
  68                                                 true,
  69                                                 false,
  70                                                 Some(cm.clone()));
  71         ParseSess::with_span_handler(handler, cm)
  72     }
  73
  74     pub fn with_span_handler(handler: Handler, code_map: Lrc<CodeMap>) -> ParseSess {
  75         ParseSess {
  76             span_diagnostic: handler,
  77             unstable_features: UnstableFeatures::from_environment(),
  78             config: HashSet::new(),
  79             missing_fragment_specifiers: Lock::new(HashSet::new()),
  80             raw_identifier_spans: Lock::new(Vec::new()),
  81             registered_diagnostics: Lock::new(ErrorMap::new()),
  82             included_mod_stack: Lock::new(vec![]),
  83             code_map,
  84             non_modrs_mods: Lock::new(vec![]),
  85             buffered_lints: Lock::new(vec![]),
  86         }
  87     }
  88
  89     pub fn codemap(&self) -> &CodeMap {
  90         &self.code_map
  91     }
  92
  93     pub fn buffer_lint<S: Into<MultiSpan>>(&self,
  94         lint_id: BufferedEarlyLintId,
  95         span: S,
  96         id: NodeId,
  97         msg: &str,
  98     ) {
  99         self.buffered_lints.with_lock(|buffered_lints| {
 100             buffered_lints.push(BufferedEarlyLint{
 101                 span: span.into(),
 102                 id,
 103                 msg: msg.into(),
 104                 lint_id,
 105             });
 106         });
 107     }
 108 }
 109
 110 #[derive(Clone)]
 111 pub struct Directory<'a> {
 112     pub path: Cow<'a, Path>,
 113     pub ownership: DirectoryOwnership,
 114 }
 115
 116 #[derive(Copy, Clone)]
 117 pub enum DirectoryOwnership {
 118     Owned {
 119         // None if `mod.rs`, `Some("foo")` if we're in `foo.rs`
 120         relative: Option<ast::Ident>,
 121     },
 122     UnownedViaBlock,
 123     UnownedViaMod(bool /* legacy warnings? */),
 124 }
 125
 126 // a bunch of utility functions of the form parse_<thing>_from_<source>
 127 // where <thing> includes crate, expr, item, stmt, tts, and one that
 128 // uses a HOF to parse anything, and <source> includes file and
 129 // source_str.
 130
 131 pub fn parse_crate_from_file<'a>(input: &Path, sess: &'a ParseSess) -> PResult<'a, ast::Crate> {
 132     let mut parser = new_parser_from_file(sess, input);
 133     parser.parse_crate_mod()
 134 }
 135
 136 pub fn parse_crate_attrs_from_file<'a>(input: &Path, sess: &'a ParseSess)
 137                                        -> PResult<'a, Vec<ast::Attribute>> {
 138     let mut parser = new_parser_from_file(sess, input);
 139     parser.parse_inner_attributes()
 140 }
 141
 142 pub fn parse_crate_from_source_str(name: FileName, source: String, sess: &ParseSess)
 143                                        -> PResult<ast::Crate> {
 144     new_parser_from_source_str(sess, name, source).parse_crate_mod()
 145 }
 146
 147 pub fn parse_crate_attrs_from_source_str(name: FileName, source: String, sess: &ParseSess)
 148                                              -> PResult<Vec<ast::Attribute>> {
 149     new_parser_from_source_str(sess, name, source).parse_inner_attributes()
 150 }
 151
 152 crate fn parse_expr_from_source_str(name: FileName, source: String, sess: &ParseSess)
 153                                       -> PResult<P<ast::Expr>> {
 154     new_parser_from_source_str(sess, name, source).parse_expr()
 155 }
 156
 157 /// Parses an item.
 158 ///
 159 /// Returns `Ok(Some(item))` when successful, `Ok(None)` when no item was found, and `Err`
 160 /// when a syntax error occurred.
 161 crate fn parse_item_from_source_str(name: FileName, source: String, sess: &ParseSess)
 162                                       -> PResult<Option<P<ast::Item>>> {
 163     new_parser_from_source_str(sess, name, source).parse_item()
 164 }
 165
 166 crate fn parse_stmt_from_source_str(name: FileName, source: String, sess: &ParseSess)
 167                                       -> PResult<Option<ast::Stmt>> {
 168     new_parser_from_source_str(sess, name, source).parse_stmt()
 169 }
 170
 171 pub fn parse_stream_from_source_str(name: FileName, source: String, sess: &ParseSess,
 172                                     override_span: Option<Span>)
 173                                     -> TokenStream {
 174     filemap_to_stream(sess, sess.codemap().new_filemap(name, source), override_span)
 175 }
 176
 177 // Create a new parser from a source string
 178 pub fn new_parser_from_source_str(sess: &ParseSess, name: FileName, source: String)
 179                                       -> Parser {
 180     let mut parser = filemap_to_parser(sess, sess.codemap().new_filemap(name, source));
 181     parser.recurse_into_file_modules = false;
 182     parser
 183 }
 184
 185 /// Create a new parser, handling errors as appropriate
 186 /// if the file doesn't exist
 187 pub fn new_parser_from_file<'a>(sess: &'a ParseSess, path: &Path) -> Parser<'a> {
 188     filemap_to_parser(sess, file_to_filemap(sess, path, None))
 189 }
 190
 191 /// Given a session, a crate config, a path, and a span, add
 192 /// the file at the given path to the codemap, and return a parser.
 193 /// On an error, use the given span as the source of the problem.
 194 crate fn new_sub_parser_from_file<'a>(sess: &'a ParseSess,
 195                                     path: &Path,
 196                                     directory_ownership: DirectoryOwnership,
 197                                     module_name: Option<String>,
 198                                     sp: Span) -> Parser<'a> {
 199     let mut p = filemap_to_parser(sess, file_to_filemap(sess, path, Some(sp)));
 200     p.directory.ownership = directory_ownership;
 201     p.root_module_name = module_name;
 202     p
 203 }
 204
 205 /// Given a filemap and config, return a parser
 206 fn filemap_to_parser(sess: & ParseSess, filemap: Lrc<FileMap>) -> Parser {
 207     let end_pos = filemap.end_pos;
 208     let mut parser = stream_to_parser(sess, filemap_to_stream(sess, filemap, None));
 209
 210     if parser.token == token::Eof && parser.span.is_dummy() {
 211         parser.span = Span::new(end_pos, end_pos, parser.span.ctxt());
 212     }
 213
 214     parser
 215 }
 216
 217 // must preserve old name for now, because quote! from the *existing*
 218 // compiler expands into it
 219 pub fn new_parser_from_tts(sess: &ParseSess, tts: Vec<TokenTree>) -> Parser {
 220     stream_to_parser(sess, tts.into_iter().collect())
 221 }
 222
 223
 224 // base abstractions
 225
 226 /// Given a session and a path and an optional span (for error reporting),
 227 /// add the path to the session's codemap and return the new filemap.
 228 fn file_to_filemap(sess: &ParseSess, path: &Path, spanopt: Option<Span>)
 229                    -> Lrc<FileMap> {
 230     match sess.codemap().load_file(path) {
 231         Ok(filemap) => filemap,
 232         Err(e) => {
 233             let msg = format!("couldn't read {:?}: {}", path.display(), e);
 234             match spanopt {
 235                 Some(sp) => sess.span_diagnostic.span_fatal(sp, &msg).raise(),
 236                 None => sess.span_diagnostic.fatal(&msg).raise()
 237             }
 238         }
 239     }
 240 }
 241
 242 /// Given a filemap, produce a sequence of token-trees
 243 pub fn filemap_to_stream(sess: &ParseSess, filemap: Lrc<FileMap>, override_span: Option<Span>)
 244                          -> TokenStream {
 245     let mut srdr = lexer::StringReader::new(sess, filemap, override_span);
 246     srdr.real_token();
 247     panictry!(srdr.parse_all_token_trees())
 248 }
 249
 250 /// Given stream and the `ParseSess`, produce a parser
 251 pub fn stream_to_parser(sess: &ParseSess, stream: TokenStream) -> Parser {
 252     Parser::new(sess, stream, None, true, false)
 253 }
 254
 255 /// Parse a string representing a character literal into its final form.
 256 /// Rather than just accepting/rejecting a given literal, unescapes it as
 257 /// well. Can take any slice prefixed by a character escape. Returns the
 258 /// character and the number of characters consumed.
 259 fn char_lit(lit: &str, diag: Option<(Span, &Handler)>) -> (char, isize) {
 260     use std::char;
 261
 262     // Handle non-escaped chars first.
 263     if lit.as_bytes()[0] != b'\\' {
 264         // If the first byte isn't '\\' it might part of a multi-byte char, so
 265         // get the char with chars().
 266         let c = lit.chars().next().unwrap();
 267         return (c, 1);
 268     }
 269
 270     // Handle escaped chars.
 271     match lit.as_bytes()[1] as char {
 272         '"' => ('"', 2),
 273         'n' => ('\n', 2),
 274         'r' => ('\r', 2),
 275         't' => ('\t', 2),
 276         '\\' => ('\\', 2),
 277         '\'' => ('\'', 2),
 278         '0' => ('\0', 2),
 279         'x' => {
 280             let v = u32::from_str_radix(&lit[2..4], 16).unwrap();
 281             let c = char::from_u32(v).unwrap();
 282             (c, 4)
 283         }
 284         'u' => {
 285             assert_eq!(lit.as_bytes()[2], b'{');
 286             let idx = lit.find('}').unwrap();
 287
 288             // All digits and '_' are ascii, so treat each byte as a char.
 289             let mut v: u32 = 0;
 290             for c in lit[3..idx].bytes() {
 291                 let c = char::from(c);
 292                 if c != '_' {
 293                     let x = c.to_digit(16).unwrap();
 294                     v = v.checked_mul(16).unwrap().checked_add(x).unwrap();
 295                 }
 296             }
 297             let c = char::from_u32(v).unwrap_or_else(|| {
 298                 if let Some((span, diag)) = diag {
 299                     let mut diag = diag.struct_span_err(span, "invalid unicode character escape");
 300                     if v > 0x10FFFF {
 301                         diag.help("unicode escape must be at most 10FFFF").emit();
 302                     } else {
 303                         diag.help("unicode escape must not be a surrogate").emit();
 304                     }
 305                 }
 306                 '\u{FFFD}'
 307             });
 308             (c, (idx + 1) as isize)
 309         }
 310         _ => panic!("lexer should have rejected a bad character escape {}", lit)
 311     }
 312 }
 313
 314 /// Parse a string representing a string literal into its final form. Does
 315 /// unescaping.
 316 pub fn str_lit(lit: &str, diag: Option<(Span, &Handler)>) -> String {
 317     debug!("str_lit: given {}", lit.escape_default());
 318     let mut res = String::with_capacity(lit.len());
 319
 320     let error = |i| format!("lexer should have rejected {} at {}", lit, i);
 321
 322     /// Eat everything up to a non-whitespace
 323     fn eat<'a>(it: &mut iter::Peekable<str::CharIndices<'a>>) {
 324         loop {
 325             match it.peek().map(|x| x.1) {
 326                 Some(' ') | Some('\n') | Some('\r') | Some('\t') => {
 327                     it.next();
 328                 },
 329                 _ => { break; }
 330             }
 331         }
 332     }
 333
 334     let mut chars = lit.char_indices().peekable();
 335     while let Some((i, c)) = chars.next() {
 336         match c {
 337             '\\' => {
 338                 let ch = chars.peek().unwrap_or_else(|| {
 339                     panic!("{}", error(i))
 340                 }).1;
 341
 342                 if ch == '\n' {
 343                     eat(&mut chars);
 344                 } else if ch == '\r' {
 345                     chars.next();
 346                     let ch = chars.peek().unwrap_or_else(|| {
 347                         panic!("{}", error(i))
 348                     }).1;
 349
 350                     if ch != '\n' {
 351                         panic!("lexer accepted bare CR");
 352                     }
 353                     eat(&mut chars);
 354                 } else {
 355                     // otherwise, a normal escape
 356                     let (c, n) = char_lit(&lit[i..], diag);
 357                     for _ in 0..n - 1 { // we don't need to move past the first \
 358                         chars.next();
 359                     }
 360                     res.push(c);
 361                 }
 362             },
 363             '\r' => {
 364                 let ch = chars.peek().unwrap_or_else(|| {
 365                     panic!("{}", error(i))
 366                 }).1;
 367
 368                 if ch != '\n' {
 369                     panic!("lexer accepted bare CR");
 370                 }
 371                 chars.next();
 372                 res.push('\n');
 373             }
 374             c => res.push(c),
 375         }
 376     }
 377
 378     res.shrink_to_fit(); // probably not going to do anything, unless there was an escape.
 379     debug!("parse_str_lit: returning {}", res);
 380     res
 381 }
 382
 383 /// Parse a string representing a raw string literal into its final form. The
 384 /// only operation this does is convert embedded CRLF into a single LF.
 385 fn raw_str_lit(lit: &str) -> String {
 386     debug!("raw_str_lit: given {}", lit.escape_default());
 387     let mut res = String::with_capacity(lit.len());
 388
 389     let mut chars = lit.chars().peekable();
 390     while let Some(c) = chars.next() {
 391         if c == '\r' {
 392             if *chars.peek().unwrap() != '\n' {
 393                 panic!("lexer accepted bare CR");
 394             }
 395             chars.next();
 396             res.push('\n');
 397         } else {
 398             res.push(c);
 399         }
 400     }
 401
 402     res.shrink_to_fit();
 403     res
 404 }
 405
 406 // check if `s` looks like i32 or u1234 etc.
 407 fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool {
 408     s.len() > 1 &&
 409         first_chars.contains(&char_at(s, 0)) &&
 410         s[1..].chars().all(|c| '0' <= c && c <= '9')
 411 }
 412
 413 macro_rules! err {
 414     ($opt_diag:expr, |$span:ident, $diag:ident| $($body:tt)*) => {
 415         match $opt_diag {
 416             Some(($span, $diag)) => { $($body)* }
 417             None => return None,
 418         }
 419     }
 420 }
 421
 422 crate fn lit_token(lit: token::Lit, suf: Option<Symbol>, diag: Option<(Span, &Handler)>)
 423                  -> (bool /* suffix illegal? */, Option<ast::LitKind>) {
 424     use ast::LitKind;
 425
 426     match lit {
 427        token::Byte(i) => (true, Some(LitKind::Byte(byte_lit(&i.as_str()).0))),
 428        token::Char(i) => (true, Some(LitKind::Char(char_lit(&i.as_str(), diag).0))),
 429
 430         // There are some valid suffixes for integer and float literals,
 431         // so all the handling is done internally.
 432         token::Integer(s) => (false, integer_lit(&s.as_str(), suf, diag)),
 433         token::Float(s) => (false, float_lit(&s.as_str(), suf, diag)),
 434
 435         token::Str_(mut sym) => {
 436             // If there are no characters requiring special treatment we can
 437             // reuse the symbol from the Token. Otherwise, we must generate a
 438             // new symbol because the string in the LitKind is different to the
 439             // string in the Token.
 440             let s = &sym.as_str();
 441             if s.as_bytes().iter().any(|&c| c == b'\\' || c == b'\r') {
 442                 sym = Symbol::intern(&str_lit(s, diag));
 443             }
 444             (true, Some(LitKind::Str(sym, ast::StrStyle::Cooked)))
 445         }
 446         token::StrRaw(mut sym, n) => {
 447             // Ditto.
 448             let s = &sym.as_str();
 449             if s.contains('\r') {
 450                 sym = Symbol::intern(&raw_str_lit(s));
 451             }
 452             (true, Some(LitKind::Str(sym, ast::StrStyle::Raw(n))))
 453         }
 454         token::ByteStr(i) => {
 455             (true, Some(LitKind::ByteStr(byte_str_lit(&i.as_str()))))
 456         }
 457         token::ByteStrRaw(i, _) => {
 458             (true, Some(LitKind::ByteStr(Lrc::new(i.to_string().into_bytes()))))
 459         }
 460     }
 461 }
 462
 463 fn filtered_float_lit(data: Symbol, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>)
 464                       -> Option<ast::LitKind> {
 465     debug!("filtered_float_lit: {}, {:?}", data, suffix);
 466     let suffix = match suffix {
 467         Some(suffix) => suffix,
 468         None => return Some(ast::LitKind::FloatUnsuffixed(data)),
 469     };
 470
 471     Some(match &*suffix.as_str() {
 472         "f32" => ast::LitKind::Float(data, ast::FloatTy::F32),
 473         "f64" => ast::LitKind::Float(data, ast::FloatTy::F64),
 474         suf => {
 475             err!(diag, |span, diag| {
 476                 if suf.len() >= 2 && looks_like_width_suffix(&['f'], suf) {
 477                     // if it looks like a width, lets try to be helpful.
 478                     let msg = format!("invalid width `{}` for float literal", &suf[1..]);
 479                     diag.struct_span_err(span, &msg).help("valid widths are 32 and 64").emit()
 480                 } else {
 481                     let msg = format!("invalid suffix `{}` for float literal", suf);
 482                     diag.struct_span_err(span, &msg)
 483                         .help("valid suffixes are `f32` and `f64`")
 484                         .emit();
 485                 }
 486             });
 487
 488             ast::LitKind::FloatUnsuffixed(data)
 489         }
 490     })
 491 }
 492 fn float_lit(s: &str, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>)
 493                  -> Option<ast::LitKind> {
 494     debug!("float_lit: {:?}, {:?}", s, suffix);
 495     // FIXME #2252: bounds checking float literals is deferred until trans
 496     let s = s.chars().filter(|&c| c != '_').collect::<String>();
 497     filtered_float_lit(Symbol::intern(&s), suffix, diag)
 498 }
 499
 500 /// Parse a string representing a byte literal into its final form. Similar to `char_lit`
 501 fn byte_lit(lit: &str) -> (u8, usize) {
 502     let err = |i| format!("lexer accepted invalid byte literal {} step {}", lit, i);
 503
 504     if lit.len() == 1 {
 505         (lit.as_bytes()[0], 1)
 506     } else {
 507         assert_eq!(lit.as_bytes()[0], b'\\', "{}", err(0));
 508         let b = match lit.as_bytes()[1] {
 509             b'"' => b'"',
 510             b'n' => b'\n',
 511             b'r' => b'\r',
 512             b't' => b'\t',
 513             b'\\' => b'\\',
 514             b'\'' => b'\'',
 515             b'0' => b'\0',
 516             _ => {
 517                 match u64::from_str_radix(&lit[2..4], 16).ok() {
 518                     Some(c) =>
 519                         if c > 0xFF {
 520                             panic!(err(2))
 521                         } else {
 522                             return (c as u8, 4)
 523                         },
 524                     None => panic!(err(3))
 525                 }
 526             }
 527         };
 528         (b, 2)
 529     }
 530 }
 531
 532 fn byte_str_lit(lit: &str) -> Lrc<Vec<u8>> {
 533     let mut res = Vec::with_capacity(lit.len());
 534
 535     let error = |i| format!("lexer should have rejected {} at {}", lit, i);
 536
 537     /// Eat everything up to a non-whitespace
 538     fn eat<I: Iterator<Item=(usize, u8)>>(it: &mut iter::Peekable<I>) {
 539         loop {
 540             match it.peek().map(|x| x.1) {
 541                 Some(b' ') | Some(b'\n') | Some(b'\r') | Some(b'\t') => {
 542                     it.next();
 543                 },
 544                 _ => { break; }
 545             }
 546         }
 547     }
 548
 549     // byte string literals *must* be ASCII, but the escapes don't have to be
 550     let mut chars = lit.bytes().enumerate().peekable();
 551     loop {
 552         match chars.next() {
 553             Some((i, b'\\')) => {
 554                 let em = error(i);
 555                 match chars.peek().expect(&em).1 {
 556                     b'\n' => eat(&mut chars),
 557                     b'\r' => {
 558                         chars.next();
 559                         if chars.peek().expect(&em).1 != b'\n' {
 560                             panic!("lexer accepted bare CR");
 561                         }
 562                         eat(&mut chars);
 563                     }
 564                     _ => {
 565                         // otherwise, a normal escape
 566                         let (c, n) = byte_lit(&lit[i..]);
 567                         // we don't need to move past the first \
 568                         for _ in 0..n - 1 {
 569                             chars.next();
 570                         }
 571                         res.push(c);
 572                     }
 573                 }
 574             },
 575             Some((i, b'\r')) => {
 576                 let em = error(i);
 577                 if chars.peek().expect(&em).1 != b'\n' {
 578                     panic!("lexer accepted bare CR");
 579                 }
 580                 chars.next();
 581                 res.push(b'\n');
 582             }
 583             Some((_, c)) => res.push(c),
 584             None => break,
 585         }
 586     }
 587
 588     Lrc::new(res)
 589 }
 590
 591 fn integer_lit(s: &str, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>)
 592                    -> Option<ast::LitKind> {
 593     // s can only be ascii, byte indexing is fine
 594
 595     let s2 = s.chars().filter(|&c| c != '_').collect::<String>();
 596     let mut s = &s2[..];
 597
 598     debug!("integer_lit: {}, {:?}", s, suffix);
 599
 600     let mut base = 10;
 601     let orig = s;
 602     let mut ty = ast::LitIntType::Unsuffixed;
 603
 604     if char_at(s, 0) == '0' && s.len() > 1 {
 605         match char_at(s, 1) {
 606             'x' => base = 16,
 607             'o' => base = 8,
 608             'b' => base = 2,
 609             _ => { }
 610         }
 611     }
 612
 613     // 1f64 and 2f32 etc. are valid float literals.
 614     if let Some(suf) = suffix {
 615         if looks_like_width_suffix(&['f'], &suf.as_str()) {
 616             let err = match base {
 617                 16 => Some("hexadecimal float literal is not supported"),
 618                 8 => Some("octal float literal is not supported"),
 619                 2 => Some("binary float literal is not supported"),
 620                 _ => None,
 621             };
 622             if let Some(err) = err {
 623                 err!(diag, |span, diag| diag.span_err(span, err));
 624             }
 625             return filtered_float_lit(Symbol::intern(s), Some(suf), diag)
 626         }
 627     }
 628
 629     if base != 10 {
 630         s = &s[2..];
 631     }
 632
 633     if let Some(suf) = suffix {
 634         if suf.as_str().is_empty() {
 635             err!(diag, |span, diag| diag.span_bug(span, "found empty literal suffix in Some"));
 636         }
 637         ty = match &*suf.as_str() {
 638             "isize" => ast::LitIntType::Signed(ast::IntTy::Isize),
 639             "i8"  => ast::LitIntType::Signed(ast::IntTy::I8),
 640             "i16" => ast::LitIntType::Signed(ast::IntTy::I16),
 641             "i32" => ast::LitIntType::Signed(ast::IntTy::I32),
 642             "i64" => ast::LitIntType::Signed(ast::IntTy::I64),
 643             "i128" => ast::LitIntType::Signed(ast::IntTy::I128),
 644             "usize" => ast::LitIntType::Unsigned(ast::UintTy::Usize),
 645             "u8"  => ast::LitIntType::Unsigned(ast::UintTy::U8),
 646             "u16" => ast::LitIntType::Unsigned(ast::UintTy::U16),
 647             "u32" => ast::LitIntType::Unsigned(ast::UintTy::U32),
 648             "u64" => ast::LitIntType::Unsigned(ast::UintTy::U64),
 649             "u128" => ast::LitIntType::Unsigned(ast::UintTy::U128),
 650             suf => {
 651                 // i<digits> and u<digits> look like widths, so lets
 652                 // give an error message along those lines
 653                 err!(diag, |span, diag| {
 654                     if looks_like_width_suffix(&['i', 'u'], suf) {
 655                         let msg = format!("invalid width `{}` for integer literal", &suf[1..]);
 656                         diag.struct_span_err(span, &msg)
 657                             .help("valid widths are 8, 16, 32, 64 and 128")
 658                             .emit();
 659                     } else {
 660                         let msg = format!("invalid suffix `{}` for numeric literal", suf);
 661                         diag.struct_span_err(span, &msg)
 662                             .help("the suffix must be one of the integral types \
 663                                    (`u32`, `isize`, etc)")
 664                             .emit();
 665                     }
 666                 });
 667
 668                 ty
 669             }
 670         }
 671     }
 672
 673     debug!("integer_lit: the type is {:?}, base {:?}, the new string is {:?}, the original \
 674            string was {:?}, the original suffix was {:?}", ty, base, s, orig, suffix);
 675
 676     Some(match u128::from_str_radix(s, base) {
 677         Ok(r) => ast::LitKind::Int(r, ty),
 678         Err(_) => {
 679             // small bases are lexed as if they were base 10, e.g, the string
 680             // might be `0b10201`. This will cause the conversion above to fail,
 681             // but these cases have errors in the lexer: we don't want to emit
 682             // two errors, and we especially don't want to emit this error since
 683             // it isn't necessarily true.
 684             let already_errored = base < 10 &&
 685                 s.chars().any(|c| c.to_digit(10).map_or(false, |d| d >= base));
 686
 687             if !already_errored {
 688                 err!(diag, |span, diag| diag.span_err(span, "int literal is too large"));
 689             }
 690             ast::LitKind::Int(0, ty)
 691         }
 692     })
 693 }
 694
 695 /// `SeqSep` : a sequence separator (token)
 696 /// and whether a trailing separator is allowed.
 697 pub struct SeqSep {
 698     pub sep: Option<token::Token>,
 699     pub trailing_sep_allowed: bool,
 700 }
 701
 702 impl SeqSep {
 703     pub fn trailing_allowed(t: token::Token) -> SeqSep {
 704         SeqSep {
 705             sep: Some(t),
 706             trailing_sep_allowed: true,
 707         }
 708     }
 709
 710     pub fn none() -> SeqSep {
 711         SeqSep {
 712             sep: None,
 713             trailing_sep_allowed: false,
 714         }
 715     }
 716 }
 717
 718 #[cfg(test)]
 719 mod tests {
 720     use super::*;
 721     use syntax_pos::{Span, BytePos, Pos, NO_EXPANSION};
 722     use ast::{self, Ident, PatKind};
 723     use attr::first_attr_value_str_by_name;
 724     use parse;
 725     use print::pprust::item_to_string;
 726     use tokenstream::{self, TokenTree};
 727     use util::parser_testing::string_to_stream;
 728     use util::parser_testing::{string_to_expr, string_to_item};
 729     use with_globals;
 730
 731     // produce a syntax_pos::span
 732     fn sp(a: u32, b: u32) -> Span {
 733         Span::new(BytePos(a), BytePos(b), NO_EXPANSION)
 734     }
 735
 736     #[should_panic]
 737     #[test] fn bad_path_expr_1() {
 738         with_globals(|| {
 739             string_to_expr("::abc::def::return".to_string());
 740         })
 741     }
 742
 743     // check the token-tree-ization of macros
 744     #[test]
 745     fn string_to_tts_macro () {
 746         with_globals(|| {
 747             let tts: Vec<_> =
 748                 string_to_stream("macro_rules! zip (($a)=>($a))".to_string()).trees().collect();
 749             let tts: &[TokenTree] = &tts[..];
 750
 751             match (tts.len(), tts.get(0), tts.get(1), tts.get(2), tts.get(3)) {
 752                 (
 753                     4,
 754                     Some(&TokenTree::Token(_, token::Ident(name_macro_rules, false))),
 755                     Some(&TokenTree::Token(_, token::Not)),
 756                     Some(&TokenTree::Token(_, token::Ident(name_zip, false))),
 757                     Some(&TokenTree::Delimited(_, ref macro_delimed)),
 758                 )
 759                 if name_macro_rules.name == "macro_rules"
 760                 && name_zip.name == "zip" => {
 761                     let tts = &macro_delimed.stream().trees().collect::<Vec<_>>();
 762                     match (tts.len(), tts.get(0), tts.get(1), tts.get(2)) {
 763                         (
 764                             3,
 765                             Some(&TokenTree::Delimited(_, ref first_delimed)),
 766                             Some(&TokenTree::Token(_, token::FatArrow)),
 767                             Some(&TokenTree::Delimited(_, ref second_delimed)),
 768                         )
 769                         if macro_delimed.delim == token::Paren => {
 770                             let tts = &first_delimed.stream().trees().collect::<Vec<_>>();
 771                             match (tts.len(), tts.get(0), tts.get(1)) {
 772                                 (
 773                                     2,
 774                                     Some(&TokenTree::Token(_, token::Dollar)),
 775                                     Some(&TokenTree::Token(_, token::Ident(ident, false))),
 776                                 )
 777                                 if first_delimed.delim == token::Paren && ident.name == "a" => {},
 778                                 _ => panic!("value 3: {:?}", *first_delimed),
 779                             }
 780                             let tts = &second_delimed.stream().trees().collect::<Vec<_>>();
 781                             match (tts.len(), tts.get(0), tts.get(1)) {
 782                                 (
 783                                     2,
 784                                     Some(&TokenTree::Token(_, token::Dollar)),
 785                                     Some(&TokenTree::Token(_, token::Ident(ident, false))),
 786                                 )
 787                                 if second_delimed.delim == token::Paren
 788                                 && ident.name == "a" => {},
 789                                 _ => panic!("value 4: {:?}", *second_delimed),
 790                             }
 791                         },
 792                         _ => panic!("value 2: {:?}", *macro_delimed),
 793                     }
 794                 },
 795                 _ => panic!("value: {:?}",tts),
 796             }
 797         })
 798     }
 799
 800     #[test]
 801     fn string_to_tts_1() {
 802         with_globals(|| {
 803             let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
 804
 805             let expected = TokenStream::concat(vec![
 806                 TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"), false)).into(),
 807                 TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(),
 808                 TokenTree::Delimited(
 809                     sp(5, 14),
 810                     tokenstream::Delimited {
 811                         delim: token::DelimToken::Paren,
 812                         tts: TokenStream::concat(vec![
 813                             TokenTree::Token(sp(6, 7),
 814                                              token::Ident(Ident::from_str("b"), false)).into(),
 815                             TokenTree::Token(sp(8, 9), token::Colon).into(),
 816                             TokenTree::Token(sp(10, 13),
 817                                              token::Ident(Ident::from_str("i32"), false)).into(),
 818                         ]).into(),
 819                     }).into(),
 820                 TokenTree::Delimited(
 821                     sp(15, 21),
 822                     tokenstream::Delimited {
 823                         delim: token::DelimToken::Brace,
 824                         tts: TokenStream::concat(vec![
 825                             TokenTree::Token(sp(17, 18),
 826                                              token::Ident(Ident::from_str("b"), false)).into(),
 827                             TokenTree::Token(sp(18, 19), token::Semi).into(),
 828                         ]).into(),
 829                     }).into()
 830             ]);
 831
 832             assert_eq!(tts, expected);
 833         })
 834     }
 835
 836     #[test] fn parse_use() {
 837         with_globals(|| {
 838             let use_s = "use foo::bar::baz;";
 839             let vitem = string_to_item(use_s.to_string()).unwrap();
 840             let vitem_s = item_to_string(&vitem);
 841             assert_eq!(&vitem_s[..], use_s);
 842
 843             let use_s = "use foo::bar as baz;";
 844             let vitem = string_to_item(use_s.to_string()).unwrap();
 845             let vitem_s = item_to_string(&vitem);
 846             assert_eq!(&vitem_s[..], use_s);
 847         })
 848     }
 849
 850     #[test] fn parse_extern_crate() {
 851         with_globals(|| {
 852             let ex_s = "extern crate foo;";
 853             let vitem = string_to_item(ex_s.to_string()).unwrap();
 854             let vitem_s = item_to_string(&vitem);
 855             assert_eq!(&vitem_s[..], ex_s);
 856
 857             let ex_s = "extern crate foo as bar;";
 858             let vitem = string_to_item(ex_s.to_string()).unwrap();
 859             let vitem_s = item_to_string(&vitem);
 860             assert_eq!(&vitem_s[..], ex_s);
 861         })
 862     }
 863
 864     fn get_spans_of_pat_idents(src: &str) -> Vec<Span> {
 865         let item = string_to_item(src.to_string()).unwrap();
 866
 867         struct PatIdentVisitor {
 868             spans: Vec<Span>
 869         }
 870         impl<'a> ::visit::Visitor<'a> for PatIdentVisitor {
 871             fn visit_pat(&mut self, p: &'a ast::Pat) {
 872                 match p.node {
 873                     PatKind::Ident(_ , ref spannedident, _) => {
 874                         self.spans.push(spannedident.span.clone());
 875                     }
 876                     _ => {
 877                         ::visit::walk_pat(self, p);
 878                     }
 879                 }
 880             }
 881         }
 882         let mut v = PatIdentVisitor { spans: Vec::new() };
 883         ::visit::walk_item(&mut v, &item);
 884         return v.spans;
 885     }
 886
 887     #[test] fn span_of_self_arg_pat_idents_are_correct() {
 888         with_globals(|| {
 889
 890             let srcs = ["impl z { fn a (&self, &myarg: i32) {} }",
 891                         "impl z { fn a (&mut self, &myarg: i32) {} }",
 892                         "impl z { fn a (&'a self, &myarg: i32) {} }",
 893                         "impl z { fn a (self, &myarg: i32) {} }",
 894                         "impl z { fn a (self: Foo, &myarg: i32) {} }",
 895                         ];
 896
 897             for &src in &srcs {
 898                 let spans = get_spans_of_pat_idents(src);
 899                 let (lo, hi) = (spans[0].lo(), spans[0].hi());
 900                 assert!("self" == &src[lo.to_usize()..hi.to_usize()],
 901                         "\"{}\" != \"self\". src=\"{}\"",
 902                         &src[lo.to_usize()..hi.to_usize()], src)
 903             }
 904         })
 905     }
 906
 907     #[test] fn parse_exprs () {
 908         with_globals(|| {
 909             // just make sure that they parse....
 910             string_to_expr("3 + 4".to_string());
 911             string_to_expr("a::z.froob(b,&(987+3))".to_string());
 912         })
 913     }
 914
 915     #[test] fn attrs_fix_bug () {
 916         with_globals(|| {
 917             string_to_item("pub fn mk_file_writer(path: &Path, flags: &[FileFlag])
 918                    -> Result<Box<Writer>, String> {
 919     #[cfg(windows)]
 920     fn wb() -> c_int {
 921       (O_WRONLY | libc::consts::os::extra::O_BINARY) as c_int
 922     }
 923
 924     #[cfg(unix)]
 925     fn wb() -> c_int { O_WRONLY as c_int }
 926
 927     let mut fflags: c_int = wb();
 928 }".to_string());
 929         })
 930     }
 931
 932     #[test] fn crlf_doc_comments() {
 933         with_globals(|| {
 934             let sess = ParseSess::new(FilePathMapping::empty());
 935
 936             let name = FileName::Custom("source".to_string());
 937             let source = "/// doc comment\r\nfn foo() {}".to_string();
 938             let item = parse_item_from_source_str(name.clone(), source, &sess)
 939                 .unwrap().unwrap();
 940             let doc = first_attr_value_str_by_name(&item.attrs, "doc").unwrap();
 941             assert_eq!(doc, "/// doc comment");
 942
 943             let source = "/// doc comment\r\n/// line 2\r\nfn foo() {}".to_string();
 944             let item = parse_item_from_source_str(name.clone(), source, &sess)
 945                 .unwrap().unwrap();
 946             let docs = item.attrs.iter().filter(|a| a.path == "doc")
 947                         .map(|a| a.value_str().unwrap().to_string()).collect::<Vec<_>>();
 948             let b: &[_] = &["/// doc comment".to_string(), "/// line 2".to_string()];
 949             assert_eq!(&docs[..], b);
 950
 951             let source = "/** doc comment\r\n *  with CRLF */\r\nfn foo() {}".to_string();
 952             let item = parse_item_from_source_str(name, source, &sess).unwrap().unwrap();
 953             let doc = first_attr_value_str_by_name(&item.attrs, "doc").unwrap();
 954             assert_eq!(doc, "/** doc comment\n *  with CRLF */");
 955         });
 956     }
 957
 958     #[test]
 959     fn ttdelim_span() {
 960         with_globals(|| {
 961             let sess = ParseSess::new(FilePathMapping::empty());
 962             let expr = parse::parse_expr_from_source_str(PathBuf::from("foo").into(),
 963                 "foo!( fn main() { body } )".to_string(), &sess).unwrap();
 964
 965             let tts: Vec<_> = match expr.node {
 966                 ast::ExprKind::Mac(ref mac) => mac.node.stream().trees().collect(),
 967                 _ => panic!("not a macro"),
 968             };
 969
 970             let span = tts.iter().rev().next().unwrap().span();
 971
 972             match sess.codemap().span_to_snippet(span) {
 973                 Ok(s) => assert_eq!(&s[..], "{ body }"),
 974                 Err(_) => panic!("could not get snippet"),
 975             }
 976         });
 977     }
 978
 979     // This tests that when parsing a string (rather than a file) we don't try
 980     // and read in a file for a module declaration and just parse a stub.
 981     // See `recurse_into_file_modules` in the parser.
 982     #[test]
 983     fn out_of_line_mod() {
 984         with_globals(|| {
 985             let sess = ParseSess::new(FilePathMapping::empty());
 986             let item = parse_item_from_source_str(
 987                 PathBuf::from("foo").into(),
 988                 "mod foo { struct S; mod this_does_not_exist; }".to_owned(),
 989                 &sess,
 990             ).unwrap().unwrap();
 991
 992             if let ast::ItemKind::Mod(ref m) = item.node {
 993                 assert!(m.items.len() == 2);
 994             } else {
 995                 panic!();
 996             }
 997         });
 998     }
 999 }