vendor/proc-macro2/src/fallback.rs

   1 #[cfg(span_locations)]
   2 use std::cell::RefCell;
   3 #[cfg(procmacro2_semver_exempt)]
   4 use std::cmp;
   5 use std::fmt;
   6 use std::iter;
   7 #[cfg(procmacro2_semver_exempt)]
   8 use std::path::Path;
   9 use std::path::PathBuf;
  10 use std::str::FromStr;
  11 use std::vec;
  12
  13 use strnom::{block_comment, skip_whitespace, whitespace, word_break, Cursor, PResult};
  14 use unicode_xid::UnicodeXID;
  15
  16 use {Delimiter, Punct, Spacing, TokenTree};
  17
  18 #[derive(Clone)]
  19 pub struct TokenStream {
  20     inner: Vec<TokenTree>,
  21 }
  22
  23 #[derive(Debug)]
  24 pub struct LexError;
  25
  26 impl TokenStream {
  27     pub fn new() -> TokenStream {
  28         TokenStream { inner: Vec::new() }
  29     }
  30
  31     pub fn is_empty(&self) -> bool {
  32         self.inner.len() == 0
  33     }
  34 }
  35
  36 #[cfg(span_locations)]
  37 fn get_cursor(src: &str) -> Cursor {
  38     // Create a dummy file & add it to the codemap
  39     CODEMAP.with(|cm| {
  40         let mut cm = cm.borrow_mut();
  41         let name = format!("<parsed string {}>", cm.files.len());
  42         let span = cm.add_file(&name, src);
  43         Cursor {
  44             rest: src,
  45             off: span.lo,
  46         }
  47     })
  48 }
  49
  50 #[cfg(not(span_locations))]
  51 fn get_cursor(src: &str) -> Cursor {
  52     Cursor { rest: src }
  53 }
  54
  55 impl FromStr for TokenStream {
  56     type Err = LexError;
  57
  58     fn from_str(src: &str) -> Result<TokenStream, LexError> {
  59         // Create a dummy file & add it to the codemap
  60         let cursor = get_cursor(src);
  61
  62         match token_stream(cursor) {
  63             Ok((input, output)) => {
  64                 if skip_whitespace(input).len() != 0 {
  65                     Err(LexError)
  66                 } else {
  67                     Ok(output)
  68                 }
  69             }
  70             Err(LexError) => Err(LexError),
  71         }
  72     }
  73 }
  74
  75 impl fmt::Display for TokenStream {
  76     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
  77         let mut joint = false;
  78         for (i, tt) in self.inner.iter().enumerate() {
  79             if i != 0 && !joint {
  80                 write!(f, " ")?;
  81             }
  82             joint = false;
  83             match *tt {
  84                 TokenTree::Group(ref tt) => {
  85                     let (start, end) = match tt.delimiter() {
  86                         Delimiter::Parenthesis => ("(", ")"),
  87                         Delimiter::Brace => ("{", "}"),
  88                         Delimiter::Bracket => ("[", "]"),
  89                         Delimiter::None => ("", ""),
  90                     };
  91                     if tt.stream().into_iter().next().is_none() {
  92                         write!(f, "{} {}", start, end)?
  93                     } else {
  94                         write!(f, "{} {} {}", start, tt.stream(), end)?
  95                     }
  96                 }
  97                 TokenTree::Ident(ref tt) => write!(f, "{}", tt)?,
  98                 TokenTree::Punct(ref tt) => {
  99                     write!(f, "{}", tt.as_char())?;
 100                     match tt.spacing() {
 101                         Spacing::Alone => {}
 102                         Spacing::Joint => joint = true,
 103                     }
 104                 }
 105                 TokenTree::Literal(ref tt) => write!(f, "{}", tt)?,
 106             }
 107         }
 108
 109         Ok(())
 110     }
 111 }
 112
 113 impl fmt::Debug for TokenStream {
 114     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 115         f.write_str("TokenStream ")?;
 116         f.debug_list().entries(self.clone()).finish()
 117     }
 118 }
 119
 120 #[cfg(use_proc_macro)]
 121 impl From<::proc_macro::TokenStream> for TokenStream {
 122     fn from(inner: ::proc_macro::TokenStream) -> TokenStream {
 123         inner
 124             .to_string()
 125             .parse()
 126             .expect("compiler token stream parse failed")
 127     }
 128 }
 129
 130 #[cfg(use_proc_macro)]
 131 impl From<TokenStream> for ::proc_macro::TokenStream {
 132     fn from(inner: TokenStream) -> ::proc_macro::TokenStream {
 133         inner
 134             .to_string()
 135             .parse()
 136             .expect("failed to parse to compiler tokens")
 137     }
 138 }
 139
 140 impl From<TokenTree> for TokenStream {
 141     fn from(tree: TokenTree) -> TokenStream {
 142         TokenStream { inner: vec![tree] }
 143     }
 144 }
 145
 146 impl iter::FromIterator<TokenTree> for TokenStream {
 147     fn from_iter<I: IntoIterator<Item = TokenTree>>(streams: I) -> Self {
 148         let mut v = Vec::new();
 149
 150         for token in streams.into_iter() {
 151             v.push(token);
 152         }
 153
 154         TokenStream { inner: v }
 155     }
 156 }
 157
 158 impl iter::FromIterator<TokenStream> for TokenStream {
 159     fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
 160         let mut v = Vec::new();
 161
 162         for stream in streams.into_iter() {
 163             v.extend(stream.inner);
 164         }
 165
 166         TokenStream { inner: v }
 167     }
 168 }
 169
 170 impl Extend<TokenTree> for TokenStream {
 171     fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, streams: I) {
 172         self.inner.extend(streams);
 173     }
 174 }
 175
 176 impl Extend<TokenStream> for TokenStream {
 177     fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
 178         self.inner
 179             .extend(streams.into_iter().flat_map(|stream| stream));
 180     }
 181 }
 182
 183 pub type TokenTreeIter = vec::IntoIter<TokenTree>;
 184
 185 impl IntoIterator for TokenStream {
 186     type Item = TokenTree;
 187     type IntoIter = TokenTreeIter;
 188
 189     fn into_iter(self) -> TokenTreeIter {
 190         self.inner.into_iter()
 191     }
 192 }
 193
 194 #[derive(Clone, PartialEq, Eq)]
 195 pub struct SourceFile {
 196     path: PathBuf,
 197 }
 198
 199 impl SourceFile {
 200     /// Get the path to this source file as a string.
 201     pub fn path(&self) -> PathBuf {
 202         self.path.clone()
 203     }
 204
 205     pub fn is_real(&self) -> bool {
 206         // XXX(nika): Support real files in the future?
 207         false
 208     }
 209 }
 210
 211 impl fmt::Debug for SourceFile {
 212     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 213         f.debug_struct("SourceFile")
 214             .field("path", &self.path())
 215             .field("is_real", &self.is_real())
 216             .finish()
 217     }
 218 }
 219
 220 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 221 pub struct LineColumn {
 222     pub line: usize,
 223     pub column: usize,
 224 }
 225
 226 #[cfg(span_locations)]
 227 thread_local! {
 228     static CODEMAP: RefCell<Codemap> = RefCell::new(Codemap {
 229         // NOTE: We start with a single dummy file which all call_site() and
 230         // def_site() spans reference.
 231         files: vec![{
 232             #[cfg(procmacro2_semver_exempt)]
 233             {
 234                 FileInfo {
 235                     name: "<unspecified>".to_owned(),
 236                     span: Span { lo: 0, hi: 0 },
 237                     lines: vec![0],
 238                 }
 239             }
 240
 241             #[cfg(not(procmacro2_semver_exempt))]
 242             {
 243                 FileInfo {
 244                     span: Span { lo: 0, hi: 0 },
 245                     lines: vec![0],
 246                 }
 247             }
 248         }],
 249     });
 250 }
 251
 252 #[cfg(span_locations)]
 253 struct FileInfo {
 254     #[cfg(procmacro2_semver_exempt)]
 255     name: String,
 256     span: Span,
 257     lines: Vec<usize>,
 258 }
 259
 260 #[cfg(span_locations)]
 261 impl FileInfo {
 262     fn offset_line_column(&self, offset: usize) -> LineColumn {
 263         assert!(self.span_within(Span {
 264             lo: offset as u32,
 265             hi: offset as u32
 266         }));
 267         let offset = offset - self.span.lo as usize;
 268         match self.lines.binary_search(&offset) {
 269             Ok(found) => LineColumn {
 270                 line: found + 1,
 271                 column: 0,
 272             },
 273             Err(idx) => LineColumn {
 274                 line: idx,
 275                 column: offset - self.lines[idx - 1],
 276             },
 277         }
 278     }
 279
 280     fn span_within(&self, span: Span) -> bool {
 281         span.lo >= self.span.lo && span.hi <= self.span.hi
 282     }
 283 }
 284
 285 /// Computesthe offsets of each line in the given source string.
 286 #[cfg(span_locations)]
 287 fn lines_offsets(s: &str) -> Vec<usize> {
 288     let mut lines = vec![0];
 289     let mut prev = 0;
 290     while let Some(len) = s[prev..].find('\n') {
 291         prev += len + 1;
 292         lines.push(prev);
 293     }
 294     lines
 295 }
 296
 297 #[cfg(span_locations)]
 298 struct Codemap {
 299     files: Vec<FileInfo>,
 300 }
 301
 302 #[cfg(span_locations)]
 303 impl Codemap {
 304     fn next_start_pos(&self) -> u32 {
 305         // Add 1 so there's always space between files.
 306         //
 307         // We'll always have at least 1 file, as we initialize our files list
 308         // with a dummy file.
 309         self.files.last().unwrap().span.hi + 1
 310     }
 311
 312     fn add_file(&mut self, name: &str, src: &str) -> Span {
 313         let lines = lines_offsets(src);
 314         let lo = self.next_start_pos();
 315         // XXX(nika): Shouild we bother doing a checked cast or checked add here?
 316         let span = Span {
 317             lo: lo,
 318             hi: lo + (src.len() as u32),
 319         };
 320
 321         #[cfg(procmacro2_semver_exempt)]
 322         self.files.push(FileInfo {
 323             name: name.to_owned(),
 324             span: span,
 325             lines: lines,
 326         });
 327
 328         #[cfg(not(procmacro2_semver_exempt))]
 329         self.files.push(FileInfo {
 330             span: span,
 331             lines: lines,
 332         });
 333         let _ = name;
 334
 335         span
 336     }
 337
 338     fn fileinfo(&self, span: Span) -> &FileInfo {
 339         for file in &self.files {
 340             if file.span_within(span) {
 341                 return file;
 342             }
 343         }
 344         panic!("Invalid span with no related FileInfo!");
 345     }
 346 }
 347
 348 #[derive(Clone, Copy, PartialEq, Eq)]
 349 pub struct Span {
 350     #[cfg(span_locations)]
 351     lo: u32,
 352     #[cfg(span_locations)]
 353     hi: u32,
 354 }
 355
 356 impl Span {
 357     #[cfg(not(span_locations))]
 358     pub fn call_site() -> Span {
 359         Span {}
 360     }
 361
 362     #[cfg(span_locations)]
 363     pub fn call_site() -> Span {
 364         Span { lo: 0, hi: 0 }
 365     }
 366
 367     #[cfg(procmacro2_semver_exempt)]
 368     pub fn def_site() -> Span {
 369         Span::call_site()
 370     }
 371
 372     #[cfg(procmacro2_semver_exempt)]
 373     pub fn resolved_at(&self, _other: Span) -> Span {
 374         // Stable spans consist only of line/column information, so
 375         // `resolved_at` and `located_at` only select which span the
 376         // caller wants line/column information from.
 377         *self
 378     }
 379
 380     #[cfg(procmacro2_semver_exempt)]
 381     pub fn located_at(&self, other: Span) -> Span {
 382         other
 383     }
 384
 385     #[cfg(procmacro2_semver_exempt)]
 386     pub fn source_file(&self) -> SourceFile {
 387         CODEMAP.with(|cm| {
 388             let cm = cm.borrow();
 389             let fi = cm.fileinfo(*self);
 390             SourceFile {
 391                 path: Path::new(&fi.name).to_owned(),
 392             }
 393         })
 394     }
 395
 396     #[cfg(span_locations)]
 397     pub fn start(&self) -> LineColumn {
 398         CODEMAP.with(|cm| {
 399             let cm = cm.borrow();
 400             let fi = cm.fileinfo(*self);
 401             fi.offset_line_column(self.lo as usize)
 402         })
 403     }
 404
 405     #[cfg(span_locations)]
 406     pub fn end(&self) -> LineColumn {
 407         CODEMAP.with(|cm| {
 408             let cm = cm.borrow();
 409             let fi = cm.fileinfo(*self);
 410             fi.offset_line_column(self.hi as usize)
 411         })
 412     }
 413
 414     #[cfg(procmacro2_semver_exempt)]
 415     pub fn join(&self, other: Span) -> Option<Span> {
 416         CODEMAP.with(|cm| {
 417             let cm = cm.borrow();
 418             // If `other` is not within the same FileInfo as us, return None.
 419             if !cm.fileinfo(*self).span_within(other) {
 420                 return None;
 421             }
 422             Some(Span {
 423                 lo: cmp::min(self.lo, other.lo),
 424                 hi: cmp::max(self.hi, other.hi),
 425             })
 426         })
 427     }
 428 }
 429
 430 impl fmt::Debug for Span {
 431     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 432         #[cfg(procmacro2_semver_exempt)]
 433         return write!(f, "bytes({}..{})", self.lo, self.hi);
 434
 435         #[cfg(not(procmacro2_semver_exempt))]
 436         write!(f, "Span")
 437     }
 438 }
 439
 440 pub fn debug_span_field_if_nontrivial(debug: &mut fmt::DebugStruct, span: Span) {
 441     if cfg!(procmacro2_semver_exempt) {
 442         debug.field("span", &span);
 443     }
 444 }
 445
 446 #[derive(Clone)]
 447 pub struct Group {
 448     delimiter: Delimiter,
 449     stream: TokenStream,
 450     span: Span,
 451 }
 452
 453 impl Group {
 454     pub fn new(delimiter: Delimiter, stream: TokenStream) -> Group {
 455         Group {
 456             delimiter: delimiter,
 457             stream: stream,
 458             span: Span::call_site(),
 459         }
 460     }
 461
 462     pub fn delimiter(&self) -> Delimiter {
 463         self.delimiter
 464     }
 465
 466     pub fn stream(&self) -> TokenStream {
 467         self.stream.clone()
 468     }
 469
 470     pub fn span(&self) -> Span {
 471         self.span
 472     }
 473
 474     #[cfg(procmacro2_semver_exempt)]
 475     pub fn span_open(&self) -> Span {
 476         self.span
 477     }
 478
 479     #[cfg(procmacro2_semver_exempt)]
 480     pub fn span_close(&self) -> Span {
 481         self.span
 482     }
 483
 484     pub fn set_span(&mut self, span: Span) {
 485         self.span = span;
 486     }
 487 }
 488
 489 impl fmt::Display for Group {
 490     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 491         let (left, right) = match self.delimiter {
 492             Delimiter::Parenthesis => ("(", ")"),
 493             Delimiter::Brace => ("{", "}"),
 494             Delimiter::Bracket => ("[", "]"),
 495             Delimiter::None => ("", ""),
 496         };
 497
 498         f.write_str(left)?;
 499         self.stream.fmt(f)?;
 500         f.write_str(right)?;
 501
 502         Ok(())
 503     }
 504 }
 505
 506 impl fmt::Debug for Group {
 507     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
 508         let mut debug = fmt.debug_struct("Group");
 509         debug.field("delimiter", &self.delimiter);
 510         debug.field("stream", &self.stream);
 511         #[cfg(procmacro2_semver_exempt)]
 512         debug.field("span", &self.span);
 513         debug.finish()
 514     }
 515 }
 516
 517 #[derive(Clone)]
 518 pub struct Ident {
 519     sym: String,
 520     span: Span,
 521     raw: bool,
 522 }
 523
 524 impl Ident {
 525     fn _new(string: &str, raw: bool, span: Span) -> Ident {
 526         validate_term(string);
 527
 528         Ident {
 529             sym: string.to_owned(),
 530             span: span,
 531             raw: raw,
 532         }
 533     }
 534
 535     pub fn new(string: &str, span: Span) -> Ident {
 536         Ident::_new(string, false, span)
 537     }
 538
 539     pub fn new_raw(string: &str, span: Span) -> Ident {
 540         Ident::_new(string, true, span)
 541     }
 542
 543     pub fn span(&self) -> Span {
 544         self.span
 545     }
 546
 547     pub fn set_span(&mut self, span: Span) {
 548         self.span = span;
 549     }
 550 }
 551
 552 #[inline]
 553 fn is_ident_start(c: char) -> bool {
 554     ('a' <= c && c <= 'z')
 555         || ('A' <= c && c <= 'Z')
 556         || c == '_'
 557         || (c > '\x7f' && UnicodeXID::is_xid_start(c))
 558 }
 559
 560 #[inline]
 561 fn is_ident_continue(c: char) -> bool {
 562     ('a' <= c && c <= 'z')
 563         || ('A' <= c && c <= 'Z')
 564         || c == '_'
 565         || ('0' <= c && c <= '9')
 566         || (c > '\x7f' && UnicodeXID::is_xid_continue(c))
 567 }
 568
 569 fn validate_term(string: &str) {
 570     let validate = string;
 571     if validate.is_empty() {
 572         panic!("Ident is not allowed to be empty; use Option<Ident>");
 573     }
 574
 575     if validate.bytes().all(|digit| digit >= b'0' && digit <= b'9') {
 576         panic!("Ident cannot be a number; use Literal instead");
 577     }
 578
 579     fn ident_ok(string: &str) -> bool {
 580         let mut chars = string.chars();
 581         let first = chars.next().unwrap();
 582         if !is_ident_start(first) {
 583             return false;
 584         }
 585         for ch in chars {
 586             if !is_ident_continue(ch) {
 587                 return false;
 588             }
 589         }
 590         true
 591     }
 592
 593     if !ident_ok(validate) {
 594         panic!("{:?} is not a valid Ident", string);
 595     }
 596 }
 597
 598 impl PartialEq for Ident {
 599     fn eq(&self, other: &Ident) -> bool {
 600         self.sym == other.sym && self.raw == other.raw
 601     }
 602 }
 603
 604 impl<T> PartialEq<T> for Ident
 605 where
 606     T: ?Sized + AsRef<str>,
 607 {
 608     fn eq(&self, other: &T) -> bool {
 609         let other = other.as_ref();
 610         if self.raw {
 611             other.starts_with("r#") && self.sym == other[2..]
 612         } else {
 613             self.sym == other
 614         }
 615     }
 616 }
 617
 618 impl fmt::Display for Ident {
 619     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 620         if self.raw {
 621             "r#".fmt(f)?;
 622         }
 623         self.sym.fmt(f)
 624     }
 625 }
 626
 627 impl fmt::Debug for Ident {
 628     // Ident(proc_macro), Ident(r#union)
 629     #[cfg(not(procmacro2_semver_exempt))]
 630     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 631         let mut debug = f.debug_tuple("Ident");
 632         debug.field(&format_args!("{}", self));
 633         debug.finish()
 634     }
 635
 636     // Ident {
 637     //     sym: proc_macro,
 638     //     span: bytes(128..138)
 639     // }
 640     #[cfg(procmacro2_semver_exempt)]
 641     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 642         let mut debug = f.debug_struct("Ident");
 643         debug.field("sym", &format_args!("{}", self));
 644         debug.field("span", &self.span);
 645         debug.finish()
 646     }
 647 }
 648
 649 #[derive(Clone)]
 650 pub struct Literal {
 651     text: String,
 652     span: Span,
 653 }
 654
 655 macro_rules! suffixed_numbers {
 656     ($($name:ident => $kind:ident,)*) => ($(
 657         pub fn $name(n: $kind) -> Literal {
 658             Literal::_new(format!(concat!("{}", stringify!($kind)), n))
 659         }
 660     )*)
 661 }
 662
 663 macro_rules! unsuffixed_numbers {
 664     ($($name:ident => $kind:ident,)*) => ($(
 665         pub fn $name(n: $kind) -> Literal {
 666             Literal::_new(n.to_string())
 667         }
 668     )*)
 669 }
 670
 671 impl Literal {
 672     fn _new(text: String) -> Literal {
 673         Literal {
 674             text: text,
 675             span: Span::call_site(),
 676         }
 677     }
 678
 679     suffixed_numbers! {
 680         u8_suffixed => u8,
 681         u16_suffixed => u16,
 682         u32_suffixed => u32,
 683         u64_suffixed => u64,
 684         usize_suffixed => usize,
 685         i8_suffixed => i8,
 686         i16_suffixed => i16,
 687         i32_suffixed => i32,
 688         i64_suffixed => i64,
 689         isize_suffixed => isize,
 690
 691         f32_suffixed => f32,
 692         f64_suffixed => f64,
 693     }
 694
 695     #[cfg(u128)]
 696     suffixed_numbers! {
 697         u128_suffixed => u128,
 698         i128_suffixed => i128,
 699     }
 700
 701     unsuffixed_numbers! {
 702         u8_unsuffixed => u8,
 703         u16_unsuffixed => u16,
 704         u32_unsuffixed => u32,
 705         u64_unsuffixed => u64,
 706         usize_unsuffixed => usize,
 707         i8_unsuffixed => i8,
 708         i16_unsuffixed => i16,
 709         i32_unsuffixed => i32,
 710         i64_unsuffixed => i64,
 711         isize_unsuffixed => isize,
 712     }
 713
 714     #[cfg(u128)]
 715     unsuffixed_numbers! {
 716         u128_unsuffixed => u128,
 717         i128_unsuffixed => i128,
 718     }
 719
 720     pub fn f32_unsuffixed(f: f32) -> Literal {
 721         let mut s = f.to_string();
 722         if !s.contains(".") {
 723             s.push_str(".0");
 724         }
 725         Literal::_new(s)
 726     }
 727
 728     pub fn f64_unsuffixed(f: f64) -> Literal {
 729         let mut s = f.to_string();
 730         if !s.contains(".") {
 731             s.push_str(".0");
 732         }
 733         Literal::_new(s)
 734     }
 735
 736     pub fn string(t: &str) -> Literal {
 737         let mut s = t
 738             .chars()
 739             .flat_map(|c| c.escape_default())
 740             .collect::<String>();
 741         s.push('"');
 742         s.insert(0, '"');
 743         Literal::_new(s)
 744     }
 745
 746     pub fn character(t: char) -> Literal {
 747         Literal::_new(format!("'{}'", t.escape_default().collect::<String>()))
 748     }
 749
 750     pub fn byte_string(bytes: &[u8]) -> Literal {
 751         let mut escaped = "b\"".to_string();
 752         for b in bytes {
 753             match *b {
 754                 b'\0' => escaped.push_str(r"\0"),
 755                 b'\t' => escaped.push_str(r"\t"),
 756                 b'\n' => escaped.push_str(r"\n"),
 757                 b'\r' => escaped.push_str(r"\r"),
 758                 b'"' => escaped.push_str("\\\""),
 759                 b'\\' => escaped.push_str("\\\\"),
 760                 b'\x20'...b'\x7E' => escaped.push(*b as char),
 761                 _ => escaped.push_str(&format!("\\x{:02X}", b)),
 762             }
 763         }
 764         escaped.push('"');
 765         Literal::_new(escaped)
 766     }
 767
 768     pub fn span(&self) -> Span {
 769         self.span
 770     }
 771
 772     pub fn set_span(&mut self, span: Span) {
 773         self.span = span;
 774     }
 775 }
 776
 777 impl fmt::Display for Literal {
 778     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 779         self.text.fmt(f)
 780     }
 781 }
 782
 783 impl fmt::Debug for Literal {
 784     fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
 785         let mut debug = fmt.debug_struct("Literal");
 786         debug.field("lit", &format_args!("{}", self.text));
 787         #[cfg(procmacro2_semver_exempt)]
 788         debug.field("span", &self.span);
 789         debug.finish()
 790     }
 791 }
 792
 793 fn token_stream(mut input: Cursor) -> PResult<TokenStream> {
 794     let mut trees = Vec::new();
 795     loop {
 796         let input_no_ws = skip_whitespace(input);
 797         if input_no_ws.rest.len() == 0 {
 798             break;
 799         }
 800         if let Ok((a, tokens)) = doc_comment(input_no_ws) {
 801             input = a;
 802             trees.extend(tokens);
 803             continue;
 804         }
 805
 806         let (a, tt) = match token_tree(input_no_ws) {
 807             Ok(p) => p,
 808             Err(_) => break,
 809         };
 810         trees.push(tt);
 811         input = a;
 812     }
 813     Ok((input, TokenStream { inner: trees }))
 814 }
 815
 816 #[cfg(not(span_locations))]
 817 fn spanned<'a, T>(
 818     input: Cursor<'a>,
 819     f: fn(Cursor<'a>) -> PResult<'a, T>,
 820 ) -> PResult<'a, (T, ::Span)> {
 821     let (a, b) = f(skip_whitespace(input))?;
 822     Ok((a, ((b, ::Span::_new_stable(Span::call_site())))))
 823 }
 824
 825 #[cfg(span_locations)]
 826 fn spanned<'a, T>(
 827     input: Cursor<'a>,
 828     f: fn(Cursor<'a>) -> PResult<'a, T>,
 829 ) -> PResult<'a, (T, ::Span)> {
 830     let input = skip_whitespace(input);
 831     let lo = input.off;
 832     let (a, b) = f(input)?;
 833     let hi = a.off;
 834     let span = ::Span::_new_stable(Span { lo: lo, hi: hi });
 835     Ok((a, (b, span)))
 836 }
 837
 838 fn token_tree(input: Cursor) -> PResult<TokenTree> {
 839     let (rest, (mut tt, span)) = spanned(input, token_kind)?;
 840     tt.set_span(span);
 841     Ok((rest, tt))
 842 }
 843
 844 named!(token_kind -> TokenTree, alt!(
 845     map!(group, |g| TokenTree::Group(::Group::_new_stable(g)))
 846     |
 847     map!(literal, |l| TokenTree::Literal(::Literal::_new_stable(l))) // must be before symbol
 848     |
 849     map!(op, TokenTree::Punct)
 850     |
 851     symbol_leading_ws
 852 ));
 853
 854 named!(group -> Group, alt!(
 855     delimited!(
 856         punct!("("),
 857         token_stream,
 858         punct!(")")
 859     ) => { |ts| Group::new(Delimiter::Parenthesis, ts) }
 860     |
 861     delimited!(
 862         punct!("["),
 863         token_stream,
 864         punct!("]")
 865     ) => { |ts| Group::new(Delimiter::Bracket, ts) }
 866     |
 867     delimited!(
 868         punct!("{"),
 869         token_stream,
 870         punct!("}")
 871     ) => { |ts| Group::new(Delimiter::Brace, ts) }
 872 ));
 873
 874 fn symbol_leading_ws(input: Cursor) -> PResult<TokenTree> {
 875     symbol(skip_whitespace(input))
 876 }
 877
 878 fn symbol(input: Cursor) -> PResult<TokenTree> {
 879     let mut chars = input.char_indices();
 880
 881     let raw = input.starts_with("r#");
 882     if raw {
 883         chars.next();
 884         chars.next();
 885     }
 886
 887     match chars.next() {
 888         Some((_, ch)) if is_ident_start(ch) => {}
 889         _ => return Err(LexError),
 890     }
 891
 892     let mut end = input.len();
 893     for (i, ch) in chars {
 894         if !is_ident_continue(ch) {
 895             end = i;
 896             break;
 897         }
 898     }
 899
 900     let a = &input.rest[..end];
 901     if a == "r#_" {
 902         Err(LexError)
 903     } else {
 904         let ident = if raw {
 905             ::Ident::_new_raw(&a[2..], ::Span::call_site())
 906         } else {
 907             ::Ident::new(a, ::Span::call_site())
 908         };
 909         Ok((input.advance(end), ident.into()))
 910     }
 911 }
 912
 913 fn literal(input: Cursor) -> PResult<Literal> {
 914     let input_no_ws = skip_whitespace(input);
 915
 916     match literal_nocapture(input_no_ws) {
 917         Ok((a, ())) => {
 918             let start = input.len() - input_no_ws.len();
 919             let len = input_no_ws.len() - a.len();
 920             let end = start + len;
 921             Ok((a, Literal::_new(input.rest[start..end].to_string())))
 922         }
 923         Err(LexError) => Err(LexError),
 924     }
 925 }
 926
 927 named!(literal_nocapture -> (), alt!(
 928     string
 929     |
 930     byte_string
 931     |
 932     byte
 933     |
 934     character
 935     |
 936     float
 937     |
 938     int
 939 ));
 940
 941 named!(string -> (), alt!(
 942     quoted_string
 943     |
 944     preceded!(
 945         punct!("r"),
 946         raw_string
 947     ) => { |_| () }
 948 ));
 949
 950 named!(quoted_string -> (), delimited!(
 951     punct!("\""),
 952     cooked_string,
 953     tag!("\"")
 954 ));
 955
 956 fn cooked_string(input: Cursor) -> PResult<()> {
 957     let mut chars = input.char_indices().peekable();
 958     while let Some((byte_offset, ch)) = chars.next() {
 959         match ch {
 960             '"' => {
 961                 return Ok((input.advance(byte_offset), ()));
 962             }
 963             '\r' => {
 964                 if let Some((_, '\n')) = chars.next() {
 965                     // ...
 966                 } else {
 967                     break;
 968                 }
 969             }
 970             '\\' => match chars.next() {
 971                 Some((_, 'x')) => {
 972                     if !backslash_x_char(&mut chars) {
 973                         break;
 974                     }
 975                 }
 976                 Some((_, 'n')) | Some((_, 'r')) | Some((_, 't')) | Some((_, '\\'))
 977                 | Some((_, '\'')) | Some((_, '"')) | Some((_, '0')) => {}
 978                 Some((_, 'u')) => {
 979                     if !backslash_u(&mut chars) {
 980                         break;
 981                     }
 982                 }
 983                 Some((_, '\n')) | Some((_, '\r')) => {
 984                     while let Some(&(_, ch)) = chars.peek() {
 985                         if ch.is_whitespace() {
 986                             chars.next();
 987                         } else {
 988                             break;
 989                         }
 990                     }
 991                 }
 992                 _ => break,
 993             },
 994             _ch => {}
 995         }
 996     }
 997     Err(LexError)
 998 }
 999
1000 named!(byte_string -> (), alt!(
1001     delimited!(
1002         punct!("b\""),
1003         cooked_byte_string,
1004         tag!("\"")
1005     ) => { |_| () }
1006     |
1007     preceded!(
1008         punct!("br"),
1009         raw_string
1010     ) => { |_| () }
1011 ));
1012
1013 fn cooked_byte_string(mut input: Cursor) -> PResult<()> {
1014     let mut bytes = input.bytes().enumerate();
1015     'outer: while let Some((offset, b)) = bytes.next() {
1016         match b {
1017             b'"' => {
1018                 return Ok((input.advance(offset), ()));
1019             }
1020             b'\r' => {
1021                 if let Some((_, b'\n')) = bytes.next() {
1022                     // ...
1023                 } else {
1024                     break;
1025                 }
1026             }
1027             b'\\' => match bytes.next() {
1028                 Some((_, b'x')) => {
1029                     if !backslash_x_byte(&mut bytes) {
1030                         break;
1031                     }
1032                 }
1033                 Some((_, b'n')) | Some((_, b'r')) | Some((_, b't')) | Some((_, b'\\'))
1034                 | Some((_, b'0')) | Some((_, b'\'')) | Some((_, b'"')) => {}
1035                 Some((newline, b'\n')) | Some((newline, b'\r')) => {
1036                     let rest = input.advance(newline + 1);
1037                     for (offset, ch) in rest.char_indices() {
1038                         if !ch.is_whitespace() {
1039                             input = rest.advance(offset);
1040                             bytes = input.bytes().enumerate();
1041                             continue 'outer;
1042                         }
1043                     }
1044                     break;
1045                 }
1046                 _ => break,
1047             },
1048             b if b < 0x80 => {}
1049             _ => break,
1050         }
1051     }
1052     Err(LexError)
1053 }
1054
1055 fn raw_string(input: Cursor) -> PResult<()> {
1056     let mut chars = input.char_indices();
1057     let mut n = 0;
1058     while let Some((byte_offset, ch)) = chars.next() {
1059         match ch {
1060             '"' => {
1061                 n = byte_offset;
1062                 break;
1063             }
1064             '#' => {}
1065             _ => return Err(LexError),
1066         }
1067     }
1068     for (byte_offset, ch) in chars {
1069         match ch {
1070             '"' if input.advance(byte_offset + 1).starts_with(&input.rest[..n]) => {
1071                 let rest = input.advance(byte_offset + 1 + n);
1072                 return Ok((rest, ()));
1073             }
1074             '\r' => {}
1075             _ => {}
1076         }
1077     }
1078     Err(LexError)
1079 }
1080
1081 named!(byte -> (), do_parse!(
1082     punct!("b") >>
1083     tag!("'") >>
1084     cooked_byte >>
1085     tag!("'") >>
1086     (())
1087 ));
1088
1089 fn cooked_byte(input: Cursor) -> PResult<()> {
1090     let mut bytes = input.bytes().enumerate();
1091     let ok = match bytes.next().map(|(_, b)| b) {
1092         Some(b'\\') => match bytes.next().map(|(_, b)| b) {
1093             Some(b'x') => backslash_x_byte(&mut bytes),
1094             Some(b'n') | Some(b'r') | Some(b't') | Some(b'\\') | Some(b'0') | Some(b'\'')
1095             | Some(b'"') => true,
1096             _ => false,
1097         },
1098         b => b.is_some(),
1099     };
1100     if ok {
1101         match bytes.next() {
1102             Some((offset, _)) => {
1103                 if input.chars().as_str().is_char_boundary(offset) {
1104                     Ok((input.advance(offset), ()))
1105                 } else {
1106                     Err(LexError)
1107                 }
1108             }
1109             None => Ok((input.advance(input.len()), ())),
1110         }
1111     } else {
1112         Err(LexError)
1113     }
1114 }
1115
1116 named!(character -> (), do_parse!(
1117     punct!("'") >>
1118     cooked_char >>
1119     tag!("'") >>
1120     (())
1121 ));
1122
1123 fn cooked_char(input: Cursor) -> PResult<()> {
1124     let mut chars = input.char_indices();
1125     let ok = match chars.next().map(|(_, ch)| ch) {
1126         Some('\\') => match chars.next().map(|(_, ch)| ch) {
1127             Some('x') => backslash_x_char(&mut chars),
1128             Some('u') => backslash_u(&mut chars),
1129             Some('n') | Some('r') | Some('t') | Some('\\') | Some('0') | Some('\'') | Some('"') => {
1130                 true
1131             }
1132             _ => false,
1133         },
1134         ch => ch.is_some(),
1135     };
1136     if ok {
1137         match chars.next() {
1138             Some((idx, _)) => Ok((input.advance(idx), ())),
1139             None => Ok((input.advance(input.len()), ())),
1140         }
1141     } else {
1142         Err(LexError)
1143     }
1144 }
1145
1146 macro_rules! next_ch {
1147     ($chars:ident @ $pat:pat $(| $rest:pat)*) => {
1148         match $chars.next() {
1149             Some((_, ch)) => match ch {
1150                 $pat $(| $rest)*  => ch,
1151                 _ => return false,
1152             },
1153             None => return false
1154         }
1155     };
1156 }
1157
1158 fn backslash_x_char<I>(chars: &mut I) -> bool
1159 where
1160     I: Iterator<Item = (usize, char)>,
1161 {
1162     next_ch!(chars @ '0'...'7');
1163     next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F');
1164     true
1165 }
1166
1167 fn backslash_x_byte<I>(chars: &mut I) -> bool
1168 where
1169     I: Iterator<Item = (usize, u8)>,
1170 {
1171     next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F');
1172     next_ch!(chars @ b'0'...b'9' | b'a'...b'f' | b'A'...b'F');
1173     true
1174 }
1175
1176 fn backslash_u<I>(chars: &mut I) -> bool
1177 where
1178     I: Iterator<Item = (usize, char)>,
1179 {
1180     next_ch!(chars @ '{');
1181     next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F');
1182     loop {
1183         let c = next_ch!(chars @ '0'...'9' | 'a'...'f' | 'A'...'F' | '_' | '}');
1184         if c == '}' {
1185             return true;
1186         }
1187     }
1188 }
1189
1190 fn float(input: Cursor) -> PResult<()> {
1191     let (rest, ()) = float_digits(input)?;
1192     for suffix in &["f32", "f64"] {
1193         if rest.starts_with(suffix) {
1194             return word_break(rest.advance(suffix.len()));
1195         }
1196     }
1197     word_break(rest)
1198 }
1199
1200 fn float_digits(input: Cursor) -> PResult<()> {
1201     let mut chars = input.chars().peekable();
1202     match chars.next() {
1203         Some(ch) if ch >= '0' && ch <= '9' => {}
1204         _ => return Err(LexError),
1205     }
1206
1207     let mut len = 1;
1208     let mut has_dot = false;
1209     let mut has_exp = false;
1210     while let Some(&ch) = chars.peek() {
1211         match ch {
1212             '0'...'9' | '_' => {
1213                 chars.next();
1214                 len += 1;
1215             }
1216             '.' => {
1217                 if has_dot {
1218                     break;
1219                 }
1220                 chars.next();
1221                 if chars
1222                     .peek()
1223                     .map(|&ch| ch == '.' || UnicodeXID::is_xid_start(ch))
1224                     .unwrap_or(false)
1225                 {
1226                     return Err(LexError);
1227                 }
1228                 len += 1;
1229                 has_dot = true;
1230             }
1231             'e' | 'E' => {
1232                 chars.next();
1233                 len += 1;
1234                 has_exp = true;
1235                 break;
1236             }
1237             _ => break,
1238         }
1239     }
1240
1241     let rest = input.advance(len);
1242     if !(has_dot || has_exp || rest.starts_with("f32") || rest.starts_with("f64")) {
1243         return Err(LexError);
1244     }
1245
1246     if has_exp {
1247         let mut has_exp_value = false;
1248         while let Some(&ch) = chars.peek() {
1249             match ch {
1250                 '+' | '-' => {
1251                     if has_exp_value {
1252                         break;
1253                     }
1254                     chars.next();
1255                     len += 1;
1256                 }
1257                 '0'...'9' => {
1258                     chars.next();
1259                     len += 1;
1260                     has_exp_value = true;
1261                 }
1262                 '_' => {
1263                     chars.next();
1264                     len += 1;
1265                 }
1266                 _ => break,
1267             }
1268         }
1269         if !has_exp_value {
1270             return Err(LexError);
1271         }
1272     }
1273
1274     Ok((input.advance(len), ()))
1275 }
1276
1277 fn int(input: Cursor) -> PResult<()> {
1278     let (rest, ()) = digits(input)?;
1279     for suffix in &[
1280         "isize", "i8", "i16", "i32", "i64", "i128", "usize", "u8", "u16", "u32", "u64", "u128",
1281     ] {
1282         if rest.starts_with(suffix) {
1283             return word_break(rest.advance(suffix.len()));
1284         }
1285     }
1286     word_break(rest)
1287 }
1288
1289 fn digits(mut input: Cursor) -> PResult<()> {
1290     let base = if input.starts_with("0x") {
1291         input = input.advance(2);
1292         16
1293     } else if input.starts_with("0o") {
1294         input = input.advance(2);
1295         8
1296     } else if input.starts_with("0b") {
1297         input = input.advance(2);
1298         2
1299     } else {
1300         10
1301     };
1302
1303     let mut len = 0;
1304     let mut empty = true;
1305     for b in input.bytes() {
1306         let digit = match b {
1307             b'0'...b'9' => (b - b'0') as u64,
1308             b'a'...b'f' => 10 + (b - b'a') as u64,
1309             b'A'...b'F' => 10 + (b - b'A') as u64,
1310             b'_' => {
1311                 if empty && base == 10 {
1312                     return Err(LexError);
1313                 }
1314                 len += 1;
1315                 continue;
1316             }
1317             _ => break,
1318         };
1319         if digit >= base {
1320             return Err(LexError);
1321         }
1322         len += 1;
1323         empty = false;
1324     }
1325     if empty {
1326         Err(LexError)
1327     } else {
1328         Ok((input.advance(len), ()))
1329     }
1330 }
1331
1332 fn op(input: Cursor) -> PResult<Punct> {
1333     let input = skip_whitespace(input);
1334     match op_char(input) {
1335         Ok((rest, '\'')) => {
1336             symbol(rest)?;
1337             Ok((rest, Punct::new('\'', Spacing::Joint)))
1338         }
1339         Ok((rest, ch)) => {
1340             let kind = match op_char(rest) {
1341                 Ok(_) => Spacing::Joint,
1342                 Err(LexError) => Spacing::Alone,
1343             };
1344             Ok((rest, Punct::new(ch, kind)))
1345         }
1346         Err(LexError) => Err(LexError),
1347     }
1348 }
1349
1350 fn op_char(input: Cursor) -> PResult<char> {
1351     if input.starts_with("//") || input.starts_with("/*") {
1352         // Do not accept `/` of a comment as an op.
1353         return Err(LexError);
1354     }
1355
1356     let mut chars = input.chars();
1357     let first = match chars.next() {
1358         Some(ch) => ch,
1359         None => {
1360             return Err(LexError);
1361         }
1362     };
1363     let recognized = "~!@#$%^&*-=+|;:,<.>/?'";
1364     if recognized.contains(first) {
1365         Ok((input.advance(first.len_utf8()), first))
1366     } else {
1367         Err(LexError)
1368     }
1369 }
1370
1371 fn doc_comment(input: Cursor) -> PResult<Vec<TokenTree>> {
1372     let mut trees = Vec::new();
1373     let (rest, ((comment, inner), span)) = spanned(input, doc_comment_contents)?;
1374     trees.push(TokenTree::Punct(Punct::new('#', Spacing::Alone)));
1375     if inner {
1376         trees.push(Punct::new('!', Spacing::Alone).into());
1377     }
1378     let mut stream = vec![
1379         TokenTree::Ident(::Ident::new("doc", span)),
1380         TokenTree::Punct(Punct::new('=', Spacing::Alone)),
1381         TokenTree::Literal(::Literal::string(comment)),
1382     ];
1383     for tt in stream.iter_mut() {
1384         tt.set_span(span);
1385     }
1386     let group = Group::new(Delimiter::Bracket, stream.into_iter().collect());
1387     trees.push(::Group::_new_stable(group).into());
1388     for tt in trees.iter_mut() {
1389         tt.set_span(span);
1390     }
1391     Ok((rest, trees))
1392 }
1393
1394 named!(doc_comment_contents -> (&str, bool), alt!(
1395     do_parse!(
1396         punct!("//!") >>
1397         s: take_until_newline_or_eof!() >>
1398         ((s, true))
1399     )
1400     |
1401     do_parse!(
1402         option!(whitespace) >>
1403         peek!(tag!("/*!")) >>
1404         s: block_comment >>
1405         ((s, true))
1406     )
1407     |
1408     do_parse!(
1409         punct!("///") >>
1410         not!(tag!("/")) >>
1411         s: take_until_newline_or_eof!() >>
1412         ((s, false))
1413     )
1414     |
1415     do_parse!(
1416         option!(whitespace) >>
1417         peek!(tuple!(tag!("/**"), not!(tag!("*")))) >>
1418         s: block_comment >>
1419         ((s, false))
1420     )
1421 ));