compiler/rustc_lexer/src/lib.rs

   1 //! Low-level Rust lexer.
   2 //!
   3 //! The idea with `rustc_lexer` is to make a reusable library,
   4 //! by separating out pure lexing and rustc-specific concerns, like spans,
   5 //! error reporting, and interning.  So, rustc_lexer operates directly on `&str`,
   6 //! produces simple tokens which are a pair of type-tag and a bit of original text,
   7 //! and does not report errors, instead storing them as flags on the token.
   8 //!
   9 //! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
  10 //! For that see [`rustc_parse::lexer`], which converts this basic token stream
  11 //! into wide tokens used by actual parser.
  12 //!
  13 //! The purpose of this crate is to convert raw sources into a labeled sequence
  14 //! of well-known token types, so building an actual Rust token stream will
  15 //! be easier.
  16 //!
  17 //! The main entity of this crate is the [`TokenKind`] enum which represents common
  18 //! lexeme types.
  19 //!
  20 //! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
  21 #![deny(rustc::untranslatable_diagnostic)]
  22 #![deny(rustc::diagnostic_outside_of_impl)]
  23 // We want to be able to build this crate with a stable compiler, so no
  24 // `#![feature]` attributes should be added.
  25
  26 mod cursor;
  27 pub mod unescape;
  28
  29 #[cfg(test)]
  30 mod tests;
  31
  32 use self::LiteralKind::*;
  33 use self::TokenKind::*;
  34 use crate::cursor::{Cursor, EOF_CHAR};
  35 use std::convert::TryFrom;
  36
  37 /// Parsed token.
  38 /// It doesn't contain information about data that has been parsed,
  39 /// only the type of the token and its size.
  40 #[derive(Debug)]
  41 pub struct Token {
  42     pub kind: TokenKind,
  43     pub len: u32,
  44 }
  45
  46 impl Token {
  47     fn new(kind: TokenKind, len: u32) -> Token {
  48         Token { kind, len }
  49     }
  50 }
  51
  52 /// Enum representing common lexeme types.
  53 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
  54 pub enum TokenKind {
  55     // Multi-char tokens:
  56     /// "// comment"
  57     LineComment { doc_style: Option<DocStyle> },
  58     /// `/* block comment */`
  59     ///
  60     /// Block comments can be recursive, so the sequence like `/* /* */`
  61     /// will not be considered terminated and will result in a parsing error.
  62     BlockComment { doc_style: Option<DocStyle>, terminated: bool },
  63     /// Any whitespace characters sequence.
  64     Whitespace,
  65     /// "ident" or "continue"
  66     /// At this step keywords are also considered identifiers.
  67     Ident,
  68     /// Like the above, but containing invalid unicode codepoints.
  69     InvalidIdent,
  70     /// "r#ident"
  71     RawIdent,
  72     /// An unknown prefix like `foo#`, `foo'`, `foo"`. Note that only the
  73     /// prefix (`foo`) is included in the token, not the separator (which is
  74     /// lexed as its own distinct token). In Rust 2021 and later, reserved
  75     /// prefixes are reported as errors; in earlier editions, they result in a
  76     /// (allowed by default) lint, and are treated as regular identifier
  77     /// tokens.
  78     UnknownPrefix,
  79     /// "12_u8", "1.0e-40", "b"123"". See `LiteralKind` for more details.
  80     Literal { kind: LiteralKind, suffix_start: u32 },
  81     /// "'a"
  82     Lifetime { starts_with_number: bool },
  83
  84     // One-char tokens:
  85     /// ";"
  86     Semi,
  87     /// ","
  88     Comma,
  89     /// "."
  90     Dot,
  91     /// "("
  92     OpenParen,
  93     /// ")"
  94     CloseParen,
  95     /// "{"
  96     OpenBrace,
  97     /// "}"
  98     CloseBrace,
  99     /// "["
 100     OpenBracket,
 101     /// "]"
 102     CloseBracket,
 103     /// "@"
 104     At,
 105     /// "#"
 106     Pound,
 107     /// "~"
 108     Tilde,
 109     /// "?"
 110     Question,
 111     /// ":"
 112     Colon,
 113     /// "$"
 114     Dollar,
 115     /// "="
 116     Eq,
 117     /// "!"
 118     Bang,
 119     /// "<"
 120     Lt,
 121     /// ">"
 122     Gt,
 123     /// "-"
 124     Minus,
 125     /// "&"
 126     And,
 127     /// "|"
 128     Or,
 129     /// "+"
 130     Plus,
 131     /// "*"
 132     Star,
 133     /// "/"
 134     Slash,
 135     /// "^"
 136     Caret,
 137     /// "%"
 138     Percent,
 139
 140     /// Unknown token, not expected by the lexer, e.g. "№"
 141     Unknown,
 142 }
 143
 144 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 145 pub enum DocStyle {
 146     Outer,
 147     Inner,
 148 }
 149
 150 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 151 pub enum LiteralKind {
 152     /// "12_u8", "0o100", "0b120i99"
 153     Int { base: Base, empty_int: bool },
 154     /// "12.34f32", "0b100.100"
 155     Float { base: Base, empty_exponent: bool },
 156     /// "'a'", "'\\'", "'''", "';"
 157     Char { terminated: bool },
 158     /// "b'a'", "b'\\'", "b'''", "b';"
 159     Byte { terminated: bool },
 160     /// ""abc"", ""abc"
 161     Str { terminated: bool },
 162     /// "b"abc"", "b"abc"
 163     ByteStr { terminated: bool },
 164     /// "r"abc"", "r#"abc"#", "r####"ab"###"c"####", "r#"a". `None` indicates
 165     /// an invalid literal.
 166     RawStr { n_hashes: Option<u8> },
 167     /// "br"abc"", "br#"abc"#", "br####"ab"###"c"####", "br#"a". `None`
 168     /// indicates an invalid literal.
 169     RawByteStr { n_hashes: Option<u8> },
 170 }
 171
 172 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 173 pub enum RawStrError {
 174     /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
 175     InvalidStarter { bad_char: char },
 176     /// The string was not terminated, e.g. `r###"abcde"##`.
 177     /// `possible_terminator_offset` is the number of characters after `r` or
 178     /// `br` where they may have intended to terminate it.
 179     NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
 180     /// More than 255 `#`s exist.
 181     TooManyDelimiters { found: u32 },
 182 }
 183
 184 /// Base of numeric literal encoding according to its prefix.
 185 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 186 pub enum Base {
 187     /// Literal starts with "0b".
 188     Binary,
 189     /// Literal starts with "0o".
 190     Octal,
 191     /// Literal starts with "0x".
 192     Hexadecimal,
 193     /// Literal doesn't contain a prefix.
 194     Decimal,
 195 }
 196
 197 /// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
 198 /// but shebang isn't a part of rust syntax.
 199 pub fn strip_shebang(input: &str) -> Option<usize> {
 200     // Shebang must start with `#!` literally, without any preceding whitespace.
 201     // For simplicity we consider any line starting with `#!` a shebang,
 202     // regardless of restrictions put on shebangs by specific platforms.
 203     if let Some(input_tail) = input.strip_prefix("#!") {
 204         // Ok, this is a shebang but if the next non-whitespace token is `[`,
 205         // then it may be valid Rust code, so consider it Rust code.
 206         let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| {
 207             !matches!(
 208                 tok,
 209                 TokenKind::Whitespace
 210                     | TokenKind::LineComment { doc_style: None }
 211                     | TokenKind::BlockComment { doc_style: None, .. }
 212             )
 213         });
 214         if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
 215             // No other choice than to consider this a shebang.
 216             return Some(2 + input_tail.lines().next().unwrap_or_default().len());
 217         }
 218     }
 219     None
 220 }
 221
 222 /// Parses the first token from the provided input string.
 223 #[inline]
 224 pub fn first_token(input: &str) -> Token {
 225     debug_assert!(!input.is_empty());
 226     Cursor::new(input).advance_token()
 227 }
 228
 229 /// Validates a raw string literal. Used for getting more information about a
 230 /// problem with a `RawStr`/`RawByteStr` with a `None` field.
 231 #[inline]
 232 pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
 233     debug_assert!(!input.is_empty());
 234     let mut cursor = Cursor::new(input);
 235     // Move past the leading `r` or `br`.
 236     for _ in 0..prefix_len {
 237         cursor.bump().unwrap();
 238     }
 239     cursor.raw_double_quoted_string(prefix_len).map(|_| ())
 240 }
 241
 242 /// Creates an iterator that produces tokens from the input string.
 243 pub fn tokenize(input: &str) -> impl Iterator<Item = Token> + '_ {
 244     let mut cursor = Cursor::new(input);
 245     std::iter::from_fn(move || {
 246         if cursor.is_eof() {
 247             None
 248         } else {
 249             cursor.reset_len_consumed();
 250             Some(cursor.advance_token())
 251         }
 252     })
 253 }
 254
 255 /// True if `c` is considered a whitespace according to Rust language definition.
 256 /// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
 257 /// for definitions of these classes.
 258 pub fn is_whitespace(c: char) -> bool {
 259     // This is Pattern_White_Space.
 260     //
 261     // Note that this set is stable (ie, it doesn't change with different
 262     // Unicode versions), so it's ok to just hard-code the values.
 263
 264     matches!(
 265         c,
 266         // Usual ASCII suspects
 267         '\u{0009}'   // \t
 268         | '\u{000A}' // \n
 269         | '\u{000B}' // vertical tab
 270         | '\u{000C}' // form feed
 271         | '\u{000D}' // \r
 272         | '\u{0020}' // space
 273
 274         // NEXT LINE from latin1
 275         | '\u{0085}'
 276
 277         // Bidi markers
 278         | '\u{200E}' // LEFT-TO-RIGHT MARK
 279         | '\u{200F}' // RIGHT-TO-LEFT MARK
 280
 281         // Dedicated whitespace characters from Unicode
 282         | '\u{2028}' // LINE SEPARATOR
 283         | '\u{2029}' // PARAGRAPH SEPARATOR
 284     )
 285 }
 286
 287 /// True if `c` is valid as a first character of an identifier.
 288 /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
 289 /// a formal definition of valid identifier name.
 290 pub fn is_id_start(c: char) -> bool {
 291     // This is XID_Start OR '_' (which formally is not a XID_Start).
 292     c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
 293 }
 294
 295 /// True if `c` is valid as a non-first character of an identifier.
 296 /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
 297 /// a formal definition of valid identifier name.
 298 pub fn is_id_continue(c: char) -> bool {
 299     unicode_xid::UnicodeXID::is_xid_continue(c)
 300 }
 301
 302 /// The passed string is lexically an identifier.
 303 pub fn is_ident(string: &str) -> bool {
 304     let mut chars = string.chars();
 305     if let Some(start) = chars.next() {
 306         is_id_start(start) && chars.all(is_id_continue)
 307     } else {
 308         false
 309     }
 310 }
 311
 312 impl Cursor<'_> {
 313     /// Parses a token from the input string.
 314     fn advance_token(&mut self) -> Token {
 315         let first_char = self.bump().unwrap();
 316         let token_kind = match first_char {
 317             // Slash, comment or block comment.
 318             '/' => match self.first() {
 319                 '/' => self.line_comment(),
 320                 '*' => self.block_comment(),
 321                 _ => Slash,
 322             },
 323
 324             // Whitespace sequence.
 325             c if is_whitespace(c) => self.whitespace(),
 326
 327             // Raw identifier, raw string literal or identifier.
 328             'r' => match (self.first(), self.second()) {
 329                 ('#', c1) if is_id_start(c1) => self.raw_ident(),
 330                 ('#', _) | ('"', _) => {
 331                     let res = self.raw_double_quoted_string(1);
 332                     let suffix_start = self.len_consumed();
 333                     if res.is_ok() {
 334                         self.eat_literal_suffix();
 335                     }
 336                     let kind = RawStr { n_hashes: res.ok() };
 337                     Literal { kind, suffix_start }
 338                 }
 339                 _ => self.ident_or_unknown_prefix(),
 340             },
 341
 342             // Byte literal, byte string literal, raw byte string literal or identifier.
 343             'b' => match (self.first(), self.second()) {
 344                 ('\'', _) => {
 345                     self.bump();
 346                     let terminated = self.single_quoted_string();
 347                     let suffix_start = self.len_consumed();
 348                     if terminated {
 349                         self.eat_literal_suffix();
 350                     }
 351                     let kind = Byte { terminated };
 352                     Literal { kind, suffix_start }
 353                 }
 354                 ('"', _) => {
 355                     self.bump();
 356                     let terminated = self.double_quoted_string();
 357                     let suffix_start = self.len_consumed();
 358                     if terminated {
 359                         self.eat_literal_suffix();
 360                     }
 361                     let kind = ByteStr { terminated };
 362                     Literal { kind, suffix_start }
 363                 }
 364                 ('r', '"') | ('r', '#') => {
 365                     self.bump();
 366                     let res = self.raw_double_quoted_string(2);
 367                     let suffix_start = self.len_consumed();
 368                     if res.is_ok() {
 369                         self.eat_literal_suffix();
 370                     }
 371                     let kind = RawByteStr { n_hashes: res.ok() };
 372                     Literal { kind, suffix_start }
 373                 }
 374                 _ => self.ident_or_unknown_prefix(),
 375             },
 376
 377             // Identifier (this should be checked after other variant that can
 378             // start as identifier).
 379             c if is_id_start(c) => self.ident_or_unknown_prefix(),
 380
 381             // Numeric literal.
 382             c @ '0'..='9' => {
 383                 let literal_kind = self.number(c);
 384                 let suffix_start = self.len_consumed();
 385                 self.eat_literal_suffix();
 386                 TokenKind::Literal { kind: literal_kind, suffix_start }
 387             }
 388
 389             // One-symbol tokens.
 390             ';' => Semi,
 391             ',' => Comma,
 392             '.' => Dot,
 393             '(' => OpenParen,
 394             ')' => CloseParen,
 395             '{' => OpenBrace,
 396             '}' => CloseBrace,
 397             '[' => OpenBracket,
 398             ']' => CloseBracket,
 399             '@' => At,
 400             '#' => Pound,
 401             '~' => Tilde,
 402             '?' => Question,
 403             ':' => Colon,
 404             '$' => Dollar,
 405             '=' => Eq,
 406             '!' => Bang,
 407             '<' => Lt,
 408             '>' => Gt,
 409             '-' => Minus,
 410             '&' => And,
 411             '|' => Or,
 412             '+' => Plus,
 413             '*' => Star,
 414             '^' => Caret,
 415             '%' => Percent,
 416
 417             // Lifetime or character literal.
 418             '\'' => self.lifetime_or_char(),
 419
 420             // String literal.
 421             '"' => {
 422                 let terminated = self.double_quoted_string();
 423                 let suffix_start = self.len_consumed();
 424                 if terminated {
 425                     self.eat_literal_suffix();
 426                 }
 427                 let kind = Str { terminated };
 428                 Literal { kind, suffix_start }
 429             }
 430             // Identifier starting with an emoji. Only lexed for graceful error recovery.
 431             c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => {
 432                 self.fake_ident_or_unknown_prefix()
 433             }
 434             _ => Unknown,
 435         };
 436         Token::new(token_kind, self.len_consumed())
 437     }
 438
 439     fn line_comment(&mut self) -> TokenKind {
 440         debug_assert!(self.prev() == '/' && self.first() == '/');
 441         self.bump();
 442
 443         let doc_style = match self.first() {
 444             // `//!` is an inner line doc comment.
 445             '!' => Some(DocStyle::Inner),
 446             // `////` (more than 3 slashes) is not considered a doc comment.
 447             '/' if self.second() != '/' => Some(DocStyle::Outer),
 448             _ => None,
 449         };
 450
 451         self.eat_while(|c| c != '\n');
 452         LineComment { doc_style }
 453     }
 454
 455     fn block_comment(&mut self) -> TokenKind {
 456         debug_assert!(self.prev() == '/' && self.first() == '*');
 457         self.bump();
 458
 459         let doc_style = match self.first() {
 460             // `/*!` is an inner block doc comment.
 461             '!' => Some(DocStyle::Inner),
 462             // `/***` (more than 2 stars) is not considered a doc comment.
 463             // `/**/` is not considered a doc comment.
 464             '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
 465             _ => None,
 466         };
 467
 468         let mut depth = 1usize;
 469         while let Some(c) = self.bump() {
 470             match c {
 471                 '/' if self.first() == '*' => {
 472                     self.bump();
 473                     depth += 1;
 474                 }
 475                 '*' if self.first() == '/' => {
 476                     self.bump();
 477                     depth -= 1;
 478                     if depth == 0 {
 479                         // This block comment is closed, so for a construction like "/* */ */"
 480                         // there will be a successfully parsed block comment "/* */"
 481                         // and " */" will be processed separately.
 482                         break;
 483                     }
 484                 }
 485                 _ => (),
 486             }
 487         }
 488
 489         BlockComment { doc_style, terminated: depth == 0 }
 490     }
 491
 492     fn whitespace(&mut self) -> TokenKind {
 493         debug_assert!(is_whitespace(self.prev()));
 494         self.eat_while(is_whitespace);
 495         Whitespace
 496     }
 497
 498     fn raw_ident(&mut self) -> TokenKind {
 499         debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
 500         // Eat "#" symbol.
 501         self.bump();
 502         // Eat the identifier part of RawIdent.
 503         self.eat_identifier();
 504         RawIdent
 505     }
 506
 507     fn ident_or_unknown_prefix(&mut self) -> TokenKind {
 508         debug_assert!(is_id_start(self.prev()));
 509         // Start is already eaten, eat the rest of identifier.
 510         self.eat_while(is_id_continue);
 511         // Known prefixes must have been handled earlier. So if
 512         // we see a prefix here, it is definitely an unknown prefix.
 513         match self.first() {
 514             '#' | '"' | '\'' => UnknownPrefix,
 515             c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => {
 516                 self.fake_ident_or_unknown_prefix()
 517             }
 518             _ => Ident,
 519         }
 520     }
 521
 522     fn fake_ident_or_unknown_prefix(&mut self) -> TokenKind {
 523         // Start is already eaten, eat the rest of identifier.
 524         self.eat_while(|c| {
 525             unicode_xid::UnicodeXID::is_xid_continue(c)
 526                 || (!c.is_ascii() && unic_emoji_char::is_emoji(c))
 527                 || c == '\u{200d}'
 528         });
 529         // Known prefixes must have been handled earlier. So if
 530         // we see a prefix here, it is definitely an unknown prefix.
 531         match self.first() {
 532             '#' | '"' | '\'' => UnknownPrefix,
 533             _ => InvalidIdent,
 534         }
 535     }
 536
 537     fn number(&mut self, first_digit: char) -> LiteralKind {
 538         debug_assert!('0' <= self.prev() && self.prev() <= '9');
 539         let mut base = Base::Decimal;
 540         if first_digit == '0' {
 541             // Attempt to parse encoding base.
 542             let has_digits = match self.first() {
 543                 'b' => {
 544                     base = Base::Binary;
 545                     self.bump();
 546                     self.eat_decimal_digits()
 547                 }
 548                 'o' => {
 549                     base = Base::Octal;
 550                     self.bump();
 551                     self.eat_decimal_digits()
 552                 }
 553                 'x' => {
 554                     base = Base::Hexadecimal;
 555                     self.bump();
 556                     self.eat_hexadecimal_digits()
 557                 }
 558                 // Not a base prefix.
 559                 '0'..='9' | '_' | '.' | 'e' | 'E' => {
 560                     self.eat_decimal_digits();
 561                     true
 562                 }
 563                 // Just a 0.
 564                 _ => return Int { base, empty_int: false },
 565             };
 566             // Base prefix was provided, but there were no digits
 567             // after it, e.g. "0x".
 568             if !has_digits {
 569                 return Int { base, empty_int: true };
 570             }
 571         } else {
 572             // No base prefix, parse number in the usual way.
 573             self.eat_decimal_digits();
 574         };
 575
 576         match self.first() {
 577             // Don't be greedy if this is actually an
 578             // integer literal followed by field/method access or a range pattern
 579             // (`0..2` and `12.foo()`)
 580             '.' if self.second() != '.' && !is_id_start(self.second()) => {
 581                 // might have stuff after the ., and if it does, it needs to start
 582                 // with a number
 583                 self.bump();
 584                 let mut empty_exponent = false;
 585                 if self.first().is_digit(10) {
 586                     self.eat_decimal_digits();
 587                     match self.first() {
 588                         'e' | 'E' => {
 589                             self.bump();
 590                             empty_exponent = !self.eat_float_exponent();
 591                         }
 592                         _ => (),
 593                     }
 594                 }
 595                 Float { base, empty_exponent }
 596             }
 597             'e' | 'E' => {
 598                 self.bump();
 599                 let empty_exponent = !self.eat_float_exponent();
 600                 Float { base, empty_exponent }
 601             }
 602             _ => Int { base, empty_int: false },
 603         }
 604     }
 605
 606     fn lifetime_or_char(&mut self) -> TokenKind {
 607         debug_assert!(self.prev() == '\'');
 608
 609         let can_be_a_lifetime = if self.second() == '\'' {
 610             // It's surely not a lifetime.
 611             false
 612         } else {
 613             // If the first symbol is valid for identifier, it can be a lifetime.
 614             // Also check if it's a number for a better error reporting (so '0 will
 615             // be reported as invalid lifetime and not as unterminated char literal).
 616             is_id_start(self.first()) || self.first().is_digit(10)
 617         };
 618
 619         if !can_be_a_lifetime {
 620             let terminated = self.single_quoted_string();
 621             let suffix_start = self.len_consumed();
 622             if terminated {
 623                 self.eat_literal_suffix();
 624             }
 625             let kind = Char { terminated };
 626             return Literal { kind, suffix_start };
 627         }
 628
 629         // Either a lifetime or a character literal with
 630         // length greater than 1.
 631
 632         let starts_with_number = self.first().is_digit(10);
 633
 634         // Skip the literal contents.
 635         // First symbol can be a number (which isn't a valid identifier start),
 636         // so skip it without any checks.
 637         self.bump();
 638         self.eat_while(is_id_continue);
 639
 640         // Check if after skipping literal contents we've met a closing
 641         // single quote (which means that user attempted to create a
 642         // string with single quotes).
 643         if self.first() == '\'' {
 644             self.bump();
 645             let kind = Char { terminated: true };
 646             Literal { kind, suffix_start: self.len_consumed() }
 647         } else {
 648             Lifetime { starts_with_number }
 649         }
 650     }
 651
 652     fn single_quoted_string(&mut self) -> bool {
 653         debug_assert!(self.prev() == '\'');
 654         // Check if it's a one-symbol literal.
 655         if self.second() == '\'' && self.first() != '\\' {
 656             self.bump();
 657             self.bump();
 658             return true;
 659         }
 660
 661         // Literal has more than one symbol.
 662
 663         // Parse until either quotes are terminated or error is detected.
 664         loop {
 665             match self.first() {
 666                 // Quotes are terminated, finish parsing.
 667                 '\'' => {
 668                     self.bump();
 669                     return true;
 670                 }
 671                 // Probably beginning of the comment, which we don't want to include
 672                 // to the error report.
 673                 '/' => break,
 674                 // Newline without following '\'' means unclosed quote, stop parsing.
 675                 '\n' if self.second() != '\'' => break,
 676                 // End of file, stop parsing.
 677                 EOF_CHAR if self.is_eof() => break,
 678                 // Escaped slash is considered one character, so bump twice.
 679                 '\\' => {
 680                     self.bump();
 681                     self.bump();
 682                 }
 683                 // Skip the character.
 684                 _ => {
 685                     self.bump();
 686                 }
 687             }
 688         }
 689         // String was not terminated.
 690         false
 691     }
 692
 693     /// Eats double-quoted string and returns true
 694     /// if string is terminated.
 695     fn double_quoted_string(&mut self) -> bool {
 696         debug_assert!(self.prev() == '"');
 697         while let Some(c) = self.bump() {
 698             match c {
 699                 '"' => {
 700                     return true;
 701                 }
 702                 '\\' if self.first() == '\\' || self.first() == '"' => {
 703                     // Bump again to skip escaped character.
 704                     self.bump();
 705                 }
 706                 _ => (),
 707             }
 708         }
 709         // End of file reached.
 710         false
 711     }
 712
 713     /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
 714     fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
 715         // Wrap the actual function to handle the error with too many hashes.
 716         // This way, it eats the whole raw string.
 717         let n_hashes = self.raw_string_unvalidated(prefix_len)?;
 718         // Only up to 255 `#`s are allowed in raw strings
 719         match u8::try_from(n_hashes) {
 720             Ok(num) => Ok(num),
 721             Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
 722         }
 723     }
 724
 725     fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
 726         debug_assert!(self.prev() == 'r');
 727         let start_pos = self.len_consumed();
 728         let mut possible_terminator_offset = None;
 729         let mut max_hashes = 0;
 730
 731         // Count opening '#' symbols.
 732         let mut eaten = 0;
 733         while self.first() == '#' {
 734             eaten += 1;
 735             self.bump();
 736         }
 737         let n_start_hashes = eaten;
 738
 739         // Check that string is started.
 740         match self.bump() {
 741             Some('"') => (),
 742             c => {
 743                 let c = c.unwrap_or(EOF_CHAR);
 744                 return Err(RawStrError::InvalidStarter { bad_char: c });
 745             }
 746         }
 747
 748         // Skip the string contents and on each '#' character met, check if this is
 749         // a raw string termination.
 750         loop {
 751             self.eat_while(|c| c != '"');
 752
 753             if self.is_eof() {
 754                 return Err(RawStrError::NoTerminator {
 755                     expected: n_start_hashes,
 756                     found: max_hashes,
 757                     possible_terminator_offset,
 758                 });
 759             }
 760
 761             // Eat closing double quote.
 762             self.bump();
 763
 764             // Check that amount of closing '#' symbols
 765             // is equal to the amount of opening ones.
 766             // Note that this will not consume extra trailing `#` characters:
 767             // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
 768             // followed by a `#` token.
 769             let mut n_end_hashes = 0;
 770             while self.first() == '#' && n_end_hashes < n_start_hashes {
 771                 n_end_hashes += 1;
 772                 self.bump();
 773             }
 774
 775             if n_end_hashes == n_start_hashes {
 776                 return Ok(n_start_hashes);
 777             } else if n_end_hashes > max_hashes {
 778                 // Keep track of possible terminators to give a hint about
 779                 // where there might be a missing terminator
 780                 possible_terminator_offset =
 781                     Some(self.len_consumed() - start_pos - n_end_hashes + prefix_len);
 782                 max_hashes = n_end_hashes;
 783             }
 784         }
 785     }
 786
 787     fn eat_decimal_digits(&mut self) -> bool {
 788         let mut has_digits = false;
 789         loop {
 790             match self.first() {
 791                 '_' => {
 792                     self.bump();
 793                 }
 794                 '0'..='9' => {
 795                     has_digits = true;
 796                     self.bump();
 797                 }
 798                 _ => break,
 799             }
 800         }
 801         has_digits
 802     }
 803
 804     fn eat_hexadecimal_digits(&mut self) -> bool {
 805         let mut has_digits = false;
 806         loop {
 807             match self.first() {
 808                 '_' => {
 809                     self.bump();
 810                 }
 811                 '0'..='9' | 'a'..='f' | 'A'..='F' => {
 812                     has_digits = true;
 813                     self.bump();
 814                 }
 815                 _ => break,
 816             }
 817         }
 818         has_digits
 819     }
 820
 821     /// Eats the float exponent. Returns true if at least one digit was met,
 822     /// and returns false otherwise.
 823     fn eat_float_exponent(&mut self) -> bool {
 824         debug_assert!(self.prev() == 'e' || self.prev() == 'E');
 825         if self.first() == '-' || self.first() == '+' {
 826             self.bump();
 827         }
 828         self.eat_decimal_digits()
 829     }
 830
 831     // Eats the suffix of the literal, e.g. "_u8".
 832     fn eat_literal_suffix(&mut self) {
 833         self.eat_identifier();
 834     }
 835
 836     // Eats the identifier.
 837     fn eat_identifier(&mut self) {
 838         if !is_id_start(self.first()) {
 839             return;
 840         }
 841         self.bump();
 842
 843         self.eat_while(is_id_continue);
 844     }
 845 }