compiler/rustc_lexer/src/lib.rs

   1 //! Low-level Rust lexer.
   2 //!
   3 //! The idea with `rustc_lexer` is to make a reusable library,
   4 //! by separating out pure lexing and rustc-specific concerns, like spans,
   5 //! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
   6 //! produces simple tokens which are a pair of type-tag and a bit of original text,
   7 //! and does not report errors, instead storing them as flags on the token.
   8 //!
   9 //! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
  10 //! For that see [`rustc_parse::lexer`], which converts this basic token stream
  11 //! into wide tokens used by actual parser.
  12 //!
  13 //! The purpose of this crate is to convert raw sources into a labeled sequence
  14 //! of well-known token types, so building an actual Rust token stream will
  15 //! be easier.
  16 //!
  17 //! The main entity of this crate is the [`TokenKind`] enum which represents common
  18 //! lexeme types.
  19 //!
  20 //! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
  21 #![deny(rustc::untranslatable_diagnostic)]
  22 #![deny(rustc::diagnostic_outside_of_impl)]
  23 // We want to be able to build this crate with a stable compiler, so no
  24 // `#![feature]` attributes should be added.
  25
  26 mod cursor;
  27 pub mod unescape;
  28
  29 #[cfg(test)]
  30 mod tests;
  31
  32 pub use crate::cursor::Cursor;
  33
  34 use self::LiteralKind::*;
  35 use self::TokenKind::*;
  36 use crate::cursor::EOF_CHAR;
  37
  38 /// Parsed token.
  39 /// It doesn't contain information about data that has been parsed,
  40 /// only the type of the token and its size.
  41 #[derive(Debug)]
  42 pub struct Token {
  43     pub kind: TokenKind,
  44     pub len: u32,
  45 }
  46
  47 impl Token {
  48     fn new(kind: TokenKind, len: u32) -> Token {
  49         Token { kind, len }
  50     }
  51 }
  52
  53 /// Enum representing common lexeme types.
  54 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
  55 pub enum TokenKind {
  56     // Multi-char tokens:
  57     /// "// comment"
  58     LineComment { doc_style: Option<DocStyle> },
  59
  60     /// `/* block comment */`
  61     ///
  62     /// Block comments can be recursive, so a sequence like `/* /* */`
  63     /// will not be considered terminated and will result in a parsing error.
  64     BlockComment { doc_style: Option<DocStyle>, terminated: bool },
  65
  66     /// Any whitespace character sequence.
  67     Whitespace,
  68
  69     /// "ident" or "continue"
  70     ///
  71     /// At this step, keywords are also considered identifiers.
  72     Ident,
  73
  74     /// Like the above, but containing invalid unicode codepoints.
  75     InvalidIdent,
  76
  77     /// "r#ident"
  78     RawIdent,
  79
  80     /// An unknown prefix, like `foo#`, `foo'`, `foo"`.
  81     ///
  82     /// Note that only the
  83     /// prefix (`foo`) is included in the token, not the separator (which is
  84     /// lexed as its own distinct token). In Rust 2021 and later, reserved
  85     /// prefixes are reported as errors; in earlier editions, they result in a
  86     /// (allowed by default) lint, and are treated as regular identifier
  87     /// tokens.
  88     UnknownPrefix,
  89
  90     /// Examples: `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
  91     /// suffix, but may be present here on string and float literals. Users of
  92     /// this type will need to check for and reject that case.
  93     ///
  94     /// See [LiteralKind] for more details.
  95     Literal { kind: LiteralKind, suffix_start: u32 },
  96
  97     /// "'a"
  98     Lifetime { starts_with_number: bool },
  99
 100     // One-char tokens:
 101     /// ";"
 102     Semi,
 103     /// ","
 104     Comma,
 105     /// "."
 106     Dot,
 107     /// "("
 108     OpenParen,
 109     /// ")"
 110     CloseParen,
 111     /// "{"
 112     OpenBrace,
 113     /// "}"
 114     CloseBrace,
 115     /// "["
 116     OpenBracket,
 117     /// "]"
 118     CloseBracket,
 119     /// "@"
 120     At,
 121     /// "#"
 122     Pound,
 123     /// "~"
 124     Tilde,
 125     /// "?"
 126     Question,
 127     /// ":"
 128     Colon,
 129     /// "$"
 130     Dollar,
 131     /// "="
 132     Eq,
 133     /// "!"
 134     Bang,
 135     /// "<"
 136     Lt,
 137     /// ">"
 138     Gt,
 139     /// "-"
 140     Minus,
 141     /// "&"
 142     And,
 143     /// "|"
 144     Or,
 145     /// "+"
 146     Plus,
 147     /// "*"
 148     Star,
 149     /// "/"
 150     Slash,
 151     /// "^"
 152     Caret,
 153     /// "%"
 154     Percent,
 155
 156     /// Unknown token, not expected by the lexer, e.g. "№"
 157     Unknown,
 158
 159     /// End of input.
 160     Eof,
 161 }
 162
 163 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 164 pub enum DocStyle {
 165     Outer,
 166     Inner,
 167 }
 168
 169 // Note that the suffix is *not* considered when deciding the `LiteralKind` in
 170 // this type. This means that float literals like `1f32` are classified by this
 171 // type as `Int`. (Compare against `rustc_ast::token::LitKind` and
 172 // `rustc_ast::ast::LitKind.)
 173 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 174 pub enum LiteralKind {
 175     /// "12_u8", "0o100", "0b120i99", "1f32".
 176     Int { base: Base, empty_int: bool },
 177     /// "12.34f32", "1e3", but not "1f32`.
 178     Float { base: Base, empty_exponent: bool },
 179     /// "'a'", "'\\'", "'''", "';"
 180     Char { terminated: bool },
 181     /// "b'a'", "b'\\'", "b'''", "b';"
 182     Byte { terminated: bool },
 183     /// ""abc"", ""abc"
 184     Str { terminated: bool },
 185     /// "b"abc"", "b"abc"
 186     ByteStr { terminated: bool },
 187     /// "r"abc"", "r#"abc"#", "r####"ab"###"c"####", "r#"a". `None` indicates
 188     /// an invalid literal.
 189     RawStr { n_hashes: Option<u8> },
 190     /// "br"abc"", "br#"abc"#", "br####"ab"###"c"####", "br#"a". `None`
 191     /// indicates an invalid literal.
 192     RawByteStr { n_hashes: Option<u8> },
 193 }
 194
 195 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 196 pub enum RawStrError {
 197     /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
 198     InvalidStarter { bad_char: char },
 199     /// The string was not terminated, e.g. `r###"abcde"##`.
 200     /// `possible_terminator_offset` is the number of characters after `r` or
 201     /// `br` where they may have intended to terminate it.
 202     NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
 203     /// More than 255 `#`s exist.
 204     TooManyDelimiters { found: u32 },
 205 }
 206
 207 /// Base of numeric literal encoding according to its prefix.
 208 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
 209 pub enum Base {
 210     /// Literal starts with "0b".
 211     Binary = 2,
 212     /// Literal starts with "0o".
 213     Octal = 8,
 214     /// Literal doesn't contain a prefix.
 215     Decimal = 10,
 216     /// Literal starts with "0x".
 217     Hexadecimal = 16,
 218 }
 219
 220 /// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
 221 /// but shebang isn't a part of rust syntax.
 222 pub fn strip_shebang(input: &str) -> Option<usize> {
 223     // Shebang must start with `#!` literally, without any preceding whitespace.
 224     // For simplicity we consider any line starting with `#!` a shebang,
 225     // regardless of restrictions put on shebangs by specific platforms.
 226     if let Some(input_tail) = input.strip_prefix("#!") {
 227         // Ok, this is a shebang but if the next non-whitespace token is `[`,
 228         // then it may be valid Rust code, so consider it Rust code.
 229         let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| {
 230             !matches!(
 231                 tok,
 232                 TokenKind::Whitespace
 233                     | TokenKind::LineComment { doc_style: None }
 234                     | TokenKind::BlockComment { doc_style: None, .. }
 235             )
 236         });
 237         if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
 238             // No other choice than to consider this a shebang.
 239             return Some(2 + input_tail.lines().next().unwrap_or_default().len());
 240         }
 241     }
 242     None
 243 }
 244
 245 /// Validates a raw string literal. Used for getting more information about a
 246 /// problem with a `RawStr`/`RawByteStr` with a `None` field.
 247 #[inline]
 248 pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
 249     debug_assert!(!input.is_empty());
 250     let mut cursor = Cursor::new(input);
 251     // Move past the leading `r` or `br`.
 252     for _ in 0..prefix_len {
 253         cursor.bump().unwrap();
 254     }
 255     cursor.raw_double_quoted_string(prefix_len).map(|_| ())
 256 }
 257
 258 /// Creates an iterator that produces tokens from the input string.
 259 pub fn tokenize(input: &str) -> impl Iterator<Item = Token> + '_ {
 260     let mut cursor = Cursor::new(input);
 261     std::iter::from_fn(move || {
 262         let token = cursor.advance_token();
 263         if token.kind != TokenKind::Eof { Some(token) } else { None }
 264     })
 265 }
 266
 267 /// True if `c` is considered a whitespace according to Rust language definition.
 268 /// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
 269 /// for definitions of these classes.
 270 pub fn is_whitespace(c: char) -> bool {
 271     // This is Pattern_White_Space.
 272     //
 273     // Note that this set is stable (ie, it doesn't change with different
 274     // Unicode versions), so it's ok to just hard-code the values.
 275
 276     matches!(
 277         c,
 278         // Usual ASCII suspects
 279         '\u{0009}'   // \t
 280         | '\u{000A}' // \n
 281         | '\u{000B}' // vertical tab
 282         | '\u{000C}' // form feed
 283         | '\u{000D}' // \r
 284         | '\u{0020}' // space
 285
 286         // NEXT LINE from latin1
 287         | '\u{0085}'
 288
 289         // Bidi markers
 290         | '\u{200E}' // LEFT-TO-RIGHT MARK
 291         | '\u{200F}' // RIGHT-TO-LEFT MARK
 292
 293         // Dedicated whitespace characters from Unicode
 294         | '\u{2028}' // LINE SEPARATOR
 295         | '\u{2029}' // PARAGRAPH SEPARATOR
 296     )
 297 }
 298
 299 /// True if `c` is valid as a first character of an identifier.
 300 /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
 301 /// a formal definition of valid identifier name.
 302 pub fn is_id_start(c: char) -> bool {
 303     // This is XID_Start OR '_' (which formally is not a XID_Start).
 304     c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
 305 }
 306
 307 /// True if `c` is valid as a non-first character of an identifier.
 308 /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
 309 /// a formal definition of valid identifier name.
 310 pub fn is_id_continue(c: char) -> bool {
 311     unicode_xid::UnicodeXID::is_xid_continue(c)
 312 }
 313
 314 /// The passed string is lexically an identifier.
 315 pub fn is_ident(string: &str) -> bool {
 316     let mut chars = string.chars();
 317     if let Some(start) = chars.next() {
 318         is_id_start(start) && chars.all(is_id_continue)
 319     } else {
 320         false
 321     }
 322 }
 323
 324 impl Cursor<'_> {
 325     /// Parses a token from the input string.
 326     pub fn advance_token(&mut self) -> Token {
 327         let first_char = match self.bump() {
 328             Some(c) => c,
 329             None => return Token::new(TokenKind::Eof, 0),
 330         };
 331         let token_kind = match first_char {
 332             // Slash, comment or block comment.
 333             '/' => match self.first() {
 334                 '/' => self.line_comment(),
 335                 '*' => self.block_comment(),
 336                 _ => Slash,
 337             },
 338
 339             // Whitespace sequence.
 340             c if is_whitespace(c) => self.whitespace(),
 341
 342             // Raw identifier, raw string literal or identifier.
 343             'r' => match (self.first(), self.second()) {
 344                 ('#', c1) if is_id_start(c1) => self.raw_ident(),
 345                 ('#', _) | ('"', _) => {
 346                     let res = self.raw_double_quoted_string(1);
 347                     let suffix_start = self.pos_within_token();
 348                     if res.is_ok() {
 349                         self.eat_literal_suffix();
 350                     }
 351                     let kind = RawStr { n_hashes: res.ok() };
 352                     Literal { kind, suffix_start }
 353                 }
 354                 _ => self.ident_or_unknown_prefix(),
 355             },
 356
 357             // Byte literal, byte string literal, raw byte string literal or identifier.
 358             'b' => match (self.first(), self.second()) {
 359                 ('\'', _) => {
 360                     self.bump();
 361                     let terminated = self.single_quoted_string();
 362                     let suffix_start = self.pos_within_token();
 363                     if terminated {
 364                         self.eat_literal_suffix();
 365                     }
 366                     let kind = Byte { terminated };
 367                     Literal { kind, suffix_start }
 368                 }
 369                 ('"', _) => {
 370                     self.bump();
 371                     let terminated = self.double_quoted_string();
 372                     let suffix_start = self.pos_within_token();
 373                     if terminated {
 374                         self.eat_literal_suffix();
 375                     }
 376                     let kind = ByteStr { terminated };
 377                     Literal { kind, suffix_start }
 378                 }
 379                 ('r', '"') | ('r', '#') => {
 380                     self.bump();
 381                     let res = self.raw_double_quoted_string(2);
 382                     let suffix_start = self.pos_within_token();
 383                     if res.is_ok() {
 384                         self.eat_literal_suffix();
 385                     }
 386                     let kind = RawByteStr { n_hashes: res.ok() };
 387                     Literal { kind, suffix_start }
 388                 }
 389                 _ => self.ident_or_unknown_prefix(),
 390             },
 391
 392             // Identifier (this should be checked after other variant that can
 393             // start as identifier).
 394             c if is_id_start(c) => self.ident_or_unknown_prefix(),
 395
 396             // Numeric literal.
 397             c @ '0'..='9' => {
 398                 let literal_kind = self.number(c);
 399                 let suffix_start = self.pos_within_token();
 400                 self.eat_literal_suffix();
 401                 TokenKind::Literal { kind: literal_kind, suffix_start }
 402             }
 403
 404             // One-symbol tokens.
 405             ';' => Semi,
 406             ',' => Comma,
 407             '.' => Dot,
 408             '(' => OpenParen,
 409             ')' => CloseParen,
 410             '{' => OpenBrace,
 411             '}' => CloseBrace,
 412             '[' => OpenBracket,
 413             ']' => CloseBracket,
 414             '@' => At,
 415             '#' => Pound,
 416             '~' => Tilde,
 417             '?' => Question,
 418             ':' => Colon,
 419             '$' => Dollar,
 420             '=' => Eq,
 421             '!' => Bang,
 422             '<' => Lt,
 423             '>' => Gt,
 424             '-' => Minus,
 425             '&' => And,
 426             '|' => Or,
 427             '+' => Plus,
 428             '*' => Star,
 429             '^' => Caret,
 430             '%' => Percent,
 431
 432             // Lifetime or character literal.
 433             '\'' => self.lifetime_or_char(),
 434
 435             // String literal.
 436             '"' => {
 437                 let terminated = self.double_quoted_string();
 438                 let suffix_start = self.pos_within_token();
 439                 if terminated {
 440                     self.eat_literal_suffix();
 441                 }
 442                 let kind = Str { terminated };
 443                 Literal { kind, suffix_start }
 444             }
 445             // Identifier starting with an emoji. Only lexed for graceful error recovery.
 446             c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => {
 447                 self.fake_ident_or_unknown_prefix()
 448             }
 449             _ => Unknown,
 450         };
 451         let res = Token::new(token_kind, self.pos_within_token());
 452         self.reset_pos_within_token();
 453         res
 454     }
 455
 456     fn line_comment(&mut self) -> TokenKind {
 457         debug_assert!(self.prev() == '/' && self.first() == '/');
 458         self.bump();
 459
 460         let doc_style = match self.first() {
 461             // `//!` is an inner line doc comment.
 462             '!' => Some(DocStyle::Inner),
 463             // `////` (more than 3 slashes) is not considered a doc comment.
 464             '/' if self.second() != '/' => Some(DocStyle::Outer),
 465             _ => None,
 466         };
 467
 468         self.eat_while(|c| c != '\n');
 469         LineComment { doc_style }
 470     }
 471
 472     fn block_comment(&mut self) -> TokenKind {
 473         debug_assert!(self.prev() == '/' && self.first() == '*');
 474         self.bump();
 475
 476         let doc_style = match self.first() {
 477             // `/*!` is an inner block doc comment.
 478             '!' => Some(DocStyle::Inner),
 479             // `/***` (more than 2 stars) is not considered a doc comment.
 480             // `/**/` is not considered a doc comment.
 481             '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
 482             _ => None,
 483         };
 484
 485         let mut depth = 1usize;
 486         while let Some(c) = self.bump() {
 487             match c {
 488                 '/' if self.first() == '*' => {
 489                     self.bump();
 490                     depth += 1;
 491                 }
 492                 '*' if self.first() == '/' => {
 493                     self.bump();
 494                     depth -= 1;
 495                     if depth == 0 {
 496                         // This block comment is closed, so for a construction like "/* */ */"
 497                         // there will be a successfully parsed block comment "/* */"
 498                         // and " */" will be processed separately.
 499                         break;
 500                     }
 501                 }
 502                 _ => (),
 503             }
 504         }
 505
 506         BlockComment { doc_style, terminated: depth == 0 }
 507     }
 508
 509     fn whitespace(&mut self) -> TokenKind {
 510         debug_assert!(is_whitespace(self.prev()));
 511         self.eat_while(is_whitespace);
 512         Whitespace
 513     }
 514
 515     fn raw_ident(&mut self) -> TokenKind {
 516         debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
 517         // Eat "#" symbol.
 518         self.bump();
 519         // Eat the identifier part of RawIdent.
 520         self.eat_identifier();
 521         RawIdent
 522     }
 523
 524     fn ident_or_unknown_prefix(&mut self) -> TokenKind {
 525         debug_assert!(is_id_start(self.prev()));
 526         // Start is already eaten, eat the rest of identifier.
 527         self.eat_while(is_id_continue);
 528         // Known prefixes must have been handled earlier. So if
 529         // we see a prefix here, it is definitely an unknown prefix.
 530         match self.first() {
 531             '#' | '"' | '\'' => UnknownPrefix,
 532             c if !c.is_ascii() && unic_emoji_char::is_emoji(c) => {
 533                 self.fake_ident_or_unknown_prefix()
 534             }
 535             _ => Ident,
 536         }
 537     }
 538
 539     fn fake_ident_or_unknown_prefix(&mut self) -> TokenKind {
 540         // Start is already eaten, eat the rest of identifier.
 541         self.eat_while(|c| {
 542             unicode_xid::UnicodeXID::is_xid_continue(c)
 543                 || (!c.is_ascii() && unic_emoji_char::is_emoji(c))
 544                 || c == '\u{200d}'
 545         });
 546         // Known prefixes must have been handled earlier. So if
 547         // we see a prefix here, it is definitely an unknown prefix.
 548         match self.first() {
 549             '#' | '"' | '\'' => UnknownPrefix,
 550             _ => InvalidIdent,
 551         }
 552     }
 553
 554     fn number(&mut self, first_digit: char) -> LiteralKind {
 555         debug_assert!('0' <= self.prev() && self.prev() <= '9');
 556         let mut base = Base::Decimal;
 557         if first_digit == '0' {
 558             // Attempt to parse encoding base.
 559             let has_digits = match self.first() {
 560                 'b' => {
 561                     base = Base::Binary;
 562                     self.bump();
 563                     self.eat_decimal_digits()
 564                 }
 565                 'o' => {
 566                     base = Base::Octal;
 567                     self.bump();
 568                     self.eat_decimal_digits()
 569                 }
 570                 'x' => {
 571                     base = Base::Hexadecimal;
 572                     self.bump();
 573                     self.eat_hexadecimal_digits()
 574                 }
 575                 // Not a base prefix.
 576                 '0'..='9' | '_' | '.' | 'e' | 'E' => {
 577                     self.eat_decimal_digits();
 578                     true
 579                 }
 580                 // Just a 0.
 581                 _ => return Int { base, empty_int: false },
 582             };
 583             // Base prefix was provided, but there were no digits
 584             // after it, e.g. "0x".
 585             if !has_digits {
 586                 return Int { base, empty_int: true };
 587             }
 588         } else {
 589             // No base prefix, parse number in the usual way.
 590             self.eat_decimal_digits();
 591         };
 592
 593         match self.first() {
 594             // Don't be greedy if this is actually an
 595             // integer literal followed by field/method access or a range pattern
 596             // (`0..2` and `12.foo()`)
 597             '.' if self.second() != '.' && !is_id_start(self.second()) => {
 598                 // might have stuff after the ., and if it does, it needs to start
 599                 // with a number
 600                 self.bump();
 601                 let mut empty_exponent = false;
 602                 if self.first().is_digit(10) {
 603                     self.eat_decimal_digits();
 604                     match self.first() {
 605                         'e' | 'E' => {
 606                             self.bump();
 607                             empty_exponent = !self.eat_float_exponent();
 608                         }
 609                         _ => (),
 610                     }
 611                 }
 612                 Float { base, empty_exponent }
 613             }
 614             'e' | 'E' => {
 615                 self.bump();
 616                 let empty_exponent = !self.eat_float_exponent();
 617                 Float { base, empty_exponent }
 618             }
 619             _ => Int { base, empty_int: false },
 620         }
 621     }
 622
 623     fn lifetime_or_char(&mut self) -> TokenKind {
 624         debug_assert!(self.prev() == '\'');
 625
 626         let can_be_a_lifetime = if self.second() == '\'' {
 627             // It's surely not a lifetime.
 628             false
 629         } else {
 630             // If the first symbol is valid for identifier, it can be a lifetime.
 631             // Also check if it's a number for a better error reporting (so '0 will
 632             // be reported as invalid lifetime and not as unterminated char literal).
 633             is_id_start(self.first()) || self.first().is_digit(10)
 634         };
 635
 636         if !can_be_a_lifetime {
 637             let terminated = self.single_quoted_string();
 638             let suffix_start = self.pos_within_token();
 639             if terminated {
 640                 self.eat_literal_suffix();
 641             }
 642             let kind = Char { terminated };
 643             return Literal { kind, suffix_start };
 644         }
 645
 646         // Either a lifetime or a character literal with
 647         // length greater than 1.
 648
 649         let starts_with_number = self.first().is_digit(10);
 650
 651         // Skip the literal contents.
 652         // First symbol can be a number (which isn't a valid identifier start),
 653         // so skip it without any checks.
 654         self.bump();
 655         self.eat_while(is_id_continue);
 656
 657         // Check if after skipping literal contents we've met a closing
 658         // single quote (which means that user attempted to create a
 659         // string with single quotes).
 660         if self.first() == '\'' {
 661             self.bump();
 662             let kind = Char { terminated: true };
 663             Literal { kind, suffix_start: self.pos_within_token() }
 664         } else {
 665             Lifetime { starts_with_number }
 666         }
 667     }
 668
 669     fn single_quoted_string(&mut self) -> bool {
 670         debug_assert!(self.prev() == '\'');
 671         // Check if it's a one-symbol literal.
 672         if self.second() == '\'' && self.first() != '\\' {
 673             self.bump();
 674             self.bump();
 675             return true;
 676         }
 677
 678         // Literal has more than one symbol.
 679
 680         // Parse until either quotes are terminated or error is detected.
 681         loop {
 682             match self.first() {
 683                 // Quotes are terminated, finish parsing.
 684                 '\'' => {
 685                     self.bump();
 686                     return true;
 687                 }
 688                 // Probably beginning of the comment, which we don't want to include
 689                 // to the error report.
 690                 '/' => break,
 691                 // Newline without following '\'' means unclosed quote, stop parsing.
 692                 '\n' if self.second() != '\'' => break,
 693                 // End of file, stop parsing.
 694                 EOF_CHAR if self.is_eof() => break,
 695                 // Escaped slash is considered one character, so bump twice.
 696                 '\\' => {
 697                     self.bump();
 698                     self.bump();
 699                 }
 700                 // Skip the character.
 701                 _ => {
 702                     self.bump();
 703                 }
 704             }
 705         }
 706         // String was not terminated.
 707         false
 708     }
 709
 710     /// Eats double-quoted string and returns true
 711     /// if string is terminated.
 712     fn double_quoted_string(&mut self) -> bool {
 713         debug_assert!(self.prev() == '"');
 714         while let Some(c) = self.bump() {
 715             match c {
 716                 '"' => {
 717                     return true;
 718                 }
 719                 '\\' if self.first() == '\\' || self.first() == '"' => {
 720                     // Bump again to skip escaped character.
 721                     self.bump();
 722                 }
 723                 _ => (),
 724             }
 725         }
 726         // End of file reached.
 727         false
 728     }
 729
 730     /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
 731     fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
 732         // Wrap the actual function to handle the error with too many hashes.
 733         // This way, it eats the whole raw string.
 734         let n_hashes = self.raw_string_unvalidated(prefix_len)?;
 735         // Only up to 255 `#`s are allowed in raw strings
 736         match u8::try_from(n_hashes) {
 737             Ok(num) => Ok(num),
 738             Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
 739         }
 740     }
 741
 742     fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
 743         debug_assert!(self.prev() == 'r');
 744         let start_pos = self.pos_within_token();
 745         let mut possible_terminator_offset = None;
 746         let mut max_hashes = 0;
 747
 748         // Count opening '#' symbols.
 749         let mut eaten = 0;
 750         while self.first() == '#' {
 751             eaten += 1;
 752             self.bump();
 753         }
 754         let n_start_hashes = eaten;
 755
 756         // Check that string is started.
 757         match self.bump() {
 758             Some('"') => (),
 759             c => {
 760                 let c = c.unwrap_or(EOF_CHAR);
 761                 return Err(RawStrError::InvalidStarter { bad_char: c });
 762             }
 763         }
 764
 765         // Skip the string contents and on each '#' character met, check if this is
 766         // a raw string termination.
 767         loop {
 768             self.eat_while(|c| c != '"');
 769
 770             if self.is_eof() {
 771                 return Err(RawStrError::NoTerminator {
 772                     expected: n_start_hashes,
 773                     found: max_hashes,
 774                     possible_terminator_offset,
 775                 });
 776             }
 777
 778             // Eat closing double quote.
 779             self.bump();
 780
 781             // Check that amount of closing '#' symbols
 782             // is equal to the amount of opening ones.
 783             // Note that this will not consume extra trailing `#` characters:
 784             // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
 785             // followed by a `#` token.
 786             let mut n_end_hashes = 0;
 787             while self.first() == '#' && n_end_hashes < n_start_hashes {
 788                 n_end_hashes += 1;
 789                 self.bump();
 790             }
 791
 792             if n_end_hashes == n_start_hashes {
 793                 return Ok(n_start_hashes);
 794             } else if n_end_hashes > max_hashes {
 795                 // Keep track of possible terminators to give a hint about
 796                 // where there might be a missing terminator
 797                 possible_terminator_offset =
 798                     Some(self.pos_within_token() - start_pos - n_end_hashes + prefix_len);
 799                 max_hashes = n_end_hashes;
 800             }
 801         }
 802     }
 803
 804     fn eat_decimal_digits(&mut self) -> bool {
 805         let mut has_digits = false;
 806         loop {
 807             match self.first() {
 808                 '_' => {
 809                     self.bump();
 810                 }
 811                 '0'..='9' => {
 812                     has_digits = true;
 813                     self.bump();
 814                 }
 815                 _ => break,
 816             }
 817         }
 818         has_digits
 819     }
 820
 821     fn eat_hexadecimal_digits(&mut self) -> bool {
 822         let mut has_digits = false;
 823         loop {
 824             match self.first() {
 825                 '_' => {
 826                     self.bump();
 827                 }
 828                 '0'..='9' | 'a'..='f' | 'A'..='F' => {
 829                     has_digits = true;
 830                     self.bump();
 831                 }
 832                 _ => break,
 833             }
 834         }
 835         has_digits
 836     }
 837
 838     /// Eats the float exponent. Returns true if at least one digit was met,
 839     /// and returns false otherwise.
 840     fn eat_float_exponent(&mut self) -> bool {
 841         debug_assert!(self.prev() == 'e' || self.prev() == 'E');
 842         if self.first() == '-' || self.first() == '+' {
 843             self.bump();
 844         }
 845         self.eat_decimal_digits()
 846     }
 847
 848     // Eats the suffix of the literal, e.g. "u8".
 849     fn eat_literal_suffix(&mut self) {
 850         self.eat_identifier();
 851     }
 852
 853     // Eats the identifier. Note: succeeds on `_`, which isn't a valid
 854     // identifier.
 855     fn eat_identifier(&mut self) {
 856         if !is_id_start(self.first()) {
 857             return;
 858         }
 859         self.bump();
 860
 861         self.eat_while(is_id_continue);
 862     }
 863 }