src/vendor/minifier/src/js/token.rs

   1 // MIT License
   2 //
   3 // Copyright (c) 2018 Guillaume Gomez
   4 //
   5 // Permission is hereby granted, free of charge, to any person obtaining a copy
   6 // of this software and associated documentation files (the "Software"), to deal
   7 // in the Software without restriction, including without limitation the rights
   8 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 // copies of the Software, and to permit persons to whom the Software is
  10 // furnished to do so, subject to the following conditions:
  11 //
  12 // The above copyright notice and this permission notice shall be included in all
  13 // copies or substantial portions of the Software.
  14 //
  15 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21 // SOFTWARE.
  22
  23 use std::fmt;
  24 use std::iter::Peekable;
  25 use std::str::CharIndices;
  26
  27 pub trait MyTryFrom<T>: Sized {
  28     type Error;
  29     fn try_from(value: T) -> Result<Self, Self::Error>;
  30 }
  31
  32 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
  33 pub enum ReservedChar {
  34     Comma,
  35     OpenParenthese,
  36     CloseParenthese,
  37     OpenCurlyBrace,
  38     CloseCurlyBrace,
  39     OpenBracket,
  40     CloseBracket,
  41     Colon,
  42     SemiColon,
  43     Dot,
  44     Quote,
  45     DoubleQuote,
  46     ExclamationMark,
  47     QuestionMark,
  48     Slash,
  49     Modulo,
  50     Star,
  51     Minus,
  52     Plus,
  53     EqualSign,
  54     Backslash,
  55     Space,
  56     Tab,
  57     Backline,
  58     LessThan,
  59     SuperiorThan,
  60     Pipe,
  61     Ampersand,
  62 }
  63
  64 impl ReservedChar {
  65     fn is_useless(&self) -> bool {
  66         *self == ReservedChar::Space ||
  67         *self == ReservedChar::Tab ||
  68         *self == ReservedChar::Backline
  69     }
  70 }
  71
  72 impl fmt::Display for ReservedChar {
  73     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
  74         write!(f, "{}",
  75                match *self {
  76                    ReservedChar::Comma           => ',',
  77                    ReservedChar::OpenParenthese  => '(',
  78                    ReservedChar::CloseParenthese => ')',
  79                    ReservedChar::OpenCurlyBrace  => '{',
  80                    ReservedChar::CloseCurlyBrace => '}',
  81                    ReservedChar::OpenBracket     => '[',
  82                    ReservedChar::CloseBracket    => ']',
  83                    ReservedChar::Colon           => ':',
  84                    ReservedChar::SemiColon       => ';',
  85                    ReservedChar::Dot             => '.',
  86                    ReservedChar::Quote           => '\'',
  87                    ReservedChar::DoubleQuote     => '"',
  88                    ReservedChar::ExclamationMark => '!',
  89                    ReservedChar::QuestionMark    => '?',
  90                    ReservedChar::Slash           => '/',
  91                    ReservedChar::Modulo          => '%',
  92                    ReservedChar::Star            => '*',
  93                    ReservedChar::Minus           => '-',
  94                    ReservedChar::Plus            => '+',
  95                    ReservedChar::EqualSign       => '=',
  96                    ReservedChar::Backslash       => '\\',
  97                    ReservedChar::Space           => ' ',
  98                    ReservedChar::Tab             => '\t',
  99                    ReservedChar::Backline        => '\n',
 100                    ReservedChar::LessThan        => '<',
 101                    ReservedChar::SuperiorThan    => '>',
 102                    ReservedChar::Pipe            => '|',
 103                    ReservedChar::Ampersand       => '&',
 104                })
 105     }
 106 }
 107
 108 impl MyTryFrom<char> for ReservedChar {
 109     type Error = &'static str;
 110
 111     fn try_from(value: char) -> Result<ReservedChar, Self::Error> {
 112         match value {
 113             ','  => Ok(ReservedChar::Comma),
 114             '('  => Ok(ReservedChar::OpenParenthese),
 115             ')'  => Ok(ReservedChar::CloseParenthese),
 116             '{'  => Ok(ReservedChar::OpenCurlyBrace),
 117             '}'  => Ok(ReservedChar::CloseCurlyBrace),
 118             '['  => Ok(ReservedChar::OpenBracket),
 119             ']'  => Ok(ReservedChar::CloseBracket),
 120             ':'  => Ok(ReservedChar::Colon),
 121             ';'  => Ok(ReservedChar::SemiColon),
 122             '.'  => Ok(ReservedChar::Dot),
 123             '\'' => Ok(ReservedChar::Quote),
 124             '"'  => Ok(ReservedChar::DoubleQuote),
 125             '!'  => Ok(ReservedChar::ExclamationMark),
 126             '?'  => Ok(ReservedChar::QuestionMark),
 127             '/'  => Ok(ReservedChar::Slash),
 128             '%'  => Ok(ReservedChar::Modulo),
 129             '*'  => Ok(ReservedChar::Star),
 130             '-'  => Ok(ReservedChar::Minus),
 131             '+'  => Ok(ReservedChar::Plus),
 132             '='  => Ok(ReservedChar::EqualSign),
 133             '\\' => Ok(ReservedChar::Backslash),
 134             ' '  => Ok(ReservedChar::Space),
 135             '\t' => Ok(ReservedChar::Tab),
 136             '\n' |
 137             '\r' => Ok(ReservedChar::Backline),
 138             '<'  => Ok(ReservedChar::LessThan),
 139             '>'  => Ok(ReservedChar::SuperiorThan),
 140             '|'  => Ok(ReservedChar::Pipe),
 141             '&'  => Ok(ReservedChar::Ampersand),
 142             _    => Err("Unknown reserved char"),
 143         }
 144     }
 145 }
 146
 147 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 148 pub enum Keyword {
 149     Break,
 150     Case,
 151     Catch,
 152     Const,
 153     Continue,
 154     Default,
 155     Do,
 156     Else,
 157     False,
 158     Finally,
 159     Function,
 160     For,
 161     If,
 162     In,
 163     InstanceOf,
 164     New,
 165     Null,
 166     Private,
 167     Protected,
 168     Public,
 169     Return,
 170     Switch,
 171     This,
 172     Throw,
 173     True,
 174     Try,
 175     Typeof,
 176     Static,
 177     Var,
 178     While,
 179 }
 180
 181 fn get_required<'a>(next: &Token<'a>) -> Option<char> {
 182     match *next {
 183         Token::Keyword(_) | Token::Other(_) => Some(' '),
 184         _ => None,
 185     }
 186 }
 187
 188 impl fmt::Display for Keyword {
 189     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 190         write!(f, "{}",
 191                match *self {
 192                    Keyword::Break => "break",
 193                    Keyword::Case => "case",
 194                    Keyword::Catch => "catch",
 195                    Keyword::Const => "const",
 196                    Keyword::Continue => "continue",
 197                    Keyword::Default => "default",
 198                    Keyword::Do => "do",
 199                    Keyword::Else => "else",
 200                    Keyword::False => "false",
 201                    Keyword::Finally => "finally",
 202                    Keyword::Function => "function",
 203                    Keyword::For => "for",
 204                    Keyword::If => "if",
 205                    Keyword::In => "in",
 206                    Keyword::InstanceOf => "instanceof",
 207                    Keyword::New => "new",
 208                    Keyword::Null => "null",
 209                    Keyword::Private => "private",
 210                    Keyword::Protected => "protected",
 211                    Keyword::Public => "public",
 212                    Keyword::Return => "return",
 213                    Keyword::Switch => "switch",
 214                    Keyword::This => "this",
 215                    Keyword::Throw => "throw",
 216                    Keyword::True => "true",
 217                    Keyword::Try => "try",
 218                    Keyword::Typeof => "typeof",
 219                    Keyword::Static => "static",
 220                    Keyword::Var => "var",
 221                    Keyword::While => "while",
 222                })
 223     }
 224 }
 225
 226 impl<'a> MyTryFrom<&'a str> for Keyword {
 227     type Error = &'static str;
 228
 229     fn try_from(value: &str) -> Result<Keyword, Self::Error> {
 230         match value {
 231             "break" => Ok(Keyword::Break),
 232             "case" => Ok(Keyword::Case),
 233             "catch" => Ok(Keyword::Catch),
 234             "const" => Ok(Keyword::Const),
 235             "continue" => Ok(Keyword::Continue),
 236             "default" => Ok(Keyword::Default),
 237             "do" => Ok(Keyword::Do),
 238             "else" => Ok(Keyword::Else),
 239             "false" => Ok(Keyword::False),
 240             "finally" => Ok(Keyword::Finally),
 241             "function" => Ok(Keyword::Function),
 242             "for" => Ok(Keyword::For),
 243             "if" => Ok(Keyword::If),
 244             "in" => Ok(Keyword::In),
 245             "instanceof" => Ok(Keyword::InstanceOf),
 246             "new" => Ok(Keyword::New),
 247             "null" => Ok(Keyword::Null),
 248             "private" => Ok(Keyword::Private),
 249             "protected" => Ok(Keyword::Protected),
 250             "public" => Ok(Keyword::Public),
 251             "return" => Ok(Keyword::Return),
 252             "switch" => Ok(Keyword::Switch),
 253             "this" => Ok(Keyword::This),
 254             "throw" => Ok(Keyword::Throw),
 255             "true" => Ok(Keyword::True),
 256             "try" => Ok(Keyword::Try),
 257             "typeof" => Ok(Keyword::Typeof),
 258             "static" => Ok(Keyword::Static),
 259             "var" => Ok(Keyword::Var),
 260             "while" => Ok(Keyword::While),
 261             _ => Err("Unkown keyword"),
 262         }
 263     }
 264 }
 265
 266 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 267 pub enum Condition {
 268     And,
 269     Or,
 270     DifferentThan,
 271     SuperDifferentThan,
 272     EqualTo,
 273     SuperEqualTo,
 274     SuperiorThan,
 275     SuperiorOrEqualTo,
 276     InferiorThan,
 277     InferiorOrEqualTo,
 278 }
 279
 280 impl fmt::Display for Condition {
 281     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 282         write!(f, "{}",
 283                match *self {
 284                    Condition::And => "&&",
 285                    Condition::Or => "||",
 286                    Condition::DifferentThan => "!=",
 287                    Condition::SuperDifferentThan => "!==",
 288                    Condition::EqualTo => "==",
 289                    Condition::SuperEqualTo => "===",
 290                    Condition::SuperiorThan => ">",
 291                    Condition::SuperiorOrEqualTo => ">=",
 292                    Condition::InferiorThan => "<",
 293                    Condition::InferiorOrEqualTo => "<=",
 294                })
 295     }
 296 }
 297
 298 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 299 pub enum Operation {
 300     Addition,
 301     AdditionEqual,
 302     Subtract,
 303     SubtractEqual,
 304     Multiply,
 305     MultiplyEqual,
 306     Divide,
 307     DivideEqual,
 308     Modulo,
 309     ModuloEqual,
 310     Equal,
 311 }
 312
 313 impl fmt::Display for Operation {
 314     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 315         write!(f, "{}",
 316                match *self {
 317                    Operation::Addition => "+",
 318                    Operation::AdditionEqual => "+=",
 319                    Operation::Subtract => "-",
 320                    Operation::SubtractEqual => "-=",
 321                    Operation::Multiply => "*",
 322                    Operation::MultiplyEqual => "*=",
 323                    Operation::Divide => "/",
 324                    Operation::DivideEqual => "/=",
 325                    Operation::Modulo => "%",
 326                    Operation::ModuloEqual => "%=",
 327                    Operation::Equal => "=",
 328                })
 329     }
 330 }
 331
 332 impl MyTryFrom<ReservedChar> for Operation {
 333     type Error = &'static str;
 334
 335     fn try_from(value: ReservedChar) -> Result<Operation, Self::Error> {
 336         Ok(match value {
 337             ReservedChar::Plus => Operation::Addition,
 338             ReservedChar::Minus => Operation::Subtract,
 339             ReservedChar::Slash => Operation::Divide,
 340             ReservedChar::Star => Operation::Multiply,
 341             ReservedChar::Modulo => Operation::Modulo,
 342             _ => return Err("Unkown operation"),
 343         })
 344     }
 345 }
 346
 347 #[derive(Debug, PartialEq)]
 348 pub enum Token<'a> {
 349     Keyword(Keyword),
 350     Char(ReservedChar),
 351     String(&'a str),
 352     Comment(&'a str),
 353     License(&'a str),
 354     Other(&'a str),
 355     Regex {
 356         regex: &'a str,
 357         is_global: bool,
 358         is_interactive: bool,
 359     },
 360     Condition(Condition),
 361     Operation(Operation),
 362 }
 363
 364 impl<'a> fmt::Display for Token<'a> {
 365     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 366         match *self {
 367             Token::Keyword(x) => write!(f, "{}", x),
 368             Token::Char(x) => write!(f, "{}", x),
 369             Token::String(x) |
 370             Token::Comment(x) |
 371             Token::Other(x) => write!(f, "{}", x),
 372             Token::License(x) => write!(f, "/*!{}*/", x),
 373             Token::Regex { regex, is_global, is_interactive } => {
 374                 let x = write!(f, "/{}/", regex);
 375                 if is_global {
 376                     write!(f, "g")?;
 377                 }
 378                 if is_interactive {
 379                     write!(f, "i")?;
 380                 }
 381                 x
 382             }
 383             Token::Condition(x) => write!(f, "{}", x),
 384             Token::Operation(x) => write!(f, "{}", x),
 385         }
 386     }
 387 }
 388
 389 impl<'a> Token<'a> {
 390     fn is_comment(&self) -> bool {
 391         match *self {
 392             Token::Comment(_) => true,
 393             _ => false,
 394         }
 395     }
 396
 397     fn get_char(&self) -> Option<ReservedChar> {
 398         match *self {
 399             Token::Char(c) => Some(c),
 400             _ => None,
 401         }
 402     }
 403
 404     #[allow(dead_code)]
 405     fn is_condition(&self) -> bool {
 406         match *self {
 407             Token::Condition(_) => true,
 408             _ => false,
 409         }
 410     }
 411
 412     fn is_other(&self) -> bool {
 413         match *self {
 414             Token::Other(_) => true,
 415             _ => false,
 416         }
 417     }
 418
 419     fn is_white_character(&self) -> bool {
 420         match *self {
 421             Token::Char(c) => c.is_useless(),
 422             _ => false,
 423         }
 424     }
 425 }
 426
 427 fn get_line_comment<'a>(source: &'a str, iterator: &mut Peekable<CharIndices>,
 428                         start_pos: &mut usize) -> Option<Token<'a>> {
 429     *start_pos += 1;
 430     while let Some((pos, c)) = iterator.next() {
 431         if let Ok(c) = ReservedChar::try_from(c) {
 432             if c == ReservedChar::Backline {
 433                 let ret = Some(Token::Comment(&source[*start_pos..pos]));
 434                 *start_pos = pos;
 435                 return ret;
 436             }
 437         }
 438     }
 439     None
 440 }
 441
 442 fn get_regex<'a>(source: &'a str, iterator: &mut Peekable<CharIndices>,
 443                  start_pos: &mut usize) -> Option<Token<'a>> {
 444     *start_pos += 1;
 445     while let Some((pos, c)) = iterator.next() {
 446         if c == '\\' {
 447             // we skip next character
 448             iterator.next();
 449             continue
 450         }
 451         if let Ok(c) = ReservedChar::try_from(c) {
 452             if c == ReservedChar::Slash {
 453                 let mut is_global = false;
 454                 let mut is_interactive = false;
 455                 let mut add = 0;
 456                 loop {
 457                     match iterator.peek() {
 458                         Some((_, 'i')) => is_interactive = true,
 459                         Some((_, 'g')) => is_global = true,
 460                         _ => break,
 461                     };
 462                     iterator.next();
 463                     add += 1;
 464                 }
 465                 let ret = Some(Token::Regex {
 466                                    regex: &source[*start_pos..pos],
 467                                    is_interactive,
 468                                    is_global
 469                                });
 470                 *start_pos = pos + add;
 471                 return ret;
 472             }
 473         }
 474     }
 475     None
 476 }
 477
 478 fn get_comment<'a>(source: &'a str, iterator: &mut Peekable<CharIndices>,
 479                    start_pos: &mut usize) -> Option<Token<'a>> {
 480     let mut prev = ReservedChar::Quote;
 481     *start_pos += 1;
 482     let builder = if let Some((_, c)) = iterator.next() {
 483         if c == '!' {
 484             *start_pos += 1;
 485             Token::License
 486         } else {
 487             if let Ok(c) = ReservedChar::try_from(c) {
 488                 prev = c;
 489             }
 490             Token::Comment
 491         }
 492     } else {
 493         Token::Comment
 494     };
 495
 496     while let Some((pos, c)) = iterator.next() {
 497         if let Ok(c) = ReservedChar::try_from(c) {
 498             if c == ReservedChar::Slash && prev == ReservedChar::Star {
 499                 let ret = Some(builder(&source[*start_pos..pos - 1]));
 500                 *start_pos = pos;
 501                 return ret;
 502             }
 503             prev = c;
 504         } else {
 505             prev = ReservedChar::Space;
 506         }
 507     }
 508     None
 509 }
 510
 511 fn get_string<'a>(source: &'a str, iterator: &mut Peekable<CharIndices>, start_pos: &mut usize,
 512                   start: ReservedChar) -> Option<Token<'a>> {
 513     while let Some((pos, c)) = iterator.next() {
 514         if c == '\\' {
 515             // we skip next character
 516             iterator.next();
 517             continue
 518         }
 519         if let Ok(c) = ReservedChar::try_from(c) {
 520             if c == start {
 521                 let ret = Some(Token::String(&source[*start_pos..pos + 1]));
 522                 *start_pos = pos;
 523                 return ret;
 524             }
 525         }
 526     }
 527     None
 528 }
 529
 530 fn first_useful<'a>(v: &'a [Token<'a>]) -> Option<&'a Token<'a>> {
 531     for x in v.iter().rev() {
 532         if x.is_white_character() {
 533             continue
 534         }
 535         return Some(x);
 536     }
 537     None
 538 }
 539
 540 pub fn tokenize<'a>(source: &'a str) -> Tokens<'a> {
 541     let mut v = Vec::with_capacity(1000);
 542     let mut start = 0;
 543     let mut iterator = source.char_indices().peekable();
 544
 545     loop {
 546         let (mut pos, c) = match iterator.next() {
 547             Some(x) => x,
 548             None => break,
 549         };
 550         if let Ok(c) = ReservedChar::try_from(c) {
 551             if pos > start {
 552                 if let Ok(w) = Keyword::try_from(&source[start..pos]) {
 553                     v.push(Token::Keyword(w));
 554                 } else {
 555                     v.push(Token::Other(&source[start..pos]));
 556                 }
 557             }
 558             if c == ReservedChar::Quote || c == ReservedChar::DoubleQuote {
 559                 if let Some(s) = get_string(source, &mut iterator, &mut pos, c) {
 560                     v.push(s);
 561                 }
 562             } else if c == ReservedChar::Slash &&
 563                       *v.last().unwrap_or(&Token::Other("")) == Token::Operation(Operation::Divide) {
 564                 v.pop();
 565                 if let Some(s) = get_line_comment(source, &mut iterator, &mut pos) {
 566                     v.push(s);
 567                 }
 568             } else if c == ReservedChar::Slash &&
 569                       iterator.peek().is_some() &&
 570                       iterator.peek().unwrap().1 != '/' &&
 571                       iterator.peek().unwrap().1 != '*' &&
 572                       !first_useful(&v).unwrap_or(&Token::String("")).is_other() {
 573                 if let Some(r) = get_regex(source, &mut iterator, &mut pos) {
 574                     v.push(r);
 575                 }
 576             } else if c == ReservedChar::Star &&
 577                       *v.last().unwrap_or(&Token::Other("")) == Token::Operation(Operation::Divide) {
 578                 v.pop();
 579                 if let Some(s) = get_comment(source, &mut iterator, &mut pos) {
 580                     v.push(s);
 581                 }
 582             } else if c == ReservedChar::Pipe &&
 583                       *v.last().unwrap_or(&Token::Other("")) == Token::Char(ReservedChar::Pipe) {
 584                 v.pop();
 585                 v.push(Token::Condition(Condition::Or));
 586             } else if c == ReservedChar::Ampersand &&
 587                       *v.last().unwrap_or(&Token::Other("")) == Token::Char(ReservedChar::Ampersand) {
 588                 v.pop();
 589                 v.push(Token::Condition(Condition::And));
 590             } else if c == ReservedChar::EqualSign &&
 591                       *v.last().unwrap_or(&Token::Other("")) == Token::Char(ReservedChar::EqualSign) {
 592                 v.pop();
 593                 v.push(Token::Condition(Condition::EqualTo));
 594             } else if c == ReservedChar::EqualSign &&
 595                       *v.last().unwrap_or(&Token::Other("")) == Token::Condition(Condition::EqualTo) {
 596                 v.pop();
 597                 v.push(Token::Condition(Condition::SuperEqualTo));
 598             } else if c == ReservedChar::EqualSign &&
 599                       *v.last().unwrap_or(&Token::Other("")) == Token::Char(ReservedChar::ExclamationMark) {
 600                 v.pop();
 601                 v.push(Token::Condition(Condition::DifferentThan));
 602             } else if c == ReservedChar::EqualSign &&
 603                       *v.last().unwrap_or(&Token::Other("")) == Token::Condition(Condition::DifferentThan) {
 604                 v.pop();
 605                 v.push(Token::Condition(Condition::SuperDifferentThan));
 606             } else if c == ReservedChar::EqualSign &&
 607                       *v.last().unwrap_or(&Token::Other("")) == Token::Operation(Operation::Divide) {
 608                 v.pop();
 609                 v.push(Token::Operation(Operation::DivideEqual));
 610             } else if c == ReservedChar::EqualSign &&
 611                       *v.last().unwrap_or(&Token::Other("")) == Token::Operation(Operation::Multiply) {
 612                 v.pop();
 613                 v.push(Token::Operation(Operation::MultiplyEqual));
 614             } else if c == ReservedChar::EqualSign &&
 615                       *v.last().unwrap_or(&Token::Other("")) == Token::Operation(Operation::Addition) {
 616                 v.pop();
 617                 v.push(Token::Operation(Operation::AdditionEqual));
 618             } else if c == ReservedChar::EqualSign &&
 619                       *v.last().unwrap_or(&Token::Other("")) == Token::Operation(Operation::Subtract) {
 620                 v.pop();
 621                 v.push(Token::Operation(Operation::SubtractEqual));
 622             } else if c == ReservedChar::EqualSign &&
 623                       *v.last().unwrap_or(&Token::Other("")) == Token::Operation(Operation::Modulo) {
 624                 v.pop();
 625                 v.push(Token::Operation(Operation::ModuloEqual));
 626             } else if let Ok(o) = Operation::try_from(c) {
 627                 v.push(Token::Operation(o));
 628             } else {
 629                 v.push(Token::Char(c));
 630             }
 631             start = pos + 1;
 632         }
 633     }
 634     Tokens(v)
 635 }
 636
 637 pub struct Tokens<'a>(pub Vec<Token<'a>>);
 638
 639 impl<'a> fmt::Display for Tokens<'a> {
 640     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 641         let tokens = &self.0;
 642         for i in 0..tokens.len() {
 643             write!(f, "{}", tokens[i])?;
 644             if let Some(c) = match tokens[i] {
 645                 Token::Keyword(_) |
 646                 Token::Other(_) if i + 1 < tokens.len() => get_required(&tokens[i + 1]),
 647                 _ => None,
 648             } {
 649                 write!(f, "{}", c)?;
 650             }
 651         }
 652         Ok(())
 653     }
 654 }
 655
 656 pub fn clean_tokens<'a>(tokens: &mut Tokens<'a>) {
 657     tokens.0.retain(|c| {
 658         !c.is_comment() && {
 659             if let Some(x) = c.get_char() {
 660                 !x.is_useless()
 661             } else {
 662                 true
 663             }
 664         }
 665     });
 666 }