vendor/minifier/src/js/utils.rs

   1 // MIT License
   2 //
   3 // Copyright (c) 2018 Guillaume Gomez
   4 //
   5 // Permission is hereby granted, free of charge, to any person obtaining a copy
   6 // of this software and associated documentation files (the "Software"), to deal
   7 // in the Software without restriction, including without limitation the rights
   8 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 // copies of the Software, and to permit persons to whom the Software is
  10 // furnished to do so, subject to the following conditions:
  11 //
  12 // The above copyright notice and this permission notice shall be included in all
  13 // copies or substantial portions of the Software.
  14 //
  15 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21 // SOFTWARE.
  22
  23 use js::token::{Keyword, Operation, ReservedChar, Token, Tokens};
  24 use std::vec::IntoIter;
  25
  26 pub(crate) struct VariableNameGenerator<'a> {
  27     letter: char,
  28     lower: Option<Box<VariableNameGenerator<'a>>>,
  29     prepend: Option<&'a str>,
  30 }
  31
  32 impl<'a> VariableNameGenerator<'a> {
  33     pub(crate) fn new(prepend: Option<&'a str>, nb_letter: usize) -> VariableNameGenerator<'a> {
  34         if nb_letter > 1 {
  35             VariableNameGenerator {
  36                 letter: 'a',
  37                 lower: Some(Box::new(VariableNameGenerator::new(None, nb_letter - 1))),
  38                 prepend,
  39             }
  40         } else {
  41             VariableNameGenerator {
  42                 letter: 'a',
  43                 lower: None,
  44                 prepend,
  45             }
  46         }
  47     }
  48
  49     pub(crate) fn next(&mut self) {
  50         self.incr_letters();
  51     }
  52
  53     #[allow(clippy::inherent_to_string)]
  54     pub(crate) fn to_string(&self) -> String {
  55         if let Some(ref lower) = self.lower {
  56             format!(
  57                 "{}{}{}",
  58                 self.prepend.unwrap_or(""),
  59                 self.letter,
  60                 lower.to_string()
  61             )
  62         } else {
  63             format!("{}{}", self.prepend.unwrap_or(""), self.letter)
  64         }
  65     }
  66
  67     #[allow(dead_code)]
  68     pub(crate) fn len(&self) -> usize {
  69         let first = match self.prepend {
  70             Some(ref s) => s.len(),
  71             None => 0,
  72         } + 1;
  73         first
  74             + match self.lower {
  75                 Some(ref s) => s.len(),
  76                 None => 0,
  77             }
  78     }
  79
  80     pub(crate) fn incr_letters(&mut self) {
  81         let max = [('z', 'A'), ('Z', '0'), ('9', 'a')];
  82
  83         for (m, next) in &max {
  84             if self.letter == *m {
  85                 self.letter = *next;
  86                 if self.letter == 'a' {
  87                     if let Some(ref mut lower) = self.lower {
  88                         lower.incr_letters();
  89                     } else {
  90                         self.lower = Some(Box::new(VariableNameGenerator::new(None, 1)));
  91                     }
  92                 }
  93                 return;
  94             }
  95         }
  96         self.letter = ((self.letter as u8) + 1) as char;
  97     }
  98 }
  99
 100 /// Replace given tokens with others.
 101 ///
 102 /// # Example
 103 ///
 104 /// ```rust
 105 /// extern crate minifier;
 106 /// use minifier::js::{Keyword, Token, replace_tokens_with, simple_minify};
 107 ///
 108 /// fn main() {
 109 ///     let js = r#"
 110 ///         function replaceByNull(data, func) {
 111 ///             for (var i = 0; i < data.length; ++i) {
 112 ///                 if func(data[i]) {
 113 ///                     data[i] = null;
 114 ///                 }
 115 ///             }
 116 ///         }
 117 ///     }"#.into();
 118 ///     let js_minified = simple_minify(js)
 119 ///         .apply(|f| {
 120 ///             replace_tokens_with(f, |t| {
 121 ///                 if *t == Token::Keyword(Keyword::Null) {
 122 ///                     Some(Token::Other("N"))
 123 ///                 } else {
 124 ///                     None
 125 ///                 }
 126 ///             })
 127 ///         });
 128 ///     println!("{}", js_minified.to_string());
 129 /// }
 130 /// ```
 131 ///
 132 /// The previous code will have all its `null` keywords replaced with `N`. In such cases,
 133 /// don't forget to include the definition of `N` in the returned minified javascript:
 134 ///
 135 /// ```js
 136 /// var N = null;
 137 /// ```
 138 #[inline]
 139 pub fn replace_tokens_with<'a, 'b: 'a, F: Fn(&Token<'a>) -> Option<Token<'b>>>(
 140     mut tokens: Tokens<'a>,
 141     callback: F,
 142 ) -> Tokens<'a> {
 143     for token in tokens.0.iter_mut() {
 144         if let Some(t) = callback(token) {
 145             *token = t;
 146         }
 147     }
 148     tokens
 149 }
 150
 151 /// Replace a given token with another.
 152 #[inline]
 153 pub fn replace_token_with<'a, 'b: 'a, F: Fn(&Token<'a>) -> Option<Token<'b>>>(
 154     token: Token<'a>,
 155     callback: &F,
 156 ) -> Token<'a> {
 157     if let Some(t) = callback(&token) {
 158         t
 159     } else {
 160         token
 161     }
 162 }
 163
 164 /// When looping over `Tokens`, if you encounter `Keyword::Var`, `Keyword::Let` or
 165 /// `Token::Other` using this function will allow you to get the variable name's
 166 /// position and the variable value's position (if any).
 167 ///
 168 /// ## Note
 169 ///
 170 /// It'll return the value only if there is an `Operation::Equal` found.
 171 ///
 172 /// # Examples
 173 ///
 174 /// ```
 175 /// extern crate minifier;
 176 /// use minifier::js::{Keyword, get_variable_name_and_value_positions, simple_minify};
 177 ///
 178 /// fn main() {
 179 ///     let source = r#"var x = 1;var z;var y   =   "2";"#;
 180 ///     let mut result = Vec::new();
 181 ///
 182 ///     let tokens = simple_minify(source);
 183 ///
 184 ///     for pos in 0..tokens.len() {
 185 ///         match tokens[pos].get_keyword() {
 186 ///             Some(k) if k == Keyword::Let || k == Keyword::Var => {
 187 ///                 if let Some(x) = get_variable_name_and_value_positions(&tokens, pos) {
 188 ///                     result.push(x);
 189 ///                 }
 190 ///             }
 191 ///             _ => {}
 192 ///         }
 193 ///     }
 194 ///     assert_eq!(result, vec![(2, Some(6)), (10, None), (14, Some(22))]);
 195 /// }
 196 /// ```
 197 #[inline]
 198 pub fn get_variable_name_and_value_positions<'a>(
 199     tokens: &'a Tokens<'a>,
 200     pos: usize,
 201 ) -> Option<(usize, Option<usize>)> {
 202     if pos >= tokens.len() {
 203         return None;
 204     }
 205     let mut tmp = pos;
 206     match tokens[pos] {
 207         Token::Keyword(Keyword::Let) | Token::Keyword(Keyword::Var) => {
 208             tmp += 1;
 209         }
 210         Token::Other(_) if pos > 0 => {
 211             let mut pos = pos - 1;
 212             while pos > 0 {
 213                 if tokens[pos].is_comment() || tokens[pos].is_white_character() {
 214                     pos -= 1;
 215                 } else if tokens[pos] == Token::Char(ReservedChar::Comma)
 216                     || tokens[pos] == Token::Keyword(Keyword::Let)
 217                     || tokens[pos] == Token::Keyword(Keyword::Var)
 218                 {
 219                     break;
 220                 } else {
 221                     return None;
 222                 }
 223             }
 224         }
 225         _ => return None,
 226     }
 227     while tmp < tokens.len() {
 228         if tokens[tmp].is_other() {
 229             let mut tmp2 = tmp + 1;
 230             while tmp2 < tokens.len() {
 231                 if tokens[tmp2] == Token::Operation(Operation::Equal) {
 232                     tmp2 += 1;
 233                     while tmp2 < tokens.len() {
 234                         let token = &tokens[tmp2];
 235                         if token.is_string()
 236                             || token.is_other()
 237                             || token.is_regex()
 238                             || token.is_number()
 239                             || token.is_floating_number()
 240                         {
 241                             return Some((tmp, Some(tmp2)));
 242                         } else if !tokens[tmp2].is_comment() && !tokens[tmp2].is_white_character() {
 243                             break;
 244                         }
 245                         tmp2 += 1;
 246                     }
 247                     break;
 248                 } else if matches!(
 249                     tokens[tmp2].get_char(),
 250                     Some(ReservedChar::Comma) | Some(ReservedChar::SemiColon)
 251                 ) {
 252                     return Some((tmp, None));
 253                 } else if !(tokens[tmp2].is_comment()
 254                     || tokens[tmp2].is_white_character()
 255                         && tokens[tmp2].get_char() != Some(ReservedChar::Backline))
 256                 {
 257                     break;
 258                 }
 259                 tmp2 += 1;
 260             }
 261         } else {
 262             // We don't care about syntax errors.
 263         }
 264         tmp += 1;
 265     }
 266     None
 267 }
 268
 269 #[inline]
 270 fn get_next<'a>(it: &mut IntoIter<Token<'a>>) -> Option<Token<'a>> {
 271     for t in it {
 272         if t.is_comment() || t.is_white_character() {
 273             continue;
 274         }
 275         return Some(t);
 276     }
 277     None
 278 }
 279
 280 /// Convenient function used to clean useless tokens in a token list.
 281 ///
 282 /// # Example
 283 ///
 284 /// ```rust,no_run
 285 /// extern crate minifier;
 286 ///
 287 /// use minifier::js::{clean_tokens, simple_minify};
 288 /// use std::fs;
 289 ///
 290 /// fn main() {
 291 ///     let content = fs::read("some_file.js").expect("file not found");
 292 ///     let source = String::from_utf8_lossy(&content);
 293 ///     let s = simple_minify(&source); // First we get the tokens list.
 294 ///     let s = s.apply(clean_tokens);  // We now have a cleaned token list!
 295 ///     println!("result: {:?}", s);
 296 /// }
 297 /// ```
 298 #[inline]
 299 pub fn clean_tokens(tokens: Tokens<'_>) -> Tokens<'_> {
 300     let mut v = Vec::with_capacity(tokens.len() / 3 * 2);
 301     let mut it = tokens.0.into_iter();
 302
 303     loop {
 304         let token = get_next(&mut it);
 305         if token.is_none() {
 306             break;
 307         }
 308         let token = token.unwrap();
 309         if token.is_white_character() {
 310             continue;
 311         } else if token.get_char() == Some(ReservedChar::SemiColon) {
 312             if v.is_empty() {
 313                 continue;
 314             }
 315             if let Some(next) = get_next(&mut it) {
 316                 if next != Token::Char(ReservedChar::CloseCurlyBrace) {
 317                     v.push(token);
 318                 }
 319                 v.push(next);
 320             }
 321             continue;
 322         }
 323         v.push(token);
 324     }
 325     v.into()
 326 }
 327
 328 /// Returns true if the token is a "useful" one (so not a comment or a "useless"
 329 /// character).
 330 #[inline]
 331 pub fn clean_token(token: &Token<'_>, next_token: &Option<&Token<'_>>) -> bool {
 332     !token.is_comment() && {
 333         if let Some(x) = token.get_char() {
 334             !x.is_white_character()
 335                 && (x != ReservedChar::SemiColon
 336                     || *next_token != Some(&Token::Char(ReservedChar::CloseCurlyBrace)))
 337         } else {
 338             true
 339         }
 340     }
 341 }
 342
 343 #[inline]
 344 fn get_next_except<'a, F: Fn(&Token<'a>) -> bool>(
 345     it: &mut IntoIter<Token<'a>>,
 346     f: &F,
 347 ) -> Option<Token<'a>> {
 348     for t in it {
 349         if (t.is_comment() || t.is_white_character()) && f(&t) {
 350             continue;
 351         }
 352         return Some(t);
 353     }
 354     None
 355 }
 356
 357 /// Same as `clean_tokens` except that if a token is considered as not desired,
 358 /// the callback is called. If the callback returns `false` as well, it will
 359 /// be removed.
 360 ///
 361 /// # Example
 362 ///
 363 /// ```rust,no_run
 364 /// extern crate minifier;
 365 ///
 366 /// use minifier::js::{clean_tokens_except, simple_minify, ReservedChar};
 367 /// use std::fs;
 368 ///
 369 /// fn main() {
 370 ///     let content = fs::read("some_file.js").expect("file not found");
 371 ///     let source = String::from_utf8_lossy(&content);
 372 ///     let s = simple_minify(&source); // First we get the tokens list.
 373 ///     let s = s.apply(|f| {
 374 ///         clean_tokens_except(f, |c| {
 375 ///             c.get_char() != Some(ReservedChar::Backline)
 376 ///         })
 377 ///     });  // We now have a cleaned token list which kept backlines!
 378 ///     println!("result: {:?}", s);
 379 /// }
 380 /// ```
 381 #[inline]
 382 pub fn clean_tokens_except<'a, F: Fn(&Token<'a>) -> bool>(tokens: Tokens<'a>, f: F) -> Tokens<'a> {
 383     let mut v = Vec::with_capacity(tokens.len() / 3 * 2);
 384     let mut it = tokens.0.into_iter();
 385
 386     loop {
 387         let token = get_next_except(&mut it, &f);
 388         if token.is_none() {
 389             break;
 390         }
 391         let token = token.unwrap();
 392         if token.is_white_character() {
 393             if f(&token) {
 394                 continue;
 395             }
 396         } else if token.get_char() == Some(ReservedChar::SemiColon) {
 397             if v.is_empty() {
 398                 if !f(&token) {
 399                     v.push(token);
 400                 }
 401                 continue;
 402             }
 403             if let Some(next) = get_next_except(&mut it, &f) {
 404                 if next != Token::Char(ReservedChar::CloseCurlyBrace) || !f(&token) {
 405                     v.push(token);
 406                 }
 407                 v.push(next);
 408             } else if !f(&token) {
 409                 v.push(token);
 410             }
 411             continue;
 412         }
 413         v.push(token);
 414     }
 415     v.into()
 416 }
 417
 418 /// Returns true if the token is a "useful" one (so not a comment or a "useless"
 419 /// character).
 420 #[inline]
 421 pub fn clean_token_except<'a, F: Fn(&Token<'a>) -> bool>(
 422     token: &Token<'a>,
 423     next_token: &Option<&Token<'_>>,
 424     f: &F,
 425 ) -> bool {
 426     if !clean_token(token, next_token) {
 427         !f(token)
 428     } else {
 429         true
 430     }
 431 }
 432
 433 #[inline]
 434 pub(crate) fn get_array<'a>(
 435     tokens: &'a Tokens<'a>,
 436     array_name: &str,
 437 ) -> Option<(Vec<usize>, usize)> {
 438     let mut ret = Vec::new();
 439
 440     let mut looking_for_var = false;
 441     let mut looking_for_equal = false;
 442     let mut looking_for_array_start = false;
 443     let mut getting_values = false;
 444
 445     for pos in 0..tokens.len() {
 446         if looking_for_var {
 447             match tokens[pos] {
 448                 Token::Other(s) => {
 449                     looking_for_var = false;
 450                     if s == array_name {
 451                         looking_for_equal = true;
 452                     }
 453                 }
 454                 ref s => {
 455                     looking_for_var = s.is_comment() || s.is_white_character();
 456                 }
 457             }
 458         } else if looking_for_equal {
 459             match tokens[pos] {
 460                 Token::Operation(Operation::Equal) => {
 461                     looking_for_equal = false;
 462                     looking_for_array_start = true;
 463                 }
 464                 ref s => {
 465                     looking_for_equal = s.is_comment() || s.is_white_character();
 466                 }
 467             }
 468         } else if looking_for_array_start {
 469             match tokens[pos] {
 470                 Token::Char(ReservedChar::OpenBracket) => {
 471                     looking_for_array_start = false;
 472                     getting_values = true;
 473                 }
 474                 ref s => {
 475                     looking_for_array_start = s.is_comment() || s.is_white_character();
 476                 }
 477             }
 478         } else if getting_values {
 479             match &tokens[pos] {
 480                 Token::Char(ReservedChar::CloseBracket) => {
 481                     return Some((ret, pos));
 482                 }
 483                 s if s.is_comment() || s.is_white_character() => {}
 484                 _ => {
 485                     ret.push(pos);
 486                 }
 487             }
 488         } else {
 489             match tokens[pos] {
 490                 Token::Keyword(Keyword::Let) | Token::Keyword(Keyword::Var) => {
 491                     looking_for_var = true;
 492                 }
 493                 _ => {}
 494             }
 495         }
 496     }
 497     None
 498 }
 499
 500 #[test]
 501 fn check_get_array() {
 502     let source = r#"var x = [  ]; var y = ['hello',
 503     12]; var z = []; var w = 12;"#;
 504
 505     let tokens = ::js::token::tokenize(source);
 506
 507     let ar = get_array(&tokens, "x");
 508     assert!(ar.is_some());
 509     assert_eq!(ar.unwrap().1, 9);
 510
 511     let ar = get_array(&tokens, "y");
 512     assert!(ar.is_some());
 513     assert_eq!(ar.unwrap().1, 27);
 514
 515     let ar = get_array(&tokens, "z");
 516     assert!(ar.is_some());
 517     assert_eq!(ar.unwrap().1, 37);
 518
 519     let ar = get_array(&tokens, "w");
 520     assert!(ar.is_none());
 521
 522     let ar = get_array(&tokens, "W");
 523     assert!(ar.is_none());
 524 }
 525
 526 #[test]
 527 fn check_get_variable_name_and_value_positions() {
 528     let source = r#"var x = 1;var y   =   "2",we=4;"#;
 529     let mut result = Vec::new();
 530     let mut pos = 0;
 531
 532     let tokens = ::js::token::tokenize(source);
 533
 534     while pos < tokens.len() {
 535         if let Some(x) = get_variable_name_and_value_positions(&tokens, pos) {
 536             result.push(x);
 537             pos = x.0;
 538         }
 539         pos += 1;
 540     }
 541     assert_eq!(result, vec![(2, Some(6)), (10, Some(18)), (20, Some(22))]);
 542
 543     let mut result = Vec::new();
 544     let tokens = ::js::clean_tokens(tokens);
 545     pos = 0;
 546
 547     while pos < tokens.len() {
 548         if let Some(x) = get_variable_name_and_value_positions(&tokens, pos) {
 549             result.push(x);
 550             pos = x.0;
 551         }
 552         pos += 1;
 553     }
 554     assert_eq!(result, vec![(1, Some(3)), (6, Some(8)), (10, Some(12))]);
 555 }
 556
 557 #[test]
 558 fn replace_tokens() {
 559     let source = r#"
 560 var x = ['a', 'b', null, 'd', {'x': null, 'e': null, 'z': 'w'}];
 561 var n = null;
 562 "#;
 563     let expected_result = "var x=['a','b',N,'d',{'x':N,'e':N,'z':'w'}];var n=N";
 564
 565     let res = ::js::simple_minify(source)
 566         .apply(::js::clean_tokens)
 567         .apply(|f| {
 568             replace_tokens_with(f, |t| {
 569                 if *t == Token::Keyword(Keyword::Null) {
 570                     Some(Token::Other("N"))
 571                 } else {
 572                     None
 573                 }
 574             })
 575         });
 576     assert_eq!(res.to_string(), expected_result);
 577 }
 578
 579 #[test]
 580 fn check_iterator() {
 581     let source = r#"
 582 var x = ['a', 'b', null, 'd', {'x': null, 'e': null, 'z': 'w'}];
 583 var n = null;
 584 "#;
 585     let expected_result = "var x=['a','b',N,'d',{'x':N,'e':N,'z':'w'}];var n=N;";
 586
 587     let res: Tokens = ::js::simple_minify(source)
 588         .into_iter()
 589         .filter(|(x, next)| ::js::clean_token(x, next))
 590         .map(|(t, _)| {
 591             if t == Token::Keyword(Keyword::Null) {
 592                 Token::Other("N")
 593             } else {
 594                 t
 595             }
 596         })
 597         .collect::<Vec<_>>()
 598         .into();
 599     assert_eq!(res.to_string(), expected_result);
 600 }