vendor/minifier/src/js/tools.rs

   1 // MIT License
   2 //
   3 // Copyright (c) 2018 Guillaume Gomez
   4 //
   5 // Permission is hereby granted, free of charge, to any person obtaining a copy
   6 // of this software and associated documentation files (the "Software"), to deal
   7 // in the Software without restriction, including without limitation the rights
   8 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 // copies of the Software, and to permit persons to whom the Software is
  10 // furnished to do so, subject to the following conditions:
  11 //
  12 // The above copyright notice and this permission notice shall be included in all
  13 // copies or substantial portions of the Software.
  14 //
  15 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21 // SOFTWARE.
  22
  23 use js::token::{self, Keyword, ReservedChar, Token, Tokens};
  24 use js::utils::{get_array, get_variable_name_and_value_positions, VariableNameGenerator};
  25
  26 use std::collections::{HashMap, HashSet};
  27
  28 /*#[derive(Debug, Clone, PartialEq, Eq)]
  29 enum Elem<'a> {
  30     Function(Function<'a>),
  31     Block(Block<'a>),
  32     Variable(Variable<'a>),
  33     Condition(token::Condition),
  34     Loop(Loop<'a>),
  35     Operation(Operation<'a>),
  36 }
  37
  38 impl<'a> Elem<'a> {
  39     fn is_condition(&self) -> bool {
  40         match *self {
  41             Elem::Condition(_) => true,
  42             _ => false,
  43         }
  44     }
  45 }
  46
  47 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
  48 enum ConditionType {
  49     If,
  50     ElseIf,
  51     Else,
  52     Ternary,
  53 }
  54
  55 #[derive(Clone, PartialEq, Eq, Debug)]
  56 struct Block<'a> {
  57     elems: Vec<Elem<'a>>,
  58 }
  59
  60 #[derive(Clone, PartialEq, Eq, Debug)]
  61 struct Argument<'a> {
  62     name: &'a str,
  63 }
  64
  65 #[derive(Clone, PartialEq, Eq, Debug)]
  66 struct Function<'a> {
  67     name: Option<&'a str>,
  68     args: Vec<Argument<'a>>,
  69     block: Block<'a>,
  70 }
  71
  72 #[derive(Clone, PartialEq, Eq, Debug)]
  73 struct Variable<'a> {
  74     name: &'a str,
  75     value: Option<&'a str>,
  76 }
  77
  78 /*struct Condition<'a> {
  79     ty_: ConditionType,
  80     condition: &'a str,
  81     block: Block<'a>,
  82 }*/
  83
  84 #[derive(Clone, Copy, PartialEq, Eq, Debug)]
  85 enum LoopType {
  86     Do,
  87     For,
  88     While,
  89 }
  90
  91 #[derive(Clone, PartialEq, Eq, Debug)]
  92 struct Loop<'a> {
  93     ty_: LoopType,
  94     condition: Vec<Elem<'a>>,
  95     block: Block<'a>,
  96 }
  97
  98 #[derive(Clone, PartialEq, Eq, Debug)]
  99 struct Operation<'a> {
 100     content: &'a str,
 101 }
 102
 103 fn get_while_condition<'a>(tokens: &[token::Token<'a>], pos: &mut usize) -> Result<Vec<Elem<'a>>, String> {
 104     let tmp = *pos;
 105     *pos += 1;
 106     if let Err(e) = match tokens.get(tmp) {
 107         Some(token::Token::Char(token::ReservedChar::OpenParenthese)) => Ok(()),
 108         Some(e) => Err(format!("Expected \"(\", found \"{:?}\"", e)),
 109         None => Err("Expected \"(\", found nothing...".to_owned()),
 110     } {
 111         return Err(e);
 112     }
 113     let mut elems: Vec<Elem<'a>> = Vec::with_capacity(1);
 114
 115     while let Some(e) = tokens.get(*pos) {
 116         *pos += 1;
 117         match e {
 118             token::Token::Char(token::ReservedChar::CloseParenthese) => return Ok(elems),
 119             token::Token::Condition(e) => {
 120                 if let Some(cond) = elems.last() {
 121                     if cond.is_condition() {
 122                         return Err(format!("\"{:?}\" cannot follow \"{:?}\"", e, cond));
 123                     }
 124                 }
 125             }
 126             _ => {}
 127         }
 128     }
 129     Err("Expected \")\", found nothing...".to_owned())
 130 }
 131
 132 fn get_do<'a>(tokens: &[token::Token<'a>], pos: &mut usize) -> Result<Elem<'a>, String> {
 133     let tmp = *pos;
 134     *pos += 1;
 135     let block = match tokens.get(tmp) {
 136         Some(token::Token::Char(token::ReservedChar::OpenCurlyBrace)) => get_block(tokens, pos, true),
 137         Some(e) => Err(format!("Expected \"{{\", found \"{:?}\"", e)),
 138         None => Err("Expected \"{\", found nothing...".to_owned()),
 139     }?;
 140     let tmp = *pos;
 141     *pos += 1;
 142     let condition = match tokens.get(tmp) {
 143         Some(token::Token::Keyword(token::Keyword::While)) => get_while_condition(tokens, pos),
 144         Some(e) => Err(format!("Expected \"while\", found \"{:?}\"", e)),
 145         None => Err("Expected \"while\", found nothing...".to_owned()),
 146     }?;
 147     let mut loop_ = Loop {
 148         ty_: LoopType::Do,
 149         condition: condition,
 150         block,
 151     };
 152     Ok(Elem::Loop(loop_))
 153 }
 154
 155 fn get_block<'a>(tokens: &[token::Token<'a>], pos: &mut usize,
 156                  start_with_paren: bool) -> Result<Block<'a>, String> {
 157     let mut block = Block { elems: Vec::with_capacity(2) };
 158     while let Some(e) = tokens.get(*pos) {
 159         *pos += 1;
 160         block.elems.push(match e {
 161             token::Token::Keyword(token::Keyword::Do) => get_do(tokens, pos),
 162             token::Token::Char(token::ReservedChar::CloseCurlyBrace) => {
 163                 if start_with_paren {
 164                     return Ok(block);
 165                 }
 166                 return Err("Unexpected \"}\"".to_owned());
 167             }
 168         }?);
 169     }
 170     if !start_with_paren {
 171         Ok(block)
 172     } else {
 173         Err("Expected \"}\" at the end of the block but didn't find one...".to_owned())
 174     }
 175 }
 176
 177 fn build_ast<'a>(v: &[token::Token<'a>]) -> Result<Elem<'a>, String> {
 178     let mut pos = 0;
 179
 180     match get_block(v, &mut pos, false) {
 181         Ok(ast) => Ok(Elem::Block(ast)),
 182         Err(e) => Err(e),
 183     }
 184 }*/
 185
 186 /// Minifies a given JS source code.
 187 ///
 188 /// # Example
 189 ///
 190 /// ```rust
 191 /// extern crate minifier;
 192 /// use minifier::js::minify;
 193 ///
 194 /// fn main() {
 195 ///     let js = r#"
 196 ///         function forEach(data, func) {
 197 ///            for (var i = 0; i < data.length; ++i) {
 198 ///                func(data[i]);
 199 ///            }
 200 ///         }"#.into();
 201 ///     let js_minified = minify(js);
 202 /// }
 203 /// ```
 204 #[inline]
 205 pub fn minify(source: &str) -> String {
 206     token::tokenize(source)
 207         .apply(::js::clean_tokens)
 208         .to_string()
 209 }
 210
 211 // TODO: No scope handling or anything. Might be nice as a second step to add it...
 212 fn get_variables_name<'a>(
 213     tokens: &'a Tokens<'a>,
 214 ) -> (HashSet<&'a str>, HashMap<&'a str, (usize, usize)>) {
 215     let mut ret = HashSet::new();
 216     let mut variables = HashMap::new();
 217     let mut pos = 0;
 218
 219     while pos < tokens.len() {
 220         if tokens[pos].is_keyword() || tokens[pos].is_other() {
 221             if let Some((var_pos, Some(value_pos))) =
 222                 get_variable_name_and_value_positions(tokens, pos)
 223             {
 224                 pos = value_pos;
 225                 if let Some(var_name) = tokens[var_pos].get_other() {
 226                     if !var_name.starts_with("r_") {
 227                         pos += 1;
 228                         continue;
 229                     }
 230                     ret.insert(var_name);
 231                 }
 232                 if let Some(s) = tokens[value_pos].get_string() {
 233                     variables.insert(s, (var_pos, value_pos));
 234                 }
 235             }
 236         }
 237         pos += 1;
 238     }
 239     (ret, variables)
 240 }
 241
 242 #[inline]
 243 fn aggregate_strings_inner<'a, 'b: 'a>(
 244     mut tokens: Tokens<'a>,
 245     separation_token: Option<Token<'b>>,
 246 ) -> Tokens<'a> {
 247     let mut new_vars = Vec::with_capacity(50);
 248     let mut to_replace: Vec<(usize, usize)> = Vec::new();
 249
 250     for (var_name, positions) in {
 251         let mut strs: HashMap<&Token, Vec<usize>> = HashMap::with_capacity(1000);
 252         let mut validated: HashMap<&Token, String> = HashMap::with_capacity(100);
 253
 254         let mut var_gen = VariableNameGenerator::new(Some("r_"), 2);
 255         let mut next_name = var_gen.to_string();
 256
 257         let (all_variables, values) = get_variables_name(&tokens);
 258         while all_variables.contains(&next_name.as_str()) {
 259             var_gen.next();
 260             next_name = var_gen.to_string();
 261         }
 262
 263         for pos in 0..tokens.len() {
 264             let token = &tokens[pos];
 265             if let Some(str_token) = token.get_string() {
 266                 if let Some((var_pos, string_pos)) = values.get(&str_token) {
 267                     if pos != *string_pos {
 268                         to_replace.push((pos, *var_pos));
 269                     }
 270                     continue;
 271                 }
 272                 let x = strs.entry(token).or_insert_with(|| Vec::with_capacity(1));
 273                 x.push(pos);
 274                 if x.len() > 1 && validated.get(token).is_none() {
 275                     let len = str_token.len();
 276                     // Computation here is simple, we declare new variables when creating this so
 277                     // the total of characters must be shorter than:
 278                     // `var r_aa=...;` -> 10 + `r_aa` -> 14
 279                     if (x.len() + 2/* quotes */) * len
 280                         > next_name.len() + str_token.len() + 6 /* var _=_;*/ + x.len() * next_name.len()
 281                     {
 282                         validated.insert(token, next_name.clone());
 283                         var_gen.next();
 284                         next_name = var_gen.to_string();
 285                         while all_variables.contains(&next_name.as_str()) {
 286                             var_gen.next();
 287                             next_name = var_gen.to_string();
 288                         }
 289                     }
 290                 }
 291             }
 292         }
 293         let mut ret = Vec::with_capacity(validated.len());
 294
 295         // We need this macro to avoid having to sort the set when not testing the crate.
 296         //#[cfg(test)]
 297         macro_rules! inner_loop {
 298             ($x:ident) => {{
 299                 let mut $x = $x.into_iter().collect::<Vec<_>>();
 300                 $x.sort_unstable_by(|a, b| a.1.cmp(&b.1));
 301                 $x
 302             }};
 303         }
 304         /*#[cfg(not(test))]
 305         macro_rules! inner_loop {
 306             ($x:ident) => {
 307                 $x.into_iter()
 308             }
 309         }*/
 310
 311         for (token, var_name) in inner_loop!(validated) {
 312             ret.push((var_name, strs.remove(&token).unwrap()));
 313             var_gen.next();
 314         }
 315         ret
 316     } {
 317         if new_vars.is_empty() {
 318             new_vars.push(Token::Keyword(Keyword::Var));
 319         } else {
 320             new_vars.push(Token::Char(ReservedChar::Comma));
 321         }
 322         new_vars.push(Token::CreatedVarDecl(format!(
 323             "{}={}",
 324             var_name, tokens[positions[0]]
 325         )));
 326         for pos in positions {
 327             tokens.0[pos] = Token::CreatedVar(var_name.clone());
 328         }
 329     }
 330     if !new_vars.is_empty() {
 331         new_vars.push(Token::Char(ReservedChar::SemiColon));
 332     }
 333     for (to_replace_pos, variable_pos) in to_replace {
 334         tokens.0[to_replace_pos] = tokens.0[variable_pos].clone();
 335     }
 336     if let Some(token) = separation_token {
 337         new_vars.push(token);
 338     }
 339     new_vars.append(&mut tokens.0);
 340     Tokens(new_vars)
 341 }
 342
 343 /// Aggregate litteral strings. For instance, if the string litteral "Oh look over there!"
 344 /// appears more than once, a variable will be created with this value and used everywhere the
 345 /// string appears. Of course, this replacement is only performed when it allows to take
 346 /// less space.
 347 ///
 348 /// # Example
 349 ///
 350 /// ```rust,no_run
 351 /// extern crate minifier;
 352 /// use minifier::js::{aggregate_strings, clean_tokens, simple_minify};
 353 /// use std::fs;
 354 ///
 355 /// fn main() {
 356 ///     let content = fs::read("some_file.js").expect("file not found");
 357 ///     let source = String::from_utf8_lossy(&content);
 358 ///     let s = simple_minify(&source);    // First we get the tokens list.
 359 ///     let s = s.apply(aggregate_strings) // This `apply` aggregates string litterals.
 360 ///              .apply(clean_tokens)      // This one is used to remove useless chars.
 361 ///              .to_string();             // And we finally convert to string.
 362 ///     println!("result: {}", s);
 363 /// }
 364 /// ```
 365 #[inline]
 366 pub fn aggregate_strings(tokens: Tokens<'_>) -> Tokens<'_> {
 367     aggregate_strings_inner(tokens, None)
 368 }
 369
 370 /// Exactly like `aggregate_strings` except this one expects a separation token
 371 /// to be passed. This token will be placed between the created variables for the
 372 /// strings aggregation and the rest.
 373 ///
 374 /// # Example
 375 ///
 376 /// Let's add a backline between the created variables and the rest of the code:
 377 ///
 378 /// ```rust,no_run
 379 /// extern crate minifier;
 380 /// use minifier::js::{
 381 ///     aggregate_strings_with_separation,
 382 ///     clean_tokens,
 383 ///     simple_minify,
 384 ///     Token,
 385 ///     ReservedChar,
 386 /// };
 387 /// use std::fs;
 388 ///
 389 /// fn main() {
 390 ///     let content = fs::read("some_file.js").expect("file not found");
 391 ///     let source = String::from_utf8_lossy(&content);
 392 ///     let s = simple_minify(&source);    // First we get the tokens list.
 393 ///     let s = s.apply(|f| {
 394 ///                  aggregate_strings_with_separation(f, Token::Char(ReservedChar::Backline))
 395 ///              })                   // We add a backline between the variable and the rest.
 396 ///              .apply(clean_tokens) // We clean the tokens.
 397 ///              .to_string();        // And we finally convert to string.
 398 ///     println!("result: {}", s);
 399 /// }
 400 /// ```
 401 #[inline]
 402 pub fn aggregate_strings_with_separation<'a, 'b: 'a>(
 403     tokens: Tokens<'a>,
 404     separation_token: Token<'b>,
 405 ) -> Tokens<'a> {
 406     aggregate_strings_inner(tokens, Some(separation_token))
 407 }
 408
 409 #[inline]
 410 fn aggregate_strings_into_array_inner<'a, 'b: 'a, T: Fn(&Tokens<'a>, usize) -> bool>(
 411     mut tokens: Tokens<'a>,
 412     array_name: &str,
 413     separation_token: Option<Token<'b>>,
 414     filter: T,
 415 ) -> Tokens<'a> {
 416     let mut to_insert = Vec::with_capacity(100);
 417     let mut to_replace = Vec::with_capacity(100);
 418
 419     {
 420         let mut to_ignore = HashSet::new();
 421         // key: the token string
 422         // value: (position in the array, positions in the tokens list, need creation)
 423         let mut strs: HashMap<&str, (usize, Vec<usize>, bool)> = HashMap::with_capacity(1000);
 424         let (current_array_values, need_recreate, mut end_bracket) =
 425             match get_array(&tokens, array_name) {
 426                 Some((s, p)) => (s, false, p),
 427                 None => (Vec::new(), true, 0),
 428             };
 429         let mut validated: HashSet<&str> = HashSet::new();
 430
 431         let mut array_pos = 0;
 432         let mut array_pos_str;
 433         for s in current_array_values.iter() {
 434             if let Some(st) = tokens.0[*s].get_string() {
 435                 strs.insert(&st[1..st.len() - 1], (array_pos, vec![], false));
 436                 array_pos += 1;
 437                 validated.insert(&st[1..st.len() - 1]);
 438                 to_ignore.insert(*s);
 439             }
 440         }
 441
 442         array_pos_str = array_pos.to_string();
 443         for pos in 0..tokens.len() {
 444             if to_ignore.contains(&pos) {
 445                 continue;
 446             }
 447             let token = &tokens[pos];
 448             if let Some(str_token) = token.get_string() {
 449                 if !filter(&tokens, pos) {
 450                     continue;
 451                 }
 452                 let s = &str_token[1..str_token.len() - 1];
 453                 let x = strs
 454                     .entry(s)
 455                     .or_insert_with(|| (0, Vec::with_capacity(1), true));
 456                 x.1.push(pos);
 457                 if x.1.len() > 1 && !validated.contains(s) {
 458                     let len = s.len();
 459                     if len * x.1.len()
 460                         > (array_name.len() + array_pos_str.len() + 2) * x.1.len()
 461                             + array_pos_str.len()
 462                             + 2
 463                     {
 464                         validated.insert(&str_token[1..str_token.len() - 1]);
 465                         x.0 = array_pos;
 466                         array_pos += 1;
 467                         array_pos_str = array_pos.to_string();
 468                     }
 469                 }
 470             }
 471         }
 472
 473         // TODO:
 474         // 1. Sort strings by length (the smallest should take the smallest numbers
 475         //    for bigger gains).
 476         // 2. Compute "score" for all strings of the same length and sort the strings
 477         //    of the same length with this score.
 478         // 3. Loop again over strings and remove those who shouldn't be there anymore.
 479         // 4. Repeat.
 480         //
 481         // ALTERNATIVE:
 482         //
 483         // Compute the score based on:
 484         // current number of digits * str length * str occurence
 485         //
 486         // ^ This second solution should bring even better results.
 487         //
 488         // ALSO: if an array with such strings already exists, it'd be worth it to recompute
 489         // everything again.
 490         let mut validated = validated.iter().map(|v| (strs[v].0, v)).collect::<Vec<_>>();
 491         validated.sort_unstable_by(|(p1, _), (p2, _)| p2.cmp(p1));
 492
 493         if need_recreate && !validated.is_empty() {
 494             if let Some(token) = separation_token {
 495                 to_insert.push((0, token));
 496             }
 497             to_insert.push((0, Token::Char(ReservedChar::SemiColon)));
 498             to_insert.push((0, Token::Char(ReservedChar::CloseBracket)));
 499             to_insert.push((0, Token::Char(ReservedChar::OpenBracket)));
 500             to_insert.push((0, Token::CreatedVarDecl(format!("var {}=", array_name))));
 501
 502             end_bracket = 2;
 503         }
 504
 505         let mut iter = validated.iter().peekable();
 506         while let Some((array_pos, s)) = iter.next() {
 507             let (_, ref tokens_pos, create_array_entry) = strs[*s];
 508             let array_index = Token::CreatedVar(format!("{}[{}]", array_name, array_pos));
 509             for token in tokens_pos.iter() {
 510                 to_replace.push((*token, array_index.clone()));
 511             }
 512             if !create_array_entry {
 513                 continue;
 514             }
 515             to_insert.push((end_bracket, Token::CreatedVar(format!("\"{}\"", *s))));
 516             if iter.peek().is_none() && current_array_values.is_empty() {
 517                 continue;
 518             }
 519             to_insert.push((end_bracket, Token::Char(ReservedChar::Comma)));
 520         }
 521     }
 522     for (pos, rep) in to_replace.into_iter() {
 523         tokens.0[pos] = rep;
 524     }
 525     for (pos, rep) in to_insert.into_iter() {
 526         tokens.0.insert(pos, rep);
 527     }
 528     tokens
 529 }
 530
 531 /// Exactly like `aggregate_strings_into_array` except this one expects a separation token
 532 /// to be passed. This token will be placed between the created array for the
 533 /// strings aggregation and the rest.
 534 ///
 535 /// # Example
 536 ///
 537 /// Let's add a backline between the created variables and the rest of the code:
 538 ///
 539 /// ```rust,no_run
 540 /// extern crate minifier;
 541 /// use minifier::js::{
 542 ///     aggregate_strings_into_array_with_separation,
 543 ///     clean_tokens,
 544 ///     simple_minify,
 545 ///     Token,
 546 ///     ReservedChar,
 547 /// };
 548 /// use std::fs;
 549 ///
 550 /// fn main() {
 551 ///     let content = fs::read("some_file.js").expect("file not found");
 552 ///     let source = String::from_utf8_lossy(&content);
 553 ///     let s = simple_minify(&source);    // First we get the tokens list.
 554 ///     let s = s.apply(|f| {
 555 ///                  aggregate_strings_into_array_with_separation(f, "R", Token::Char(ReservedChar::Backline))
 556 ///              })                   // We add a backline between the variable and the rest.
 557 ///              .apply(clean_tokens) // We clean the tokens.
 558 ///              .to_string();        // And we finally convert to string.
 559 ///     println!("result: {}", s);
 560 /// }
 561 /// ```
 562 #[inline]
 563 pub fn aggregate_strings_into_array_with_separation<'a, 'b: 'a>(
 564     tokens: Tokens<'a>,
 565     array_name: &str,
 566     separation_token: Token<'b>,
 567 ) -> Tokens<'a> {
 568     aggregate_strings_into_array_inner(tokens, array_name, Some(separation_token), |_, _| true)
 569 }
 570
 571 /// Same as [`aggregate_strings_into_array_with_separation`] except it allows certain strings to
 572 /// not be aggregated thanks to the `filter` parameter. If it returns `false`, then the string will
 573 /// be ignored.
 574 #[inline]
 575 pub fn aggregate_strings_into_array_with_separation_filter<'a, 'b: 'a, T>(
 576     tokens: Tokens<'a>,
 577     array_name: &str,
 578     separation_token: Token<'b>,
 579     filter: T,
 580 ) -> Tokens<'a>
 581 where
 582     T: Fn(&Tokens<'a>, usize) -> bool,
 583 {
 584     aggregate_strings_into_array_inner(tokens, array_name, Some(separation_token), filter)
 585 }
 586
 587 /// Aggregate litteral strings. For instance, if the string litteral "Oh look over there!"
 588 /// appears more than once, it will be added to the generated array and used everywhere the
 589 /// string appears. Of course, this replacement is only performed when it allows to take
 590 /// less space.
 591 ///
 592 /// # Example
 593 ///
 594 /// ```rust,no_run
 595 /// extern crate minifier;
 596 /// use minifier::js::{aggregate_strings_into_array, clean_tokens, simple_minify};
 597 /// use std::fs;
 598 ///
 599 /// fn main() {
 600 ///     let content = fs::read("some_file.js").expect("file not found");
 601 ///     let source = String::from_utf8_lossy(&content);
 602 ///     let s = simple_minify(&source);    // First we get the tokens list.
 603 ///     let s = s.apply(|f| aggregate_strings_into_array(f, "R")) // This `apply` aggregates string litterals.
 604 ///              .apply(clean_tokens)      // This one is used to remove useless chars.
 605 ///              .to_string();             // And we finally convert to string.
 606 ///     println!("result: {}", s);
 607 /// }
 608 /// ```
 609 #[inline]
 610 pub fn aggregate_strings_into_array<'a>(tokens: Tokens<'a>, array_name: &str) -> Tokens<'a> {
 611     aggregate_strings_into_array_inner(tokens, array_name, None, |_, _| true)
 612 }
 613
 614 /// Same as [`aggregate_strings_into_array`] except it allows certain strings to not be aggregated
 615 /// thanks to the `filter` parameter. If it returns `false`, then the string will be ignored.
 616 #[inline]
 617 pub fn aggregate_strings_into_array_filter<'a, T>(
 618     tokens: Tokens<'a>,
 619     array_name: &str,
 620     filter: T,
 621 ) -> Tokens<'a>
 622 where
 623     T: Fn(&Tokens<'a>, usize) -> bool,
 624 {
 625     aggregate_strings_into_array_inner(tokens, array_name, None, filter)
 626 }
 627
 628 /// Simple function to get the untouched token list. Useful in case you want to perform some
 629 /// actions directly on it.
 630 ///
 631 /// # Example
 632 ///
 633 /// ```rust,no_run
 634 /// extern crate minifier;
 635 /// use minifier::js::simple_minify;
 636 /// use std::fs;
 637 ///
 638 /// fn main() {
 639 ///     let content = fs::read("some_file.js").expect("file not found");
 640 ///     let source = String::from_utf8_lossy(&content);
 641 ///     let s = simple_minify(&source);
 642 ///     println!("result: {:?}", s); // We now have the tokens list.
 643 /// }
 644 /// ```
 645 #[inline]
 646 pub fn simple_minify(source: &str) -> Tokens<'_> {
 647     token::tokenize(source)
 648 }
 649
 650 #[test]
 651 fn aggregate_strings_in_array() {
 652     let source = r#"var x = ["a nice string", "a nice string", "another nice string", "cake!",
 653                              "cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 654     let expected_result = "var R=[\"a nice string\",\"cake!\"];var x=[R[0],R[0],\
 655                            \"another nice string\",R[1],R[1],R[0],R[1],R[1],R[1]]";
 656
 657     let result = simple_minify(source)
 658         .apply(::js::clean_tokens)
 659         .apply(|c| aggregate_strings_into_array(c, "R"))
 660         .to_string();
 661     assert_eq!(result, expected_result);
 662
 663     let source = r#"var x = ["a nice string", "a nice string", "another nice string", "cake!",
 664                              "cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 665     let expected_result = "var R=[\"a nice string\",\"cake!\"];\nvar x=[R[0],R[0],\
 666                            \"another nice string\",R[1],R[1],R[0],R[1],R[1],R[1]]";
 667
 668     let result = simple_minify(source)
 669         .apply(::js::clean_tokens)
 670         .apply(|c| {
 671             aggregate_strings_into_array_with_separation(
 672                 c,
 673                 "R",
 674                 Token::Char(ReservedChar::Backline),
 675             )
 676         })
 677         .to_string();
 678     assert_eq!(result, expected_result);
 679
 680     let source = r#"var x = ["a nice string", "a nice string", "another nice string", "another nice string", "another nice string", "another nice string","cake!","cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 681     let expected_result = "var R=[\"a nice string\",\"another nice string\",\"cake!\"];\n\
 682                            var x=[R[0],R[0],R[1],R[1],R[1],R[1],R[2],R[2],R[0],R[2],\
 683                            R[2],R[2]]";
 684
 685     let result = simple_minify(source)
 686         .apply(::js::clean_tokens)
 687         .apply(|c| {
 688             aggregate_strings_into_array_with_separation(
 689                 c,
 690                 "R",
 691                 Token::Char(ReservedChar::Backline),
 692             )
 693         })
 694         .to_string();
 695     assert_eq!(result, expected_result);
 696 }
 697
 698 #[test]
 699 fn aggregate_strings_in_array_filter() {
 700     let source = r#"var searchIndex = {};searchIndex['duplicate_paths'] = {'aaaaaaaa': 'bbbbbbbb', 'bbbbbbbb': 'aaaaaaaa', 'duplicate_paths': 'aaaaaaaa'};"#;
 701     let expected_result = "var R=[\"bbbbbbbb\",\"aaaaaaaa\"];\nvar searchIndex={};searchIndex['duplicate_paths']={R[1]:R[0],R[0]:R[1],'duplicate_paths':R[1]}";
 702
 703     let result = simple_minify(source)
 704         .apply(::js::clean_tokens)
 705         .apply(|c| {
 706             aggregate_strings_into_array_with_separation_filter(
 707                 c,
 708                 "R",
 709                 Token::Char(ReservedChar::Backline),
 710                 |tokens, pos| {
 711                     pos < 2
 712                         || !tokens[pos - 1].eq_char(ReservedChar::OpenBracket)
 713                         || tokens[pos - 2].get_other() != Some("searchIndex")
 714                 },
 715             )
 716         })
 717         .to_string();
 718     assert_eq!(result, expected_result);
 719
 720     let source = r#"var searchIndex = {};searchIndex['duplicate_paths'] = {'aaaaaaaa': 'bbbbbbbb', 'bbbbbbbb': 'aaaaaaaa', 'duplicate_paths': 'aaaaaaaa', 'x': 'duplicate_paths'};"#;
 721     let expected_result = "var R=[\"bbbbbbbb\",\"aaaaaaaa\",\"duplicate_paths\"];\nvar searchIndex={};searchIndex['duplicate_paths']={R[1]:R[0],R[0]:R[1],R[2]:R[1],'x':R[2]}";
 722
 723     let result = simple_minify(source)
 724         .apply(::js::clean_tokens)
 725         .apply(|c| {
 726             aggregate_strings_into_array_with_separation_filter(
 727                 c,
 728                 "R",
 729                 Token::Char(ReservedChar::Backline),
 730                 |tokens, pos| {
 731                     pos < 2
 732                         || !tokens[pos - 1].eq_char(ReservedChar::OpenBracket)
 733                         || tokens[pos - 2].get_other() != Some("searchIndex")
 734                 },
 735             )
 736         })
 737         .to_string();
 738     assert_eq!(result, expected_result);
 739 }
 740
 741 #[test]
 742 fn aggregate_strings_in_array_existing() {
 743     let source = r#"var R=[];var x = ["a nice string", "a nice string", "another nice string", "cake!",
 744                              "cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 745     let expected_result = "var R=[\"a nice string\",\"cake!\"];var x=[R[0],R[0],\
 746                            \"another nice string\",R[1],R[1],R[0],R[1],R[1],R[1]]";
 747
 748     let result = simple_minify(source)
 749         .apply(::js::clean_tokens)
 750         .apply(|c| aggregate_strings_into_array(c, "R"))
 751         .to_string();
 752     assert_eq!(result, expected_result);
 753
 754     let source = r#"var R=["a nice string"];var x = ["a nice string", "a nice string", "another nice string", "cake!",
 755                              "cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 756     let expected_result = "var R=[\"a nice string\",\"cake!\"];var x=[R[0],R[0],\
 757                            \"another nice string\",R[1],R[1],R[0],R[1],R[1],R[1]]";
 758
 759     let result = simple_minify(source)
 760         .apply(::js::clean_tokens)
 761         .apply(|c| aggregate_strings_into_array(c, "R"))
 762         .to_string();
 763     assert_eq!(result, expected_result);
 764
 765     let source = r#"var y = 12;var R=["a nice string"];var x = ["a nice string", "a nice string", "another nice string", "cake!",
 766                              "cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 767     let expected_result = "var y=12;var R=[\"a nice string\",\"cake!\"];var x=[R[0],R[0],\
 768                            \"another nice string\",R[1],R[1],R[0],R[1],R[1],R[1]]";
 769
 770     let result = simple_minify(source)
 771         .apply(::js::clean_tokens)
 772         .apply(|c| aggregate_strings_into_array(c, "R"))
 773         .to_string();
 774     assert_eq!(result, expected_result);
 775
 776     let source = r#"var R=["osef1", "o2", "damn"];
 777                     var x = ["a nice string", "a nice string", "another nice string", "cake!",
 778                              "cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 779     let expected_result = "var R=[\"osef1\",\"o2\",\"damn\",\"a nice string\",\"cake!\"];\
 780                            var x=[R[3],R[3],\"another nice string\",R[4],R[4],R[3],R[4],R[4],R[4]]";
 781
 782     let result = simple_minify(source)
 783         .apply(::js::clean_tokens)
 784         .apply(|c| aggregate_strings_into_array(c, "R"))
 785         .to_string();
 786     assert_eq!(result, expected_result);
 787 }
 788
 789 #[test]
 790 fn string_duplicates() {
 791     let source = r#"var x = ["a nice string", "a nice string", "another nice string", "cake!",
 792                              "cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 793     let expected_result = "var r_aa=\"a nice string\",r_ba=\"cake!\";var x=[r_aa,r_aa,\
 794                            \"another nice string\",r_ba,r_ba,r_aa,r_ba,r_ba,r_ba]";
 795
 796     let result = simple_minify(source)
 797         .apply(aggregate_strings)
 798         .apply(::js::clean_tokens)
 799         .to_string();
 800     assert_eq!(result, expected_result);
 801 }
 802
 803 #[test]
 804 fn already_existing_var() {
 805     let source = r#"var r_aa = "a nice string"; var x = ["a nice string", "a nice string",
 806                     "another nice string", "cake!",
 807                     "cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 808     let expected_result = "var r_ba=\"cake!\";var r_aa=\"a nice string\";var x=[r_aa,r_aa,\
 809                            \"another nice string\",r_ba,r_ba,r_aa,r_ba,r_ba,r_ba]";
 810
 811     let result = simple_minify(source)
 812         .apply(aggregate_strings)
 813         .apply(::js::clean_tokens)
 814         .to_string();
 815     assert_eq!(result, expected_result);
 816 }
 817
 818 #[test]
 819 fn string_duplicates_variables_already_exist() {
 820     let source = r#"var r_aa=1;var x = ["a nice string", "a nice string", "another nice string", "cake!",
 821                              "cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 822     let expected_result = "var r_ba=\"a nice string\",r_ca=\"cake!\";\
 823                            var r_aa=1;var x=[r_ba,r_ba,\
 824                            \"another nice string\",r_ca,r_ca,r_ba,r_ca,r_ca,r_ca]";
 825
 826     let result = simple_minify(source)
 827         .apply(aggregate_strings)
 828         .apply(::js::clean_tokens)
 829         .to_string();
 830     assert_eq!(result, expected_result);
 831 }
 832
 833 #[test]
 834 fn string_duplicates_with_separator() {
 835     use self::token::ReservedChar;
 836
 837     let source = r#"var x = ["a nice string", "a nice string", "another nice string", "cake!",
 838                              "cake!", "a nice string", "cake!", "cake!", "cake!"];"#;
 839     let expected_result = "var r_aa=\"a nice string\",r_ba=\"cake!\";\nvar x=[r_aa,r_aa,\
 840                            \"another nice string\",r_ba,r_ba,r_aa,r_ba,r_ba,r_ba]";
 841     let result = simple_minify(source)
 842         .apply(::js::clean_tokens)
 843         .apply(|f| aggregate_strings_with_separation(f, Token::Char(ReservedChar::Backline)))
 844         .to_string();
 845     assert_eq!(result, expected_result);
 846 }
 847
 848 #[test]
 849 fn clean_except() {
 850     use self::token::ReservedChar;
 851
 852     let source = r#"var x = [1, 2, 3];
 853 var y = "salut";
 854 var z = "ok!";"#;
 855     let expected = r#"var x=[1,2,3];
 856 var y="salut";
 857 var z="ok!""#;
 858
 859     let result = simple_minify(source)
 860         .apply(|f| ::js::clean_tokens_except(f, |c| c.get_char() != Some(ReservedChar::Backline)))
 861         .to_string();
 862     assert_eq!(result, expected);
 863 }
 864
 865 #[test]
 866 fn clean_except2() {
 867     use self::token::ReservedChar;
 868
 869     let source = "let x = [ 1, 2, \t3];";
 870     let expected = "let x = [ 1, 2, 3];";
 871
 872     let result = simple_minify(source)
 873         .apply(|f| {
 874             ::js::clean_tokens_except(f, |c| {
 875                 c.get_char() != Some(ReservedChar::Space)
 876                     && c.get_char() != Some(ReservedChar::SemiColon)
 877             })
 878         })
 879         .to_string();
 880     assert_eq!(result, expected);
 881 }
 882
 883 #[test]
 884 fn clean_except3() {
 885     use self::token::ReservedChar;
 886
 887     let source = "let x = [ 1, 2, \t3];";
 888     let expected = "let x=[1,2,\t3];";
 889
 890     let result = simple_minify(source)
 891         .apply(|f| {
 892             ::js::clean_tokens_except(f, |c| {
 893                 c.get_char() != Some(ReservedChar::Tab)
 894                     && c.get_char() != Some(ReservedChar::SemiColon)
 895             })
 896         })
 897         .to_string();
 898     assert_eq!(result, expected);
 899 }
 900
 901 #[test]
 902 fn name_generator() {
 903     let s = ::std::iter::repeat('a').take(36).collect::<String>();
 904     // We need to generate enough long strings to reach the point that the name generator
 905     // generates names with 3 characters.
 906     let s = ::std::iter::repeat(s)
 907         .take(20000)
 908         .enumerate()
 909         .map(|(pos, s)| format!("{}{}", s, pos))
 910         .collect::<Vec<_>>();
 911     let source = format!(
 912         "var x = [{}];",
 913         s.iter()
 914             .map(|s| format!("\"{0}\",\"{0}\"", s))
 915             .collect::<Vec<_>>()
 916             .join(",")
 917     );
 918     let result = simple_minify(&source)
 919         .apply(::js::clean_tokens)
 920         .apply(aggregate_strings)
 921         .to_string();
 922     assert!(result.find(",r_aaa=").is_some());
 923     assert!(result.find(",r_ab=").unwrap() < result.find(",r_ba=").unwrap());
 924 }
 925
 926 #[test]
 927 fn simple_quote() {
 928     let source = r#"var x = "\\";"#;
 929     let expected_result = r#"var x="\\""#;
 930     assert_eq!(minify(source), expected_result);
 931 }
 932
 933 #[test]
 934 fn js_minify_test() {
 935     let source = r##"
 936 var foo = "something";
 937
 938 var another_var = 2348323;
 939
 940 // who doesn't like comments?
 941 /* and even longer comments?
 942
 943 like
 944 on
 945 a
 946 lot
 947 of
 948 lines!
 949
 950 Fun!
 951 */
 952 function far_away(x, y) {
 953     var x2 = x + 4;
 954     return x * x2 + y;
 955 }
 956
 957 // this call is useless
 958 far_away(another_var, 12);
 959 // this call is useless too
 960 far_away(another_var, 12);
 961 "##;
 962
 963     let expected_result = "var foo=\"something\";var another_var=2348323;function far_away(x,y){\
 964                            var x2=x+4;return x*x2+y}far_away(another_var,12);far_away(another_var,\
 965                            12)";
 966     assert_eq!(minify(source), expected_result);
 967 }
 968
 969 #[test]
 970 fn another_js_test() {
 971     let source = r#"
 972 /*! let's keep this license
 973  *
 974  * because everyone likes licenses!
 975  *
 976  * right?
 977  */
 978
 979 function forEach(data, func) {
 980     for (var i = 0; i < data.length; ++i) {
 981         func(data[i]);
 982     }
 983 }
 984
 985 forEach([0, 1, 2, 3, 4,
 986          5, 6, 7, 8, 9], function (x) {
 987             console.log(x);
 988          });
 989 // I think we're done?
 990 console.log('done!');
 991 "#;
 992
 993     let expected_result = r#"/*! let's keep this license
 994  *
 995  * because everyone likes licenses!
 996  *
 997  * right?
 998  */function forEach(data,func){for(var i=0;i<data.length;++i){func(data[i])}}forEach([0,1,2,3,4,5,6,7,8,9],function(x){console.log(x)});console.log('done!')"#;
 999     assert_eq!(minify(source), expected_result);
1000 }
1001
1002 #[test]
1003 fn comment_issue() {
1004     let source = r#"
1005 search_input.onchange = function(e) {
1006     // Do NOT e.preventDefault() here. It will prevent pasting.
1007     clearTimeout(searchTimeout);
1008     // zero-timeout necessary here because at the time of event handler execution the
1009     // pasted content is not in the input field yet. Shouldn’t make any difference for
1010     // change, though.
1011     setTimeout(search, 0);
1012 };
1013 "#;
1014     let expected_result = "search_input.onchange=function(e){clearTimeout(searchTimeout);\
1015                            setTimeout(search,0)}";
1016     assert_eq!(minify(source), expected_result);
1017 }
1018
1019 #[test]
1020 fn missing_whitespace() {
1021     let source = r#"
1022 for (var entry in results) {
1023     if (results.hasOwnProperty(entry)) {
1024         ar.push(results[entry]);
1025     }
1026 }"#;
1027     let expected_result = "for(var entry in results){if(results.hasOwnProperty(entry)){\
1028                            ar.push(results[entry])}}";
1029     assert_eq!(minify(source), expected_result);
1030 }
1031
1032 #[test]
1033 fn weird_regex_issue() {
1034     let source = r#"
1035 val = val.replace(/\_/g, "");
1036
1037 var valGenerics = extractGenerics(val);"#;
1038     let expected_result = "val=val.replace(/\\_/g,\"\");var valGenerics=extractGenerics(val)";
1039     assert_eq!(minify(source), expected_result);
1040 }
1041
1042 #[test]
1043 fn keep_space() {
1044     let source = "return 12;return x;";
1045
1046     let expected_result = "return 12;return x";
1047     assert_eq!(minify(source), expected_result);
1048
1049     assert_eq!("t in e", minify("t in e"));
1050     assert_eq!("t+1 in e", minify("t + 1 in e"));
1051     assert_eq!("t-1 in e", minify("t - 1 in e"));
1052     assert_eq!("'a'in e", minify("'a' in e"));
1053     assert_eq!("/a/g in e", minify("/a/g in e"));
1054     assert_eq!("/a/i in e", minify("/a/i in e"));
1055
1056     assert_eq!("t instanceof e", minify("t instanceof e"));
1057     assert_eq!("t+1 instanceof e", minify("t + 1 instanceof e"));
1058     assert_eq!("t-1 instanceof e", minify("t - 1 instanceof e"));
1059     assert_eq!("'a'instanceof e", minify("'a' instanceof e"));
1060     assert_eq!("/a/g instanceof e", minify("/a/g instanceof e"));
1061     assert_eq!("/a/i instanceof e", minify("/a/i instanceof e"));
1062 }
1063
1064 #[test]
1065 fn test_remove_extra_whitespace_before_typeof() {
1066     let source = "var x = typeof 'foo';var y = typeof x;case typeof 'foo': 'bla'";
1067
1068     let expected_result = "var x=typeof'foo';var y=typeof x;case typeof'foo':'bla'";
1069     assert_eq!(minify(source), expected_result);
1070 }
1071
1072 #[test]
1073 fn test_remove_extra_whitespace_before_in() {
1074     let source = r#"if ("key" in ev && typeof ev) { return true; }
1075 if (x in ev && typeof ev) { return true; }
1076 if (true in ev) { return true; }"#;
1077
1078     let expected_result = r#"if("key"in ev&&typeof ev){return true}if(x in ev&&typeof ev){return true}if(true in ev){return true}"#;
1079     assert_eq!(minify(source), expected_result);
1080 }
1081
1082 #[test]
1083 fn test_remove_extra_whitespace_before_operator() {
1084     let source = "( x ) / 2; x / y;x /= y";
1085
1086     let expected_result = "(x)/2;x/y;x/=y";
1087     assert_eq!(minify(source), expected_result);
1088 }
1089
1090 #[test]
1091 fn check_regex_syntax() {
1092     let source = "console.log(/MSIE|Trident|Edge/.test(window.navigator.userAgent));";
1093     let expected = "console.log(/MSIE|Trident|Edge/.test(window.navigator.userAgent))";
1094     assert_eq!(minify(source), expected);
1095 }
1096
1097 #[test]
1098 fn minify_minified() {
1099     let source = "function (i, n, a) { i[n].type.replace(/ *;(.|\\s)*/,\"\")===t&&a.push(i[n].MathJax.elementJax);return a}";
1100     let expected = "function(i,n,a){i[n].type.replace(/ *;(.|\\s)*/,\"\")===t&&a.push(i[n].MathJax.elementJax);return a}";
1101     assert_eq!(minify(source), expected);
1102 }
1103
1104 #[test]
1105 fn check_string() {
1106     let source = r###"
1107         const a = 123;
1108         const b = "123";
1109         const c = `the number is ${a}  <-- note the spaces here`;
1110         const d = `      ${a}         ${b}      `;
1111     "###;
1112     let expected = "const a=123;const b=\"123\";const c=`the number is ${a}  <-- note the spaces \
1113     here`;const d=`      ${a}         ${b}      `";
1114     assert_eq!(minify(source), expected);
1115 }
1116
1117 // TODO: requires AST to fix this issue!
1118 /*#[test]
1119 fn no_semi_colon() {
1120     let source = r#"
1121 console.log(1)
1122 console.log(2)
1123 var x = 12;
1124 "#;
1125     let expected_result = r#"console.log(1);console.log(2);var x=12;"#;
1126     assert_eq!(minify(source), expected_result);
1127 }*/
1128
1129 // TODO: requires AST to fix this issue!
1130 /*#[test]
1131 fn correct_replace_for_backline() {
1132     let source = r#"
1133 function foo() {
1134     return
1135     12;
1136 }
1137 "#;
1138     let expected_result = r#"function foo(){return 12;}"#;
1139     assert_eq!(minify(source), expected_result);
1140 }*/