vendor/minifier/src/css/token.rs

   1 // MIT License
   2 //
   3 // Copyright (c) 2018 Guillaume Gomez
   4 //
   5 // Permission is hereby granted, free of charge, to any person obtaining a copy
   6 // of this software and associated documentation files (the "Software"), to deal
   7 // in the Software without restriction, including without limitation the rights
   8 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 // copies of the Software, and to permit persons to whom the Software is
  10 // furnished to do so, subject to the following conditions:
  11 //
  12 // The above copyright notice and this permission notice shall be included in all
  13 // copies or substantial portions of the Software.
  14 //
  15 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21 // SOFTWARE.
  22
  23 use std::convert::TryFrom;
  24 use std::fmt;
  25 use std::iter::Peekable;
  26 use std::str::CharIndices;
  27
  28 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
  29 pub enum ReservedChar {
  30     Comma,
  31     SuperiorThan,
  32     OpenParenthese,
  33     CloseParenthese,
  34     OpenCurlyBrace,
  35     CloseCurlyBrace,
  36     OpenBracket,
  37     CloseBracket,
  38     Colon,
  39     SemiColon,
  40     Slash,
  41     Plus,
  42     EqualSign,
  43     Space,
  44     Tab,
  45     Backline,
  46     Star,
  47     Quote,
  48     DoubleQuote,
  49     Pipe,
  50     Tilde,
  51     Dollar,
  52     Circumflex,
  53 }
  54
  55 impl fmt::Display for ReservedChar {
  56     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
  57         write!(
  58             f,
  59             "{}",
  60             match *self {
  61                 ReservedChar::Comma => ',',
  62                 ReservedChar::OpenParenthese => '(',
  63                 ReservedChar::CloseParenthese => ')',
  64                 ReservedChar::OpenCurlyBrace => '{',
  65                 ReservedChar::CloseCurlyBrace => '}',
  66                 ReservedChar::OpenBracket => '[',
  67                 ReservedChar::CloseBracket => ']',
  68                 ReservedChar::Colon => ':',
  69                 ReservedChar::SemiColon => ';',
  70                 ReservedChar::Slash => '/',
  71                 ReservedChar::Star => '*',
  72                 ReservedChar::Plus => '+',
  73                 ReservedChar::EqualSign => '=',
  74                 ReservedChar::Space => ' ',
  75                 ReservedChar::Tab => '\t',
  76                 ReservedChar::Backline => '\n',
  77                 ReservedChar::SuperiorThan => '>',
  78                 ReservedChar::Quote => '\'',
  79                 ReservedChar::DoubleQuote => '"',
  80                 ReservedChar::Pipe => '|',
  81                 ReservedChar::Tilde => '~',
  82                 ReservedChar::Dollar => '$',
  83                 ReservedChar::Circumflex => '^',
  84             }
  85         )
  86     }
  87 }
  88
  89 impl TryFrom<char> for ReservedChar {
  90     type Error = &'static str;
  91
  92     fn try_from(value: char) -> Result<ReservedChar, Self::Error> {
  93         match value {
  94             '\'' => Ok(ReservedChar::Quote),
  95             '"' => Ok(ReservedChar::DoubleQuote),
  96             ',' => Ok(ReservedChar::Comma),
  97             '(' => Ok(ReservedChar::OpenParenthese),
  98             ')' => Ok(ReservedChar::CloseParenthese),
  99             '{' => Ok(ReservedChar::OpenCurlyBrace),
 100             '}' => Ok(ReservedChar::CloseCurlyBrace),
 101             '[' => Ok(ReservedChar::OpenBracket),
 102             ']' => Ok(ReservedChar::CloseBracket),
 103             ':' => Ok(ReservedChar::Colon),
 104             ';' => Ok(ReservedChar::SemiColon),
 105             '/' => Ok(ReservedChar::Slash),
 106             '*' => Ok(ReservedChar::Star),
 107             '+' => Ok(ReservedChar::Plus),
 108             '=' => Ok(ReservedChar::EqualSign),
 109             ' ' => Ok(ReservedChar::Space),
 110             '\t' => Ok(ReservedChar::Tab),
 111             '\n' | '\r' => Ok(ReservedChar::Backline),
 112             '>' => Ok(ReservedChar::SuperiorThan),
 113             '|' => Ok(ReservedChar::Pipe),
 114             '~' => Ok(ReservedChar::Tilde),
 115             '$' => Ok(ReservedChar::Dollar),
 116             '^' => Ok(ReservedChar::Circumflex),
 117             _ => Err("Unknown reserved char"),
 118         }
 119     }
 120 }
 121
 122 impl ReservedChar {
 123     fn is_useless(&self) -> bool {
 124         *self == ReservedChar::Space
 125             || *self == ReservedChar::Tab
 126             || *self == ReservedChar::Backline
 127     }
 128
 129     fn is_operator(&self) -> bool {
 130         Operator::try_from(*self).is_ok()
 131     }
 132 }
 133
 134 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 135 pub enum Operator {
 136     Plus,
 137     Multiply,
 138     Minus,
 139     Modulo,
 140     Divide,
 141 }
 142
 143 impl fmt::Display for Operator {
 144     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 145         write!(
 146             f,
 147             "{}",
 148             match *self {
 149                 Operator::Plus => '+',
 150                 Operator::Multiply => '*',
 151                 Operator::Minus => '-',
 152                 Operator::Modulo => '%',
 153                 Operator::Divide => '/',
 154             }
 155         )
 156     }
 157 }
 158
 159 impl TryFrom<char> for Operator {
 160     type Error = &'static str;
 161
 162     fn try_from(value: char) -> Result<Operator, Self::Error> {
 163         match value {
 164             '+' => Ok(Operator::Plus),
 165             '*' => Ok(Operator::Multiply),
 166             '-' => Ok(Operator::Minus),
 167             '%' => Ok(Operator::Modulo),
 168             '/' => Ok(Operator::Divide),
 169             _ => Err("Unknown operator"),
 170         }
 171     }
 172 }
 173
 174 impl TryFrom<ReservedChar> for Operator {
 175     type Error = &'static str;
 176
 177     fn try_from(value: ReservedChar) -> Result<Operator, Self::Error> {
 178         match value {
 179             ReservedChar::Slash => Ok(Operator::Divide),
 180             ReservedChar::Star => Ok(Operator::Multiply),
 181             ReservedChar::Plus => Ok(Operator::Plus),
 182             _ => Err("Unknown operator"),
 183         }
 184     }
 185 }
 186
 187 #[derive(Eq, PartialEq, Clone, Debug)]
 188 pub enum SelectorElement<'a> {
 189     PseudoClass(&'a str),
 190     Class(&'a str),
 191     Id(&'a str),
 192     Tag(&'a str),
 193     Media(&'a str),
 194 }
 195
 196 impl<'a> TryFrom<&'a str> for SelectorElement<'a> {
 197     type Error = &'static str;
 198
 199     fn try_from(value: &'a str) -> Result<SelectorElement, Self::Error> {
 200         if let Some(value) = value.strip_prefix('.') {
 201             if value.is_empty() {
 202                 Err("cannot determine selector")
 203             } else {
 204                 Ok(SelectorElement::Class(value))
 205             }
 206         } else if let Some(value) = value.strip_prefix('#') {
 207             if value.is_empty() {
 208                 Err("cannot determine selector")
 209             } else {
 210                 Ok(SelectorElement::Id(value))
 211             }
 212         } else if let Some(value) = value.strip_prefix('@') {
 213             if value.is_empty() {
 214                 Err("cannot determine selector")
 215             } else {
 216                 Ok(SelectorElement::Media(value))
 217             }
 218         } else if let Some(value) = value.strip_prefix(':') {
 219             if value.is_empty() {
 220                 Err("cannot determine selector")
 221             } else {
 222                 Ok(SelectorElement::PseudoClass(value))
 223             }
 224         } else if value.chars().next().unwrap_or(' ').is_alphabetic() {
 225             Ok(SelectorElement::Tag(value))
 226         } else {
 227             Err("unknown selector")
 228         }
 229     }
 230 }
 231
 232 impl<'a> fmt::Display for SelectorElement<'a> {
 233     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 234         match *self {
 235             SelectorElement::Class(c) => write!(f, ".{}", c),
 236             SelectorElement::Id(i) => write!(f, "#{}", i),
 237             SelectorElement::Tag(t) => write!(f, "{}", t),
 238             SelectorElement::Media(m) => write!(f, "@{} ", m),
 239             SelectorElement::PseudoClass(pc) => write!(f, ":{}", pc),
 240         }
 241     }
 242 }
 243
 244 #[derive(Eq, PartialEq, Clone, Debug, Copy)]
 245 pub enum SelectorOperator {
 246     /// `~=`
 247     OneAttributeEquals,
 248     /// `|=`
 249     EqualsOrStartsWithFollowedByDash,
 250     /// `$=`
 251     EndsWith,
 252     /// `^=`
 253     FirstStartsWith,
 254     /// `*=`
 255     Contains,
 256 }
 257
 258 impl fmt::Display for SelectorOperator {
 259     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 260         match *self {
 261             SelectorOperator::OneAttributeEquals => write!(f, "~="),
 262             SelectorOperator::EqualsOrStartsWithFollowedByDash => write!(f, "|="),
 263             SelectorOperator::EndsWith => write!(f, "$="),
 264             SelectorOperator::FirstStartsWith => write!(f, "^="),
 265             SelectorOperator::Contains => write!(f, "*="),
 266         }
 267     }
 268 }
 269
 270 #[derive(Eq, PartialEq, Clone, Debug)]
 271 pub enum Token<'a> {
 272     /// Comment.
 273     Comment(&'a str),
 274     /// Comment starting with `/**`.
 275     License(&'a str),
 276     Char(ReservedChar),
 277     Other(&'a str),
 278     SelectorElement(SelectorElement<'a>),
 279     String(&'a str),
 280     SelectorOperator(SelectorOperator),
 281     Operator(Operator),
 282 }
 283
 284 impl<'a> fmt::Display for Token<'a> {
 285     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 286         match *self {
 287             // Token::AtRule(at_rule) => write!(f, "{}", at_rule, content),
 288             // Token::ElementRule(selectors) => write!(f, "{}", x),
 289             Token::Comment(c) => write!(f, "{}", c),
 290             Token::License(l) => writeln!(f, "/*!{}*/", l),
 291             Token::Char(c) => write!(f, "{}", c),
 292             Token::Other(s) => write!(f, "{}", s),
 293             Token::SelectorElement(ref se) => write!(f, "{}", se),
 294             Token::String(s) => write!(f, "{}", s),
 295             Token::SelectorOperator(so) => write!(f, "{}", so),
 296             Token::Operator(op) => write!(f, "{}", op),
 297         }
 298     }
 299 }
 300
 301 impl<'a> Token<'a> {
 302     fn is_comment(&self) -> bool {
 303         matches!(*self, Token::Comment(_))
 304     }
 305
 306     fn is_char(&self) -> bool {
 307         matches!(*self, Token::Char(_))
 308     }
 309
 310     fn get_char(&self) -> Option<ReservedChar> {
 311         match *self {
 312             Token::Char(c) => Some(c),
 313             _ => None,
 314         }
 315     }
 316
 317     fn is_useless(&self) -> bool {
 318         match *self {
 319             Token::Char(c) => c.is_useless(),
 320             _ => false,
 321         }
 322     }
 323
 324     fn is_media(&self, media: &str) -> bool {
 325         match *self {
 326             Token::SelectorElement(SelectorElement::Media(s)) => s == media,
 327             _ => false,
 328         }
 329     }
 330
 331     fn is_a_media(&self) -> bool {
 332         matches!(*self, Token::SelectorElement(SelectorElement::Media(_)))
 333     }
 334
 335     fn is_a_license(&self) -> bool {
 336         matches!(*self, Token::License(_))
 337     }
 338
 339     fn is_operator(&self) -> bool {
 340         match *self {
 341             Token::Operator(_) => true,
 342             Token::Char(c) => c.is_operator(),
 343             _ => false,
 344         }
 345     }
 346 }
 347
 348 impl<'a> PartialEq<ReservedChar> for Token<'a> {
 349     fn eq(&self, other: &ReservedChar) -> bool {
 350         match *self {
 351             Token::Char(c) => c == *other,
 352             _ => false,
 353         }
 354     }
 355 }
 356
 357 fn get_comment<'a>(
 358     source: &'a str,
 359     iterator: &mut Peekable<CharIndices>,
 360     start_pos: &mut usize,
 361 ) -> Option<Token<'a>> {
 362     let mut prev = ReservedChar::Quote;
 363     *start_pos += 1;
 364     let builder = if let Some((_, c)) = iterator.next() {
 365         if c == '!' || (c == '*' && iterator.peek().map(|(_, c)| c) != Some(&'/')) {
 366             *start_pos += 1;
 367             Token::License
 368         } else {
 369             if let Ok(c) = ReservedChar::try_from(c) {
 370                 prev = c;
 371             }
 372             Token::Comment
 373         }
 374     } else {
 375         Token::Comment
 376     };
 377
 378     for (pos, c) in iterator {
 379         if let Ok(c) = ReservedChar::try_from(c) {
 380             if c == ReservedChar::Slash && prev == ReservedChar::Star {
 381                 let ret = Some(builder(&source[*start_pos..pos - 1]));
 382                 *start_pos = pos;
 383                 return ret;
 384             }
 385             prev = c;
 386         } else {
 387             prev = ReservedChar::Space;
 388         }
 389     }
 390     None
 391 }
 392
 393 fn get_string<'a>(
 394     source: &'a str,
 395     iterator: &mut Peekable<CharIndices>,
 396     start_pos: &mut usize,
 397     start: ReservedChar,
 398 ) -> Option<Token<'a>> {
 399     while let Some((pos, c)) = iterator.next() {
 400         if c == '\\' {
 401             // we skip next character
 402             iterator.next();
 403             continue;
 404         }
 405         if let Ok(c) = ReservedChar::try_from(c) {
 406             if c == start {
 407                 let ret = Some(Token::String(&source[*start_pos..pos + 1]));
 408                 *start_pos = pos;
 409                 return ret;
 410             }
 411         }
 412     }
 413     None
 414 }
 415
 416 fn fill_other<'a>(
 417     source: &'a str,
 418     v: &mut Vec<Token<'a>>,
 419     start: usize,
 420     pos: usize,
 421     is_in_block: isize,
 422     is_in_media: bool,
 423     is_in_attribute_selector: bool,
 424 ) {
 425     if start < pos {
 426         if !is_in_attribute_selector
 427             && ((is_in_block == 0 && !is_in_media) || (is_in_media && is_in_block == 1))
 428         {
 429             let mut is_pseudo_class = false;
 430             let mut add = 0;
 431             if let Some(&Token::Char(ReservedChar::Colon)) = v.last() {
 432                 is_pseudo_class = true;
 433                 add = 1;
 434             }
 435             if let Ok(s) = SelectorElement::try_from(&source[start - add..pos]) {
 436                 if is_pseudo_class {
 437                     v.pop();
 438                 }
 439                 v.push(Token::SelectorElement(s));
 440             } else {
 441                 let s = &source[start..pos];
 442                 if !s.starts_with(':')
 443                     && !s.starts_with('.')
 444                     && !s.starts_with('#')
 445                     && !s.starts_with('@')
 446                 {
 447                     v.push(Token::Other(s));
 448                 }
 449             }
 450         } else {
 451             v.push(Token::Other(&source[start..pos]));
 452         }
 453     }
 454 }
 455
 456 #[allow(clippy::comparison_chain)]
 457 pub fn tokenize<'a>(source: &'a str) -> Result<Tokens<'a>, &'static str> {
 458     let mut v = Vec::with_capacity(1000);
 459     let mut iterator = source.char_indices().peekable();
 460     let mut start = 0;
 461     let mut is_in_block: isize = 0;
 462     let mut is_in_media = false;
 463     let mut is_in_attribute_selector = false;
 464
 465     loop {
 466         let (mut pos, c) = match iterator.next() {
 467             Some(x) => x,
 468             None => {
 469                 fill_other(
 470                     source,
 471                     &mut v,
 472                     start,
 473                     source.len(),
 474                     is_in_block,
 475                     is_in_media,
 476                     is_in_attribute_selector,
 477                 );
 478                 break;
 479             }
 480         };
 481         if let Ok(c) = ReservedChar::try_from(c) {
 482             fill_other(
 483                 source,
 484                 &mut v,
 485                 start,
 486                 pos,
 487                 is_in_block,
 488                 is_in_media,
 489                 is_in_attribute_selector,
 490             );
 491             is_in_media = is_in_media
 492                 || v.last()
 493                     .unwrap_or(&Token::Char(ReservedChar::Space))
 494                     .is_media("media");
 495             if_match! {
 496                 c == ReservedChar::Quote || c == ReservedChar::DoubleQuote => {
 497                     if let Some(s) = get_string(source, &mut iterator, &mut pos, c) {
 498                         v.push(s);
 499                     }
 500                 },
 501                 c == ReservedChar::Star &&
 502                 *v.last().unwrap_or(&Token::Char(ReservedChar::Space)) == ReservedChar::Slash => {
 503                     v.pop();
 504                     if let Some(s) = get_comment(source, &mut iterator, &mut pos) {
 505                         v.push(s);
 506                     }
 507                 },
 508                 c == ReservedChar::OpenBracket => {
 509                     if is_in_attribute_selector {
 510                         return Err("Already in attribute selector");
 511                     }
 512                     is_in_attribute_selector = true;
 513                     v.push(Token::Char(c));
 514                 },
 515                 c == ReservedChar::CloseBracket => {
 516                     if !is_in_attribute_selector {
 517                         return Err("Unexpected ']'");
 518                     }
 519                     is_in_attribute_selector = false;
 520                     v.push(Token::Char(c));
 521                 },
 522                 c == ReservedChar::OpenCurlyBrace => {
 523                     is_in_block += 1;
 524                     v.push(Token::Char(c));
 525                 },
 526                 c == ReservedChar::CloseCurlyBrace => {
 527                     is_in_block -= 1;
 528                     if is_in_block < 0 {
 529                         return Err("Too much '}'");
 530                     } else if is_in_block == 0 {
 531                         is_in_media = false;
 532                     }
 533                     v.push(Token::Char(c));
 534                 },
 535                 c == ReservedChar::EqualSign => {
 536                     match match v.last()
 537                                  .unwrap_or(&Token::Char(ReservedChar::Space))
 538                                  .get_char()
 539                                  .unwrap_or(ReservedChar::Space) {
 540                         ReservedChar::Tilde => Some(SelectorOperator::OneAttributeEquals),
 541                         ReservedChar::Pipe => Some(SelectorOperator::EqualsOrStartsWithFollowedByDash),
 542                         ReservedChar::Dollar => Some(SelectorOperator::EndsWith),
 543                         ReservedChar::Circumflex => Some(SelectorOperator::FirstStartsWith),
 544                         ReservedChar::Star => Some(SelectorOperator::Contains),
 545                         _ => None,
 546                     } {
 547                         Some(r) => {
 548                             v.pop();
 549                             v.push(Token::SelectorOperator(r));
 550                         }
 551                         None => v.push(Token::Char(c)),
 552                     }
 553                 },
 554                 !c.is_useless() => {
 555                     v.push(Token::Char(c));
 556                 },
 557                 !v.last().unwrap_or(&Token::Char(ReservedChar::Space)).is_useless() &&
 558                 (!v.last().unwrap_or(&Token::Char(ReservedChar::OpenCurlyBrace)).is_char() ||
 559                  v.last().unwrap_or(&Token::Char(ReservedChar::OpenCurlyBrace)).is_operator() ||
 560                  v.last().unwrap_or(&Token::Char(ReservedChar::OpenCurlyBrace))
 561                          .get_char() == Some(ReservedChar::CloseParenthese) ||
 562                  v.last().unwrap_or(&Token::Char(ReservedChar::OpenCurlyBrace))
 563                          .get_char() == Some(ReservedChar::CloseBracket)) => {
 564                     v.push(Token::Char(ReservedChar::Space));
 565                 },
 566                 let Ok(op) = Operator::try_from(c) => {
 567                     v.push(Token::Operator(op));
 568                 },
 569             }
 570             start = pos + 1;
 571         }
 572     }
 573     Ok(Tokens(clean_tokens(v)))
 574 }
 575
 576 fn clean_tokens(mut v: Vec<Token<'_>>) -> Vec<Token<'_>> {
 577     let mut i = 0;
 578     let mut is_in_calc = false;
 579     let mut paren = 0;
 580
 581     while i < v.len() {
 582         if v[i] == Token::Other("calc") {
 583             is_in_calc = true;
 584         } else if is_in_calc {
 585             if v[i] == Token::Char(ReservedChar::CloseParenthese) {
 586                 paren -= 1;
 587                 is_in_calc = paren != 0;
 588             } else if v[i] == Token::Char(ReservedChar::OpenParenthese) {
 589                 paren += 1;
 590             }
 591         }
 592
 593         if v[i].is_useless() {
 594             if i > 0 && v[i - 1] == Token::Char(ReservedChar::CloseBracket) {
 595                 if i + 1 < v.len()
 596                     && (v[i + 1].is_useless()
 597                         || v[i + 1] == Token::Char(ReservedChar::OpenCurlyBrace))
 598                 {
 599                     v.remove(i);
 600                     continue;
 601                 }
 602             } else if i > 0 && v[i - 1] == Token::Other("and") {
 603                 // retain the space after an and
 604             } else if (is_in_calc && v[i - 1].is_useless())
 605                 || !is_in_calc
 606                     && ((i > 0
 607                         && ((v[i - 1].is_char()
 608                             && v[i - 1] != Token::Char(ReservedChar::CloseParenthese))
 609                             || v[i - 1].is_a_media()
 610                             || v[i - 1].is_a_license()))
 611                         || (i < v.len() - 1 && v[i + 1].is_char()))
 612             {
 613                 v.remove(i);
 614                 continue;
 615             }
 616         } else if v[i].is_comment() {
 617             v.remove(i);
 618             continue;
 619         }
 620         i += 1;
 621     }
 622     v
 623 }
 624
 625 #[derive(Debug, PartialEq, Eq, Clone)]
 626 pub struct Tokens<'a>(pub Vec<Token<'a>>);
 627
 628 impl<'a> fmt::Display for Tokens<'a> {
 629     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 630         for token in self.0.iter() {
 631             write!(f, "{}", token)?;
 632         }
 633         Ok(())
 634     }
 635 }
 636
 637 #[test]
 638 fn css_basic() {
 639     let s = r#"
 640 /*! just some license */
 641 .foo > #bar p:hover {
 642     color: blue;
 643     background: "blue";
 644 }
 645
 646 /* a comment! */
 647 @media screen and (max-width: 640px) {
 648     .block:hover {
 649         display:    block;
 650     }
 651 }"#;
 652     let expected = vec![
 653         Token::License(" just some license "),
 654         Token::SelectorElement(SelectorElement::Class("foo")),
 655         Token::Char(ReservedChar::SuperiorThan),
 656         Token::SelectorElement(SelectorElement::Id("bar")),
 657         Token::Char(ReservedChar::Space),
 658         Token::SelectorElement(SelectorElement::Tag("p")),
 659         Token::SelectorElement(SelectorElement::PseudoClass("hover")),
 660         Token::Char(ReservedChar::OpenCurlyBrace),
 661         Token::Other("color"),
 662         Token::Char(ReservedChar::Colon),
 663         Token::Other("blue"),
 664         Token::Char(ReservedChar::SemiColon),
 665         Token::Other("background"),
 666         Token::Char(ReservedChar::Colon),
 667         Token::String("\"blue\""),
 668         Token::Char(ReservedChar::SemiColon),
 669         Token::Char(ReservedChar::CloseCurlyBrace),
 670         Token::SelectorElement(SelectorElement::Media("media")),
 671         Token::Other("screen"),
 672         Token::Char(ReservedChar::Space),
 673         Token::Other("and"),
 674         Token::Char(ReservedChar::Space),
 675         Token::Char(ReservedChar::OpenParenthese),
 676         Token::Other("max-width"),
 677         Token::Char(ReservedChar::Colon),
 678         Token::Other("640px"),
 679         Token::Char(ReservedChar::CloseParenthese),
 680         Token::Char(ReservedChar::OpenCurlyBrace),
 681         Token::SelectorElement(SelectorElement::Class("block")),
 682         Token::SelectorElement(SelectorElement::PseudoClass("hover")),
 683         Token::Char(ReservedChar::OpenCurlyBrace),
 684         Token::Other("display"),
 685         Token::Char(ReservedChar::Colon),
 686         Token::Other("block"),
 687         Token::Char(ReservedChar::SemiColon),
 688         Token::Char(ReservedChar::CloseCurlyBrace),
 689         Token::Char(ReservedChar::CloseCurlyBrace),
 690     ];
 691     assert_eq!(tokenize(s), Ok(Tokens(expected)));
 692 }
 693
 694 #[test]
 695 fn elem_selector() {
 696     let s = r#"
 697 /** just some license */
 698 a[href*="example"] {
 699     background: yellow;
 700 }
 701 a[href$=".org"] {
 702   font-style: italic;
 703 }
 704 span[lang|="zh"] {
 705     color: red;
 706 }
 707 a[href^="/"] {
 708     background-color: gold;
 709 }
 710 div[value~="test"] {
 711     border-width: 1px;
 712 }
 713 span[lang="pt"] {
 714     font-size: 12em; /* I love big fonts */
 715 }
 716 "#;
 717     let expected = vec![
 718         Token::License(" just some license "),
 719         Token::SelectorElement(SelectorElement::Tag("a")),
 720         Token::Char(ReservedChar::OpenBracket),
 721         Token::Other("href"),
 722         Token::SelectorOperator(SelectorOperator::Contains),
 723         Token::String("\"example\""),
 724         Token::Char(ReservedChar::CloseBracket),
 725         Token::Char(ReservedChar::OpenCurlyBrace),
 726         Token::Other("background"),
 727         Token::Char(ReservedChar::Colon),
 728         Token::Other("yellow"),
 729         Token::Char(ReservedChar::SemiColon),
 730         Token::Char(ReservedChar::CloseCurlyBrace),
 731         Token::SelectorElement(SelectorElement::Tag("a")),
 732         Token::Char(ReservedChar::OpenBracket),
 733         Token::Other("href"),
 734         Token::SelectorOperator(SelectorOperator::EndsWith),
 735         Token::String("\".org\""),
 736         Token::Char(ReservedChar::CloseBracket),
 737         Token::Char(ReservedChar::OpenCurlyBrace),
 738         Token::Other("font-style"),
 739         Token::Char(ReservedChar::Colon),
 740         Token::Other("italic"),
 741         Token::Char(ReservedChar::SemiColon),
 742         Token::Char(ReservedChar::CloseCurlyBrace),
 743         Token::SelectorElement(SelectorElement::Tag("span")),
 744         Token::Char(ReservedChar::OpenBracket),
 745         Token::Other("lang"),
 746         Token::SelectorOperator(SelectorOperator::EqualsOrStartsWithFollowedByDash),
 747         Token::String("\"zh\""),
 748         Token::Char(ReservedChar::CloseBracket),
 749         Token::Char(ReservedChar::OpenCurlyBrace),
 750         Token::Other("color"),
 751         Token::Char(ReservedChar::Colon),
 752         Token::Other("red"),
 753         Token::Char(ReservedChar::SemiColon),
 754         Token::Char(ReservedChar::CloseCurlyBrace),
 755         Token::SelectorElement(SelectorElement::Tag("a")),
 756         Token::Char(ReservedChar::OpenBracket),
 757         Token::Other("href"),
 758         Token::SelectorOperator(SelectorOperator::FirstStartsWith),
 759         Token::String("\"/\""),
 760         Token::Char(ReservedChar::CloseBracket),
 761         Token::Char(ReservedChar::OpenCurlyBrace),
 762         Token::Other("background-color"),
 763         Token::Char(ReservedChar::Colon),
 764         Token::Other("gold"),
 765         Token::Char(ReservedChar::SemiColon),
 766         Token::Char(ReservedChar::CloseCurlyBrace),
 767         Token::SelectorElement(SelectorElement::Tag("div")),
 768         Token::Char(ReservedChar::OpenBracket),
 769         Token::Other("value"),
 770         Token::SelectorOperator(SelectorOperator::OneAttributeEquals),
 771         Token::String("\"test\""),
 772         Token::Char(ReservedChar::CloseBracket),
 773         Token::Char(ReservedChar::OpenCurlyBrace),
 774         Token::Other("border-width"),
 775         Token::Char(ReservedChar::Colon),
 776         Token::Other("1px"),
 777         Token::Char(ReservedChar::SemiColon),
 778         Token::Char(ReservedChar::CloseCurlyBrace),
 779         Token::SelectorElement(SelectorElement::Tag("span")),
 780         Token::Char(ReservedChar::OpenBracket),
 781         Token::Other("lang"),
 782         Token::Char(ReservedChar::EqualSign),
 783         Token::String("\"pt\""),
 784         Token::Char(ReservedChar::CloseBracket),
 785         Token::Char(ReservedChar::OpenCurlyBrace),
 786         Token::Other("font-size"),
 787         Token::Char(ReservedChar::Colon),
 788         Token::Other("12em"),
 789         Token::Char(ReservedChar::SemiColon),
 790         Token::Char(ReservedChar::CloseCurlyBrace),
 791     ];
 792     assert_eq!(tokenize(s), Ok(Tokens(expected)));
 793 }
 794
 795 #[test]
 796 fn check_media() {
 797     let s = "@media (max-width: 700px) { color: red; }";
 798
 799     let expected = vec![
 800         Token::SelectorElement(SelectorElement::Media("media")),
 801         Token::Char(ReservedChar::OpenParenthese),
 802         Token::Other("max-width"),
 803         Token::Char(ReservedChar::Colon),
 804         Token::Other("700px"),
 805         Token::Char(ReservedChar::CloseParenthese),
 806         Token::Char(ReservedChar::OpenCurlyBrace),
 807         Token::SelectorElement(SelectorElement::Tag("color")),
 808         Token::Char(ReservedChar::Colon),
 809         Token::Other("red"),
 810         Token::Char(ReservedChar::SemiColon),
 811         Token::Char(ReservedChar::CloseCurlyBrace),
 812     ];
 813
 814     assert_eq!(tokenize(s), Ok(Tokens(expected)));
 815 }
 816
 817 #[test]
 818 fn check_calc() {
 819     let s = ".foo { width: calc(100% - 34px); }";
 820
 821     let expected = vec![
 822         Token::SelectorElement(SelectorElement::Class("foo")),
 823         Token::Char(ReservedChar::OpenCurlyBrace),
 824         Token::Other("width"),
 825         Token::Char(ReservedChar::Colon),
 826         Token::Other("calc"),
 827         Token::Char(ReservedChar::OpenParenthese),
 828         Token::Other("100%"),
 829         Token::Char(ReservedChar::Space),
 830         Token::Other("-"),
 831         Token::Char(ReservedChar::Space),
 832         Token::Other("34px"),
 833         Token::Char(ReservedChar::CloseParenthese),
 834         Token::Char(ReservedChar::SemiColon),
 835         Token::Char(ReservedChar::CloseCurlyBrace),
 836     ];
 837     assert_eq!(tokenize(s), Ok(Tokens(expected)));
 838 }