New upstream version 1.29.0+dfsg1

[rustc.git] / src / vendor / minifier / src / css / token.rs

// MIT License
//
// Copyright (c) 2018 Guillaume Gomez
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

use std::fmt;
use std::iter::Peekable;
use std::str::CharIndices;

pub trait MyTryFrom<T>: Sized {
    type Error;
    fn try_from(value: T) -> Result<Self, Self::Error>;
}

#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub enum ReservedChar {
    Comma,
    SuperiorThan,
    OpenParenthese,
    CloseParenthese,
    OpenCurlyBrace,
    CloseCurlyBrace,
    OpenBracket,
    CloseBracket,
    Colon,
    SemiColon,
    Slash,
    Plus,
    EqualSign,
    Space,
    Tab,
    Backline,
    Star,
    Quote,
    DoubleQuote,
    Pipe,
    Tilde,
    Dollar,
    Circumflex,
}

impl fmt::Display for ReservedChar {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "{}",
               match *self {
                   ReservedChar::Comma           => ',',
                   ReservedChar::OpenParenthese  => '(',
                   ReservedChar::CloseParenthese => ')',
                   ReservedChar::OpenCurlyBrace  => '{',
                   ReservedChar::CloseCurlyBrace => '}',
                   ReservedChar::OpenBracket     => '[',
                   ReservedChar::CloseBracket    => ']',
                   ReservedChar::Colon           => ':',
                   ReservedChar::SemiColon       => ';',
                   ReservedChar::Slash           => '/',
                   ReservedChar::Star            => '*',
                   ReservedChar::Plus            => '+',
                   ReservedChar::EqualSign       => '=',
                   ReservedChar::Space           => ' ',
                   ReservedChar::Tab             => '\t',
                   ReservedChar::Backline        => '\n',
                   ReservedChar::SuperiorThan    => '>',
                   ReservedChar::Quote           => '\'',
                   ReservedChar::DoubleQuote     => '"',
                   ReservedChar::Pipe            => '|',
                   ReservedChar::Tilde           => '~',
                   ReservedChar::Dollar          => '$',
                   ReservedChar::Circumflex      => '^',
               })
    }
}

impl MyTryFrom<char> for ReservedChar {
    type Error = &'static str;

    fn try_from(value: char) -> Result<ReservedChar, Self::Error> {
        match value {
            '\'' => Ok(ReservedChar::Quote),
            '"'  => Ok(ReservedChar::DoubleQuote),
            ','  => Ok(ReservedChar::Comma),
            '('  => Ok(ReservedChar::OpenParenthese),
            ')'  => Ok(ReservedChar::CloseParenthese),
            '{'  => Ok(ReservedChar::OpenCurlyBrace),
            '}'  => Ok(ReservedChar::CloseCurlyBrace),
            '['  => Ok(ReservedChar::OpenBracket),
            ']'  => Ok(ReservedChar::CloseBracket),
            ':'  => Ok(ReservedChar::Colon),
            ';'  => Ok(ReservedChar::SemiColon),
            '/'  => Ok(ReservedChar::Slash),
            '*'  => Ok(ReservedChar::Star),
            '+'  => Ok(ReservedChar::Plus),
            '='  => Ok(ReservedChar::EqualSign),
            ' '  => Ok(ReservedChar::Space),
            '\t' => Ok(ReservedChar::Tab),
            '\n' |
            '\r' => Ok(ReservedChar::Backline),
            '>'  => Ok(ReservedChar::SuperiorThan),
            '|'  => Ok(ReservedChar::Pipe),
            '~'  => Ok(ReservedChar::Tilde),
            '$'  => Ok(ReservedChar::Dollar),
            '^'  => Ok(ReservedChar::Circumflex),
            _    => Err("Unknown reserved char"),
        }
    }
}

impl ReservedChar {
    fn is_useless(&self) -> bool {
        *self == ReservedChar::Space ||
        *self == ReservedChar::Tab ||
        *self == ReservedChar::Backline
    }
}

#[derive(Eq, PartialEq, Clone, Debug)]
pub enum SelectorElement<'a> {
    PseudoClass(&'a str),
    Class(&'a str),
    Id(&'a str),
    Tag(&'a str),
    Media(&'a str),
}

impl<'a> MyTryFrom<&'a str> for SelectorElement<'a> {
    type Error = &'static str;

    fn try_from(value: &'a str) -> Result<SelectorElement, Self::Error> {
        if value.starts_with('.') {
            if value.len() > 1 {
                Ok(SelectorElement::Class(&value[1..]))
            } else {
                Err("cannot determine selector")
            }
        } else if value.starts_with('#') {
            if value.len() > 1 {
                Ok(SelectorElement::Id(&value[1..]))
            } else {
                Err("cannot determine selector")
            }
        } else if value.starts_with('@') {
            if value.len() > 1 {
                Ok(SelectorElement::Media(&value[1..]))
            } else {
                Err("cannot determine selector")
            }
        } else if value.starts_with(':') {
            if value.len() > 1 {
                Ok(SelectorElement::PseudoClass(&value[1..]))
            } else {
                Err("cannot determine selector")
            }
        } else if value.chars().next().unwrap_or(' ').is_alphabetic() {
            Ok(SelectorElement::Tag(value))
        } else {
            Err("unknown selector")
        }
    }
}

impl<'a> fmt::Display for SelectorElement<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            SelectorElement::Class(c) => write!(f, ".{}", c),
            SelectorElement::Id(i) => write!(f, "#{}", i),
            SelectorElement::Tag(t) => write!(f, "{}", t),
            SelectorElement::Media(m) => write!(f, "@{} ", m),
            SelectorElement::PseudoClass(pc) => write!(f, ":{}", pc),
        }
    }
}

#[derive(Eq, PartialEq, Clone, Debug, Copy)]
pub enum SelectorOperator {
    /// `~=`
    OneAttributeEquals,
    /// `|=`
    EqualsOrStartsWithFollowedByDash,
    /// `$=`
    EndsWith,
    /// `^=`
    FirstStartsWith,
    /// `*=`
    Contains,
}

impl fmt::Display for SelectorOperator {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            SelectorOperator::OneAttributeEquals => write!(f, "{}", "~="),
            SelectorOperator::EqualsOrStartsWithFollowedByDash => write!(f, "{}", "|="),
            SelectorOperator::EndsWith => write!(f, "{}", "$="),
            SelectorOperator::FirstStartsWith => write!(f, "{}", "^="),
            SelectorOperator::Contains => write!(f, "{}", "*="),
        }
    }
}

#[derive(Eq, PartialEq, Clone, Debug)]
pub enum Token<'a> {
    /// Comment.
    Comment(&'a str),
    /// Comment starting with `/**`.
    License(&'a str),
    Char(ReservedChar),
    Other(&'a str),
    SelectorElement(SelectorElement<'a>),
    String(&'a str),
    SelectorOperator(SelectorOperator),
}

impl<'a> fmt::Display for Token<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        match *self {
            // Token::AtRule(at_rule) => write!(f, "{}", at_rule, content),
            // Token::ElementRule(selectors) => write!(f, "{}", x),
            Token::Comment(c) => write!(f, "{}", c),
            Token::License(l) => writeln!(f, "/*!{}*/", l),
            Token::Char(c) => write!(f, "{}", c),
            Token::Other(s) => write!(f, "{}", s),
            Token::SelectorElement(ref se) => write!(f, "{}", se),
            Token::String(s) => write!(f, "{}", s),
            Token::SelectorOperator(so) => write!(f, "{}", so),
        }
    }
}

impl<'a> Token<'a> {
    fn is_comment(&self) -> bool {
        match *self {
            Token::Comment(_) => true,
            _ => false,
        }
    }

    fn is_char(&self) -> bool {
        match *self {
            Token::Char(_) => true,
            _ => false,
        }
    }

    fn get_char(&self) -> Option<ReservedChar> {
        match *self {
            Token::Char(c) => Some(c),
            _ => None,
        }
    }

    fn is_useless(&self) -> bool {
        match *self {
            Token::Char(c) => c.is_useless(),
            _ => false,
        }
    }

    fn is_media(&self, media: &str) -> bool {
        match *self {
            Token::SelectorElement(SelectorElement::Media(s)) => s == media,
            _ => false,
        }
    }

    fn is_a_media(&self) -> bool {
        match *self {
            Token::SelectorElement(SelectorElement::Media(_)) => true,
            _ => false,
        }
    }

    fn is_a_license(&self) -> bool {
        match *self {
            Token::License(_) => true,
            _ => false,
        }
    }
}

impl<'a> PartialEq<ReservedChar> for Token<'a> {
    fn eq(&self, other: &ReservedChar) -> bool {
        match *self {
            Token::Char(c) => c == *other,
            _ => false,
        }
    }
}

fn get_line_comment<'a>(source: &'a str, iterator: &mut Peekable<CharIndices>,
                        start_pos: &mut usize) -> Option<Token<'a>> {
    *start_pos += 1;
    while let Some((pos, c)) = iterator.next() {
        if let Ok(c) = ReservedChar::try_from(c) {
            if c == ReservedChar::Backline {
                let ret = Some(Token::Comment(&source[*start_pos..pos]));
                *start_pos = pos;
                return ret;
            }
        }
    }
    None
}

fn get_comment<'a>(source: &'a str, iterator: &mut Peekable<CharIndices>,
                   start_pos: &mut usize) -> Option<Token<'a>> {
    let mut prev = ReservedChar::Quote;
    *start_pos += 1;
    let builder = if let Some((_, c)) = iterator.next() {
        if c == '!' || c == '*' {
            *start_pos += 1;
            Token::License
        } else {
            if let Ok(c) = ReservedChar::try_from(c) {
                prev = c;
            }
            Token::Comment
        }
    } else {
        Token::Comment
    };

    while let Some((pos, c)) = iterator.next() {
        if let Ok(c) = ReservedChar::try_from(c) {
            if c == ReservedChar::Slash && prev == ReservedChar::Star {
                let ret = Some(builder(&source[*start_pos..pos - 1]));
                *start_pos = pos;
                return ret;
            }
            prev = c;
        } else {
            prev = ReservedChar::Space;
        }
    }
    None
}

fn get_string<'a>(source: &'a str, iterator: &mut Peekable<CharIndices>, start_pos: &mut usize,
                  start: ReservedChar) -> Option<Token<'a>> {
    while let Some((pos, c)) = iterator.next() {
        if c == '\\' {
            // we skip next character
            iterator.next();
            continue
        }
        if let Ok(c) = ReservedChar::try_from(c) {
            if c == start {
                let ret = Some(Token::String(&source[*start_pos..pos + 1]));
                *start_pos = pos;
                return ret;
            }
        }
    }
    None
}

fn fill_other<'a>(source: &'a str, v: &mut Vec<Token<'a>>, start: usize, pos: usize,
                  is_in_block: isize, is_in_media: bool, is_in_attribute_selector: bool) {
    if start < pos {
        if is_in_attribute_selector == false &&
                  ((is_in_block == 0 && is_in_media == false) ||
                   (is_in_media == true && is_in_block == 1)) {
            let mut is_pseudo_class = false;
            let mut add = 0;
            if let Some(&Token::Char(ReservedChar::Colon)) = v.last() {
                is_pseudo_class = true;
                add = 1;
            }
            if let Ok(s) = SelectorElement::try_from(&source[start - add..pos]) {
                if is_pseudo_class {
                    v.pop();
                }
                v.push(Token::SelectorElement(s));
            } else {
                let s = &source[start..pos];
                if !s.starts_with(':') && !s.starts_with('.') && !s.starts_with('#') &&
                   !s.starts_with('@') {
                   v.push(Token::Other(s));
               }
            }
        } else {
            v.push(Token::Other(&source[start..pos]));
        }
    }
}

pub fn tokenize<'a>(source: &'a str) -> Result<Tokens<'a>, &'static str> {
    let mut v = Vec::with_capacity(1000);
    let mut iterator = source.char_indices().peekable();
    let mut start = 0;
    let mut is_in_block: isize = 0;
    let mut is_in_media = false;
    let mut is_in_attribute_selector = false;

    loop {
        let (mut pos, c) = match iterator.next() {
            Some(x) => x,
            None => {
                fill_other(source, &mut v, start, source.len(), is_in_block, is_in_media,
                           is_in_attribute_selector);
                break
            }
        };
        if let Ok(c) = ReservedChar::try_from(c) {
            fill_other(source, &mut v, start, pos, is_in_block, is_in_media,
                       is_in_attribute_selector);
            is_in_media = is_in_media || v.last()
                                          .unwrap_or(&Token::Char(ReservedChar::Space))
                                          .is_media("media");
            if c == ReservedChar::Quote || c == ReservedChar::DoubleQuote {
                if let Some(s) = get_string(source, &mut iterator, &mut pos, c) {
                    v.push(s);
                }
            } else if c == ReservedChar::Star &&
                      *v.last().unwrap_or(&Token::Char(ReservedChar::Space)) == ReservedChar::Slash {
                v.pop();
                if let Some(s) = get_comment(source, &mut iterator, &mut pos) {
                    v.push(s);
                }
            } else if c == ReservedChar::Slash &&
                      *v.last().unwrap_or(&Token::Char(ReservedChar::Space)) == ReservedChar::Slash {
                v.pop();
                if let Some(s) = get_line_comment(source, &mut iterator, &mut pos) {
                    v.push(s);
                }
            } else if c == ReservedChar::OpenBracket {
                if is_in_attribute_selector {
                    return Err("Already in attribute selector");
                }
                is_in_attribute_selector = true;
                v.push(Token::Char(c));
            } else if c == ReservedChar::CloseBracket {
                if !is_in_attribute_selector {
                    return Err("Unexpected ']'");
                }
                is_in_attribute_selector = false;
                v.push(Token::Char(c));
            } else if c == ReservedChar::OpenCurlyBrace {
                is_in_block += 1;
                v.push(Token::Char(c));
            } else if c == ReservedChar::CloseCurlyBrace {
                is_in_block -= 1;
                if is_in_block < 0 {
                    return Err("Too much '}'");
                } else if is_in_block == 0 {
                    is_in_media = false;
                }
                v.push(Token::Char(c));
            } else if c == ReservedChar::EqualSign {
                match match v.last()
                             .unwrap_or(&Token::Char(ReservedChar::Space))
                             .get_char()
                             .unwrap_or(ReservedChar::Space) {
                    ReservedChar::Tilde => Some(SelectorOperator::OneAttributeEquals),
                    ReservedChar::Pipe => Some(SelectorOperator::EqualsOrStartsWithFollowedByDash),
                    ReservedChar::Dollar => Some(SelectorOperator::EndsWith),
                    ReservedChar::Circumflex => Some(SelectorOperator::FirstStartsWith),
                    ReservedChar::Star => Some(SelectorOperator::Contains),
                    _ => None,
                } {
                    Some(r) => {
                        v.pop();
                        v.push(Token::SelectorOperator(r));
                    }
                    None => v.push(Token::Char(c)),
                }
            } else if !c.is_useless() {
                v.push(Token::Char(c));
            } else if !v.last().unwrap_or(&Token::Char(ReservedChar::Space)).is_useless() &&
                      !v.last().unwrap_or(&Token::Char(ReservedChar::OpenCurlyBrace)).is_char() {
                v.push(Token::Char(ReservedChar::Space));
            }
            start = pos + 1;
        }
    }
    Ok(Tokens(clean_tokens(v)))
}

fn clean_tokens<'a>(mut v: Vec<Token<'a>>) -> Vec<Token<'a>> {
    let mut i = 0;

    while i < v.len() {
        if v[i].is_useless() {
            if (i > 0 && (v[i - 1].is_char() ||
                          v[i - 1].is_a_media() ||
                          v[i - 1].is_a_license())) ||
               (i < v.len() - 1 && v[i + 1].is_char()) {
                v.remove(i);
                continue
            }
        } else if v[i].is_comment() {
            v.remove(i);
            continue
        }
        i += 1;
    }
    v
}

#[derive(Debug, PartialEq, Eq, Clone)]
pub struct Tokens<'a>(pub Vec<Token<'a>>);

impl<'a> fmt::Display for Tokens<'a> {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        for token in self.0.iter() {
            write!(f, "{}", token)?;
        }
        Ok(())
    }
}

#[test]
fn css_basic() {
    let s = r#"
/*! just some license */
.foo > #bar p:hover {
    color: blue;
    background: "blue";
}

// a comment!
@media screen and (max-width: 640px) {
    .block:hover {
        display:    block;
    }
}"#;
    let expected = vec![Token::License(" just some license "),
                        Token::SelectorElement(SelectorElement::Class("foo")),
                        Token::Char(ReservedChar::SuperiorThan),
                        Token::SelectorElement(SelectorElement::Id("bar")),
                        Token::Char(ReservedChar::Space),
                        Token::SelectorElement(SelectorElement::Tag("p")),
                        Token::SelectorElement(SelectorElement::PseudoClass("hover")),
                        Token::Char(ReservedChar::OpenCurlyBrace),
                        Token::Other("color"),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("blue"),
                        Token::Char(ReservedChar::SemiColon),
                        Token::Other("background"),
                        Token::Char(ReservedChar::Colon),
                        Token::String("\"blue\""),
                        Token::Char(ReservedChar::SemiColon),
                        Token::Char(ReservedChar::CloseCurlyBrace),
                        Token::SelectorElement(SelectorElement::Media("media")),
                        Token::Other("screen"),
                        Token::Char(ReservedChar::Space),
                        Token::Other("and"),
                        Token::Char(ReservedChar::OpenParenthese),
                        Token::Other("max-width"),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("640px"),
                        Token::Char(ReservedChar::CloseParenthese),
                        Token::Char(ReservedChar::OpenCurlyBrace),
                        Token::SelectorElement(SelectorElement::Class("block")),
                        Token::SelectorElement(SelectorElement::PseudoClass("hover")),
                        Token::Char(ReservedChar::OpenCurlyBrace),
                        Token::Other("display"),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("block"),
                        Token::Char(ReservedChar::SemiColon),
                        Token::Char(ReservedChar::CloseCurlyBrace),
                        Token::Char(ReservedChar::CloseCurlyBrace)];
    assert_eq!(tokenize(s), Ok(Tokens(expected)));
}

#[test]
fn elem_selector() {
    let s = r#"
/** just some license */
a[href*="example"] {
    background: yellow;
}
a[href$=".org"] {
  font-style: italic;
}
span[lang|="zh"] {
    color: red;
}
a[href^="/"] {
    background-color: gold;
}
div[value~="test"] {
    border-width: 1px;
}
span[lang="pt"] {
    font-size: 12em; // I love big fonts
}
"#;
    let expected = vec![Token::License(" just some license "),
                        Token::SelectorElement(SelectorElement::Tag("a")),
                        Token::Char(ReservedChar::OpenBracket),
                        Token::Other("href"),
                        Token::SelectorOperator(SelectorOperator::Contains),
                        Token::String("\"example\""),
                        Token::Char(ReservedChar::CloseBracket),
                        Token::Char(ReservedChar::OpenCurlyBrace),
                        Token::Other("background"),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("yellow"),
                        Token::Char(ReservedChar::SemiColon),
                        Token::Char(ReservedChar::CloseCurlyBrace),

                        Token::SelectorElement(SelectorElement::Tag("a")),
                        Token::Char(ReservedChar::OpenBracket),
                        Token::Other("href"),
                        Token::SelectorOperator(SelectorOperator::EndsWith),
                        Token::String("\".org\""),
                        Token::Char(ReservedChar::CloseBracket),
                        Token::Char(ReservedChar::OpenCurlyBrace),
                        Token::Other("font-style"),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("italic"),
                        Token::Char(ReservedChar::SemiColon),
                        Token::Char(ReservedChar::CloseCurlyBrace),

                        Token::SelectorElement(SelectorElement::Tag("span")),
                        Token::Char(ReservedChar::OpenBracket),
                        Token::Other("lang"),
                        Token::SelectorOperator(SelectorOperator::EqualsOrStartsWithFollowedByDash),
                        Token::String("\"zh\""),
                        Token::Char(ReservedChar::CloseBracket),
                        Token::Char(ReservedChar::OpenCurlyBrace),
                        Token::Other("color"),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("red"),
                        Token::Char(ReservedChar::SemiColon),
                        Token::Char(ReservedChar::CloseCurlyBrace),

                        Token::SelectorElement(SelectorElement::Tag("a")),
                        Token::Char(ReservedChar::OpenBracket),
                        Token::Other("href"),
                        Token::SelectorOperator(SelectorOperator::FirstStartsWith),
                        Token::String("\"/\""),
                        Token::Char(ReservedChar::CloseBracket),
                        Token::Char(ReservedChar::OpenCurlyBrace),
                        Token::Other("background-color"),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("gold"),
                        Token::Char(ReservedChar::SemiColon),
                        Token::Char(ReservedChar::CloseCurlyBrace),

                        Token::SelectorElement(SelectorElement::Tag("div")),
                        Token::Char(ReservedChar::OpenBracket),
                        Token::Other("value"),
                        Token::SelectorOperator(SelectorOperator::OneAttributeEquals),
                        Token::String("\"test\""),
                        Token::Char(ReservedChar::CloseBracket),
                        Token::Char(ReservedChar::OpenCurlyBrace),
                        Token::Other("border-width"),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("1px"),
                        Token::Char(ReservedChar::SemiColon),
                        Token::Char(ReservedChar::CloseCurlyBrace),

                        Token::SelectorElement(SelectorElement::Tag("span")),
                        Token::Char(ReservedChar::OpenBracket),
                        Token::Other("lang"),
                        Token::Char(ReservedChar::EqualSign),
                        Token::String("\"pt\""),
                        Token::Char(ReservedChar::CloseBracket),
                        Token::Char(ReservedChar::OpenCurlyBrace),
                        Token::Other("font-size"),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("12em"),
                        Token::Char(ReservedChar::SemiColon),
                        Token::Char(ReservedChar::CloseCurlyBrace)];
    assert_eq!(tokenize(s), Ok(Tokens(expected)));
}

#[test]
fn check_media() {
    let s = "@media (max-width: 700px) { color: red; }";

    let expected = vec![Token::SelectorElement(SelectorElement::Media("media")),
                        Token::Char(ReservedChar::OpenParenthese),
                        Token::Other("max-width"),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("700px"),
                        Token::Char(ReservedChar::CloseParenthese),
                        Token::Char(ReservedChar::OpenCurlyBrace),
                        Token::SelectorElement(SelectorElement::Tag("color")),
                        Token::Char(ReservedChar::Colon),
                        Token::Other("red"),
                        Token::Char(ReservedChar::SemiColon),
                        Token::Char(ReservedChar::CloseCurlyBrace)];
    assert_eq!(tokenize(s), Ok(Tokens(expected)));
}