vendor/toml-query/src/tokenizer.rs

   1 /// The tokenizer for the query interpreter
   2
   3 use crate::error::{Error, Result};
   4
   5 #[derive(Debug, PartialEq, Eq)]
   6 pub enum Token {
   7     Identifier {
   8         ident: String,
   9         next: Option<Box<Token>>
  10     },
  11
  12     Index {
  13         idx: usize,
  14         next: Option<Box<Token>>
  15     }
  16 }
  17
  18 impl Token {
  19
  20     pub fn next(&self) -> Option<&Box<Token>> {
  21         trace!("Matching token (self): {:?}", self);
  22         match self {
  23             &Token::Identifier { ref next, .. } => next.as_ref(),
  24             &Token::Index { ref next, .. }      => next.as_ref(),
  25         }
  26     }
  27
  28     /// Convenience function for `token.next().is_some()`
  29     pub fn has_next(&self) -> bool {
  30         trace!("self.has_next(): {:?}", self.next().is_some());
  31         self.next().is_some()
  32     }
  33
  34     pub fn set_next(&mut self, token: Token) {
  35         trace!("self.set_next({:?})", token);
  36         match self {
  37             &mut Token::Identifier { ref mut next, .. } => *next = Some(Box::new(token)),
  38             &mut Token::Index { ref mut next, .. }      => *next = Some(Box::new(token)),
  39         }
  40     }
  41
  42     /// Pop the last token from the chain of tokens
  43     ///
  44     /// Returns None if the current Token has no next token
  45     pub fn pop_last(&mut self) -> Option<Box<Token>> {
  46         trace!("self.pop_last()");
  47         if !self.has_next() {
  48             trace!("self.pop_last(): No next");
  49             None
  50         } else {
  51             trace!("self.pop_last(): Having next");
  52             match self {
  53                 &mut Token::Identifier { ref mut next, .. } => {
  54                     trace!("self.pop_last(): self is Identifier");
  55                     if next.is_some() {
  56                         trace!("self.pop_last(): next is Some(_)");
  57                         let mut n = next.take().unwrap();
  58                         if n.has_next() {
  59                             trace!("self.pop_last(): next also has a next");
  60
  61                             trace!("self.pop_last(): Recursing now");
  62                             let result = n.pop_last();
  63
  64                             *next = Some(n);
  65
  66                             trace!("self.pop_last(): Returning Result");
  67                             return result;
  68                         } else {
  69                             trace!("self.pop_last(): next itself has no next, returning Some");
  70                             Some(n)
  71                         }
  72                     } else {
  73                         trace!("self.pop_last(): next is none, returning None");
  74                         None
  75                     }
  76                 },
  77
  78                 &mut Token::Index { ref mut next, .. } => {
  79                     trace!("self.pop_last(): self is Index");
  80                     if next.is_some() {
  81                         trace!("self.pop_last(): next is Some(_)");
  82
  83                         let mut n = next.take().unwrap();
  84                         if n.has_next() {
  85                             trace!("self.pop_last(): next also has a next");
  86
  87                             trace!("self.pop_last(): Recursing now");
  88                             let result = n.pop_last();
  89
  90                             *next = Some(n);
  91
  92                             trace!("self.pop_last(): Returning Result");
  93                             return result;
  94                         } else {
  95                             trace!("self.pop_last(): next itself has no next, returning Some");
  96                             Some(n)
  97                         }
  98                     } else {
  99                         trace!("self.pop_last(): next is none, returning None");
 100                         None
 101                     }
 102                 },
 103             }
 104         }
 105     }
 106
 107     #[cfg(test)]
 108     pub fn identifier(&self) -> &String {
 109         trace!("self.identifier()");
 110         match self {
 111             &Token::Identifier { ref ident, .. } => &ident,
 112             _ => unreachable!(),
 113         }
 114     }
 115
 116     #[cfg(test)]
 117     pub fn idx(&self) -> usize {
 118         trace!("self.idx()");
 119         match self {
 120             &Token::Index { idx: i, .. } => i,
 121             _ => unreachable!(),
 122         }
 123     }
 124
 125 }
 126
 127 pub fn tokenize_with_seperator(query: &str, seperator: char) -> Result<Token> {
 128     use std::str::Split;
 129     trace!("tokenize_with_seperator(query: {:?}, seperator: {:?})", query, seperator);
 130
 131     /// Creates a Token object from a string
 132     ///
 133     /// # Panics
 134     ///
 135     /// * If the internal regex does not compile (should never happen)
 136     /// * If the token is non-valid (that is, a array index with a non-i64)
 137     /// * If the regex does not find anything
 138     /// * If the integer in the brackets (`[]`) cannot be parsed to a valid i64
 139     ///
 140     /// # Incorrect behaviour
 141     ///
 142     /// * If the regex finds multiple captures
 143     ///
 144     /// # Returns
 145     ///
 146     /// The `Token` object with the correct identifier/index for this token and no next token.
 147     ///
 148     fn mk_token_object(s: &str) -> Result<Token> {
 149         use regex::Regex;
 150         use std::str::FromStr;
 151
 152         trace!("mk_token_object(s: {:?})", s);
 153
 154         lazy_static! {
 155             static ref RE: Regex = Regex::new(r"^\[\d+\]$").unwrap();
 156         }
 157
 158         if !has_array_brackets(s) {
 159             trace!("returning Ok(Identifier(ident: {:?}, next: None))", s);
 160             return Ok(Token::Identifier { ident: String::from(s), next: None });
 161         }
 162
 163         match RE.captures(s) {
 164             None => return Err(Error::ArrayAccessWithoutIndex),
 165             Some(captures) => {
 166                 trace!("Captured: {:?}", captures);
 167                 match captures.get(0) {
 168                     None => Ok(Token::Identifier { ident: String::from(s), next: None }),
 169                     Some(mtch) => {
 170                         trace!("First capture: {:?}", mtch);
 171
 172                         let mtch = without_array_brackets(mtch.as_str());
 173                         trace!(".. without array brackets: {:?}", mtch);
 174
 175                         let i : usize = FromStr::from_str(&mtch).unwrap(); // save because regex
 176
 177                         trace!("returning Ok(Index(idx: {}, next: None)", i);
 178                         Ok(Token::Index {
 179                             idx: i,
 180                             next: None,
 181                         })
 182                     }
 183                 }
 184             }
 185         }
 186     }
 187
 188     /// Check whether a str begins with '[' and ends with ']'
 189     fn has_array_brackets(s: &str) -> bool {
 190         trace!("has_array_brackets({:?})", s);
 191         s.as_bytes()[0] == b'[' && s.as_bytes()[s.len() - 1] == b']'
 192     }
 193
 194     /// Remove '[' and ']' from a str
 195     fn without_array_brackets(s: &str) -> String {
 196         trace!("without_array_brackets({:?})", s);
 197         s.replace("[","").replace("]","")
 198     }
 199
 200     fn build_token_tree(split: &mut Split<char>, last: &mut Token) -> Result<()> {
 201         trace!("build_token_tree(split: {:?}, last: {:?})", split, last);
 202         match split.next() {
 203             None        => { /* No more tokens */ }
 204             Some(token) => {
 205                 trace!("build_token_tree(...): next from split: {:?}", token);
 206
 207                 if token.len() == 0 {
 208                     trace!("build_token_tree(...): Empty identifier... returning Error");
 209                     return Err(Error::EmptyIdentifier)
 210                 }
 211
 212                 let mut token = r#try!(mk_token_object(token));
 213                 r#try!(build_token_tree(split, &mut token));
 214                 last.set_next(token);
 215             }
 216         }
 217
 218         trace!("build_token_tree(...): returning Ok(())");
 219         Ok(())
 220     }
 221
 222     if query.is_empty() {
 223         trace!("Query is empty. Returning error");
 224         return Err(Error::EmptyQueryError)
 225     }
 226
 227     let mut tokens = query.split(seperator);
 228     trace!("Tokens splitted: {:?}", tokens);
 229
 230     match tokens.next() {
 231         None        => Err(Error::EmptyQueryError),
 232         Some(token) => {
 233             trace!("next Token: {:?}", token);
 234
 235             if token.len() == 0 {
 236                 trace!("Empty token. Returning Error");
 237                 return Err(Error::EmptyIdentifier);
 238             }
 239
 240             let mut tok = r#try!(mk_token_object(token));
 241             let _       = r#try!(build_token_tree(&mut tokens, &mut tok));
 242
 243             trace!("Returning Ok({:?})", tok);
 244             Ok(tok)
 245         }
 246     }
 247 }
 248
 249 #[cfg(test)]
 250 mod test {
 251     use crate::error::Error;
 252     use super::*;
 253
 254     use std::ops::Deref;
 255
 256     #[test]
 257     fn test_tokenize_empty_query_to_error() {
 258         let tokens = tokenize_with_seperator(&String::from(""), '.');
 259         assert!(tokens.is_err());
 260         let tokens = tokens.unwrap_err();
 261
 262         assert!(is_match!(tokens, Error::EmptyQueryError { .. }));
 263     }
 264
 265     #[test]
 266     fn test_tokenize_seperator_only() {
 267         let tokens = tokenize_with_seperator(&String::from("."), '.');
 268         assert!(tokens.is_err());
 269         let tokens = tokens.unwrap_err();
 270
 271         assert!(is_match!(tokens, Error::EmptyIdentifier { .. }));
 272     }
 273
 274     #[test]
 275     fn test_tokenize_array_brackets_only() {
 276         let tokens = tokenize_with_seperator(&String::from("[]"), '.');
 277         assert!(tokens.is_err());
 278         let tokens = tokens.unwrap_err();
 279
 280         assert!(is_match!(tokens, Error::ArrayAccessWithoutIndex { .. }));
 281     }
 282
 283     #[test]
 284     fn test_tokenize_identifiers_with_array_brackets_only() {
 285         let tokens = tokenize_with_seperator(&String::from("a.b.c.[]"), '.');
 286         assert!(tokens.is_err());
 287         let tokens = tokens.unwrap_err();
 288
 289         assert!(is_match!(tokens, Error::ArrayAccessWithoutIndex { .. }));
 290     }
 291
 292     #[test]
 293     fn test_tokenize_identifiers_in_array_brackets() {
 294         let tokens = tokenize_with_seperator(&String::from("[a]"), '.');
 295         assert!(tokens.is_err());
 296         let tokens = tokens.unwrap_err();
 297
 298         assert!(is_match!(tokens, Error::ArrayAccessWithoutIndex { .. }));
 299     }
 300
 301     #[test]
 302     fn test_tokenize_single_token_query() {
 303         let tokens = tokenize_with_seperator(&String::from("example"), '.');
 304         assert!(tokens.is_ok());
 305         let tokens = tokens.unwrap();
 306
 307         assert!(match tokens {
 308             Token::Identifier { ref ident, next: None } => {
 309                 assert_eq!("example", ident);
 310                 true
 311             },
 312             _ => false,
 313         });
 314     }
 315
 316     #[test]
 317     fn test_tokenize_double_token_query() {
 318         let tokens = tokenize_with_seperator(&String::from("a.b"), '.');
 319         assert!(tokens.is_ok());
 320         let tokens = tokens.unwrap();
 321
 322         assert!(match tokens {
 323             Token::Identifier { next: Some(ref next), .. } => {
 324                 assert_eq!("b", next.deref().identifier());
 325                 match next.deref() {
 326                     &Token::Identifier { next: None, .. } => true,
 327                     _ => false
 328                 }
 329             },
 330             _ => false,
 331         });
 332         assert_eq!("a", tokens.identifier());
 333     }
 334
 335     #[test]
 336     fn test_tokenize_ident_then_array_query() {
 337         let tokens = tokenize_with_seperator(&String::from("a.[0]"), '.');
 338         assert!(tokens.is_ok());
 339         let tokens = tokens.unwrap();
 340
 341         assert_eq!("a", tokens.identifier());
 342         assert!(match tokens {
 343             Token::Identifier { next: Some(ref next), .. } => {
 344                 match next.deref() {
 345                     &Token::Index { idx: 0, next: None } => true,
 346                     _ => false
 347                 }
 348             },
 349             _ => false,
 350         });
 351     }
 352
 353     #[test]
 354     fn test_tokenize_many_idents_then_array_query() {
 355         let tokens = tokenize_with_seperator(&String::from("a.b.c.[1000]"), '.');
 356         assert!(tokens.is_ok());
 357         let tokens = tokens.unwrap();
 358
 359         assert_eq!("a", tokens.identifier());
 360
 361         let expected =
 362             Token::Identifier {
 363                 ident: String::from("a"),
 364                 next: Some(Box::new(Token::Identifier {
 365                     ident: String::from("b"),
 366                     next: Some(Box::new(Token::Identifier {
 367                         ident: String::from("c"),
 368                         next: Some(Box::new(Token::Index {
 369                             idx: 1000,
 370                             next: None,
 371                         })),
 372                     })),
 373                 })),
 374             };
 375
 376         assert_eq!(expected, tokens);
 377     }
 378
 379     #[test]
 380     fn test_tokenize_empty_token_after_good_token() {
 381         let tokens = tokenize_with_seperator(&String::from("a..b"), '.');
 382         assert!(tokens.is_err());
 383         let tokens = tokens.unwrap_err();
 384
 385         assert!(is_match!(tokens, Error::EmptyIdentifier { .. }));
 386     }
 387
 388     quickcheck! {
 389         fn test_array_index(i: usize) -> bool {
 390             match tokenize_with_seperator(&format!("[{}]", i), '.') {
 391                 Ok(Token::Index { next: None, ..  }) => true,
 392                 _                                    => false,
 393             }
 394         }
 395     }
 396
 397     #[test]
 398     fn test_pop_last_token_from_single_identifier_token_is_none() {
 399         let mut token = Token::Identifier {
 400             ident: String::from("something"),
 401             next: None,
 402         };
 403
 404         let last = token.pop_last();
 405         assert!(last.is_none());
 406     }
 407
 408     #[test]
 409     fn test_pop_last_token_from_single_index_token_is_none() {
 410         let mut token = Token::Index {
 411             idx: 0,
 412             next: None,
 413         };
 414
 415         let last = token.pop_last();
 416         assert!(last.is_none());
 417     }
 418
 419     #[test]
 420     fn test_pop_last_token_from_single_identifier_token_is_one() {
 421         let mut token = Token::Identifier {
 422             ident: String::from("some"),
 423             next: Some(Box::new(Token::Identifier {
 424                 ident: String::from("thing"),
 425                 next: None,
 426             })),
 427         };
 428
 429         let last = token.pop_last();
 430
 431         assert!(last.is_some());
 432         let last = last.unwrap();
 433
 434         assert!(is_match!(*last, Token::Identifier { .. }));
 435         match *last {
 436             Token::Identifier { ident, .. } => {
 437                 assert_eq!("thing", ident);
 438             }
 439             _ => panic!("What just happened?"),
 440         }
 441     }
 442
 443     #[test]
 444     fn test_pop_last_token_from_single_index_token_is_one() {
 445         let mut token = Token::Index {
 446             idx: 0,
 447             next: Some(Box::new(Token::Index {
 448                 idx: 1,
 449                 next: None,
 450             })),
 451         };
 452
 453         let last = token.pop_last();
 454
 455         assert!(last.is_some());
 456         let last = last.unwrap();
 457
 458         assert!(is_match!(*last, Token::Index { idx: 1, .. }));
 459     }
 460
 461     #[test]
 462     fn test_pop_last_token_from_identifier_chain() {
 463         let tokens = tokenize_with_seperator(&String::from("a.b.c.d.e.f"), '.');
 464         assert!(tokens.is_ok());
 465         let mut tokens = tokens.unwrap();
 466
 467         let last = tokens.pop_last();
 468         assert!(last.is_some());
 469         assert_eq!("f", last.unwrap().identifier());
 470     }
 471
 472     #[test]
 473     fn test_pop_last_token_from_mixed_chain() {
 474         let tokens = tokenize_with_seperator(&String::from("a.[100].c.[3].e.f"), '.');
 475         assert!(tokens.is_ok());
 476         let mut tokens = tokens.unwrap();
 477
 478         let last = tokens.pop_last();
 479         assert!(last.is_some());
 480         assert_eq!("f", last.unwrap().identifier());
 481     }
 482
 483     #[test]
 484     fn test_pop_last_token_from_identifier_chain_is_array() {
 485         let tokens = tokenize_with_seperator(&String::from("a.b.c.d.e.f.[1000]"), '.');
 486         assert!(tokens.is_ok());
 487         let mut tokens = tokens.unwrap();
 488
 489         let last = tokens.pop_last();
 490         assert!(last.is_some());
 491         assert_eq!(1000, last.unwrap().idx());
 492     }
 493
 494     #[test]
 495     fn test_pop_last_token_from_mixed_chain_is_array() {
 496         let tokens = tokenize_with_seperator(&String::from("a.[100].c.[3].e.f.[1000]"), '.');
 497         assert!(tokens.is_ok());
 498         let mut tokens = tokens.unwrap();
 499
 500         let last = tokens.pop_last();
 501         assert!(last.is_some());
 502         assert_eq!(1000, last.unwrap().idx());
 503     }
 504
 505     #[test]
 506     fn test_pop_last_token_from_one_token() {
 507         let tokens = tokenize_with_seperator(&String::from("a"), '.');
 508         assert!(tokens.is_ok());
 509         let mut tokens = tokens.unwrap();
 510
 511         let last = tokens.pop_last();
 512         assert!(last.is_none());
 513     }
 514
 515     #[test]
 516     fn test_pop_last_chain() {
 517         let tokens = tokenize_with_seperator(&String::from("a.[100].c.[3].e.f.[1000]"), '.');
 518         assert!(tokens.is_ok());
 519         let mut tokens = tokens.unwrap();
 520
 521         let last = tokens.pop_last();
 522         assert!(last.is_some());
 523         assert_eq!(1000, last.unwrap().idx());
 524
 525         let last = tokens.pop_last();
 526         assert!(last.is_some());
 527         assert_eq!("f", last.unwrap().identifier());
 528
 529         let last = tokens.pop_last();
 530         assert!(last.is_some());
 531         assert_eq!("e", last.unwrap().identifier());
 532
 533         let last = tokens.pop_last();
 534         assert!(last.is_some());
 535         assert_eq!(3, last.unwrap().idx());
 536
 537         let last = tokens.pop_last();
 538         assert!(last.is_some());
 539         assert_eq!("c", last.unwrap().identifier());
 540
 541         let last = tokens.pop_last();
 542         assert!(last.is_some());
 543         assert_eq!(100, last.unwrap().idx());
 544
 545         let last = tokens.pop_last();
 546         assert!(last.is_none());
 547     }
 548
 549 }