src/vendor/toml-0.3.2/src/de.rs

   1 //! Deserializing TOML into Rust structures.
   2 //!
   3 //! This module contains all the Serde support for deserializing TOML documents
   4 //! into Rust structures. Note that some top-level functions here are also
   5 //! provided at the top of the crate.
   6
   7 use std::borrow::Cow;
   8 use std::error;
   9 use std::fmt;
  10 use std::str;
  11 use std::vec;
  12
  13 use serde::de;
  14
  15 use tokens::{Tokenizer, Token, Error as TokenError};
  16 use datetime::{SERDE_STRUCT_FIELD_NAME, SERDE_STRUCT_NAME};
  17
  18 /// Deserializes a byte slice into a type.
  19 ///
  20 /// This function will attempt to interpret `bytes` as UTF-8 data and then
  21 /// deserialize `T` from the TOML document provided.
  22 pub fn from_slice<T>(bytes: &[u8]) -> Result<T, Error>
  23     where T: de::Deserialize,
  24 {
  25     match str::from_utf8(bytes) {
  26         Ok(s) => from_str(s),
  27         Err(e) => Err(Error::custom(e.to_string())),
  28     }
  29 }
  30
  31 /// Deserializes a string into a type.
  32 ///
  33 /// This function will attempt to interpret `s` as a TOML document and
  34 /// deserialize `T` from the document.
  35 pub fn from_str<T>(s: &str) -> Result<T, Error>
  36     where T: de::Deserialize,
  37 {
  38     let mut d = Deserializer::new(s);
  39     let ret = T::deserialize(&mut d)?;
  40     d.end()?;
  41     return Ok(ret)
  42 }
  43
  44 /// Errors that can occur when deserializing a type.
  45 #[derive(Debug, Clone)]
  46 pub struct Error {
  47     inner: Box<ErrorInner>,
  48 }
  49
  50 #[derive(Debug, Clone)]
  51 struct ErrorInner {
  52     kind: ErrorKind,
  53     line: Option<usize>,
  54     col: usize,
  55     message: String,
  56     key: Vec<String>,
  57 }
  58
  59 /// Errors that can occur when deserializing a type.
  60 #[derive(Debug, Clone)]
  61 enum ErrorKind {
  62     /// EOF was reached when looking for a value
  63     UnexpectedEof,
  64
  65     /// An invalid character not allowed in a string was found
  66     InvalidCharInString(char),
  67
  68     /// An invalid character was found as an escape
  69     InvalidEscape(char),
  70
  71     /// An invalid character was found in a hex escape
  72     InvalidHexEscape(char),
  73
  74     /// An invalid escape value was specified in a hex escape in a string.
  75     ///
  76     /// Valid values are in the plane of unicode codepoints.
  77     InvalidEscapeValue(u32),
  78
  79     /// A newline in a string was encountered when one was not allowed.
  80     NewlineInString,
  81
  82     /// An unexpected character was encountered, typically when looking for a
  83     /// value.
  84     Unexpected(char),
  85
  86     /// An unterminated string was found where EOF was found before the ending
  87     /// EOF mark.
  88     UnterminatedString,
  89
  90     /// A newline was found in a table key.
  91     NewlineInTableKey,
  92
  93     /// A number failed to parse
  94     NumberInvalid,
  95
  96     /// A date or datetime was invalid
  97     DateInvalid,
  98
  99     /// Wanted one sort of token, but found another.
 100     Wanted {
 101         /// Expected token type
 102         expected: &'static str,
 103         /// Actually found token type
 104         found: &'static str,
 105     },
 106
 107     /// An array was decoded but the types inside of it were mixed, which is
 108     /// disallowed by TOML.
 109     MixedArrayType,
 110
 111     /// A duplicate table definition was found.
 112     DuplicateTable(String),
 113
 114     /// A previously defined table was redefined as an array.
 115     RedefineAsArray,
 116
 117     /// An empty table key was found.
 118     EmptyTableKey,
 119
 120     /// A custom error which could be generated when deserializing a particular
 121     /// type.
 122     Custom,
 123
 124     #[doc(hidden)]
 125     __Nonexhaustive,
 126 }
 127
 128 /// Deserialization implementation for TOML.
 129 pub struct Deserializer<'a> {
 130     require_newline_after_table: bool,
 131     input: &'a str,
 132     tokens: Tokenizer<'a>,
 133 }
 134
 135 impl<'a, 'b> de::Deserializer for &'b mut Deserializer<'a> {
 136     type Error = Error;
 137
 138     fn deserialize<V>(self, visitor: V) -> Result<V::Value, Error>
 139         where V: de::Visitor,
 140     {
 141         let mut tables = Vec::new();
 142         let mut cur_table = Table {
 143             at: 0,
 144             header: Vec::new(),
 145             values: None,
 146             array: false,
 147         };
 148         while let Some(line) = self.line()? {
 149             match line {
 150                 Line::Table { at, mut header, array } => {
 151                     if cur_table.header.len() > 0 || cur_table.values.is_some() {
 152                         tables.push(cur_table);
 153                     }
 154                     cur_table = Table {
 155                         at: at,
 156                         header: Vec::new(),
 157                         values: Some(Vec::new()),
 158                         array: array,
 159                     };
 160                     loop {
 161                         let part = header.next().map_err(|e| {
 162                             self.token_error(e)
 163                         });
 164                         match part? {
 165                             Some(part) => cur_table.header.push(part),
 166                             None => break,
 167                         }
 168                     }
 169                 }
 170                 Line::KeyValue(key, value) => {
 171                     if cur_table.values.is_none() {
 172                         cur_table.values = Some(Vec::new());
 173                     }
 174                     cur_table.values.as_mut().unwrap().push((key, value));
 175                 }
 176             }
 177         }
 178         if cur_table.header.len() > 0 || cur_table.values.is_some() {
 179             tables.push(cur_table);
 180         }
 181
 182         visitor.visit_map(MapVisitor {
 183             values: Vec::new().into_iter(),
 184             next_value: None,
 185             depth: 0,
 186             cur: 0,
 187             cur_parent: 0,
 188             max: tables.len(),
 189             tables: &mut tables,
 190             array: false,
 191             de: self,
 192         })
 193     }
 194
 195     forward_to_deserialize! {
 196         bool u8 u16 u32 u64 i8 i16 i32 i64 f32 f64 char str string seq
 197         seq_fixed_size bytes byte_buf map struct unit enum newtype_struct
 198         struct_field ignored_any unit_struct tuple_struct tuple option
 199     }
 200 }
 201
 202 struct Table<'a> {
 203     at: usize,
 204     header: Vec<Cow<'a, str>>,
 205     values: Option<Vec<(Cow<'a, str>, Value<'a>)>>,
 206     array: bool,
 207 }
 208
 209 #[doc(hidden)]
 210 pub struct MapVisitor<'a: 'b, 'b> {
 211     values: vec::IntoIter<(Cow<'a, str>, Value<'a>)>,
 212     next_value: Option<(Cow<'a, str>, Value<'a>)>,
 213     depth: usize,
 214     cur: usize,
 215     cur_parent: usize,
 216     max: usize,
 217     tables: &'b mut [Table<'a>],
 218     array: bool,
 219     de: &'b mut Deserializer<'a>,
 220 }
 221
 222 impl<'a, 'b> de::MapVisitor for MapVisitor<'a, 'b> {
 223     type Error = Error;
 224
 225     fn visit_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Error>
 226         where K: de::DeserializeSeed,
 227     {
 228         if self.cur_parent == self.max || self.cur == self.max {
 229             return Ok(None)
 230         }
 231
 232         loop {
 233             assert!(self.next_value.is_none());
 234             if let Some((key, value)) = self.values.next() {
 235                 let ret = seed.deserialize(StrDeserializer::new(key.clone()))?;
 236                 self.next_value = Some((key, value));
 237                 return Ok(Some(ret))
 238             }
 239
 240             let next_table = {
 241                 let prefix = &self.tables[self.cur_parent].header[..self.depth];
 242                 self.tables[self.cur..self.max].iter().enumerate().find(|&(_, t)| {
 243                     if t.values.is_none() {
 244                         return false
 245                     }
 246                     match t.header.get(..self.depth) {
 247                         Some(header) => header == prefix,
 248                         None => false,
 249                     }
 250                 }).map(|(i, _)| i + self.cur)
 251             };
 252
 253             let pos = match next_table {
 254                 Some(pos) => pos,
 255                 None => return Ok(None),
 256             };
 257             self.cur = pos;
 258
 259             // Test to see if we're duplicating our parent's table, and if so
 260             // then this is an error in the toml format
 261             if self.cur_parent != pos &&
 262                self.tables[self.cur_parent].header == self.tables[pos].header {
 263                 let at = self.tables[pos].at;
 264                 let name = self.tables[pos].header.join(".");
 265                 return Err(self.de.error(at, ErrorKind::DuplicateTable(name)))
 266             }
 267
 268             let table = &mut self.tables[pos];
 269
 270             // If we're not yet at the appropriate depth for this table then we
 271             // just visit the next portion of its header and then continue
 272             // decoding.
 273             if self.depth != table.header.len() {
 274                 let key = &table.header[self.depth];
 275                 let key = seed.deserialize(StrDeserializer::new(key[..].into()))?;
 276                 return Ok(Some(key))
 277             }
 278
 279             // Rule out cases like:
 280             //
 281             //      [[foo.bar]]
 282             //      [[foo]]
 283             if table.array  {
 284                 let kind = ErrorKind::RedefineAsArray;
 285                 return Err(self.de.error(table.at, kind))
 286             }
 287
 288             self.values = table.values.take().unwrap().into_iter();
 289         }
 290     }
 291
 292     fn visit_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Error>
 293         where V: de::DeserializeSeed,
 294     {
 295         if let Some((k, v)) = self.next_value.take() {
 296             match seed.deserialize(ValueDeserializer::new(v)) {
 297                 Ok(v) => return Ok(v),
 298                 Err(mut e) => {
 299                     e.add_key_context(&k);
 300                     return Err(e)
 301                 }
 302             }
 303         }
 304
 305         let array = self.tables[self.cur].array &&
 306                     self.depth == self.tables[self.cur].header.len() - 1;
 307         self.cur += 1;
 308         let res = seed.deserialize(MapVisitor {
 309             values: Vec::new().into_iter(),
 310             next_value: None,
 311             depth: self.depth + if array {0} else {1},
 312             cur_parent: self.cur - 1,
 313             cur: 0,
 314             max: self.max,
 315             array: array,
 316             tables: &mut *self.tables,
 317             de: &mut *self.de,
 318         });
 319         res.map_err(|mut e| {
 320             e.add_key_context(&self.tables[self.cur - 1].header[self.depth]);
 321             e
 322         })
 323     }
 324 }
 325
 326 impl<'a, 'b> de::SeqVisitor for MapVisitor<'a, 'b> {
 327     type Error = Error;
 328
 329     fn visit_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Error>
 330         where K: de::DeserializeSeed,
 331     {
 332         assert!(self.next_value.is_none());
 333         assert!(self.values.next().is_none());
 334
 335         if self.cur_parent == self.max {
 336             return Ok(None)
 337         }
 338
 339         let next = self.tables[..self.max]
 340             .iter()
 341             .enumerate()
 342             .skip(self.cur_parent + 1)
 343             .find(|&(_, table)| {
 344                 table.array && table.header == self.tables[self.cur_parent].header
 345             }).map(|p| p.0)
 346             .unwrap_or(self.max);
 347
 348         let ret = seed.deserialize(MapVisitor {
 349             values: self.tables[self.cur_parent].values.take().unwrap().into_iter(),
 350             next_value: None,
 351             depth: self.depth + 1,
 352             cur_parent: self.cur_parent,
 353             max: next,
 354             cur: 0,
 355             array: false,
 356             tables: &mut self.tables,
 357             de: &mut self.de,
 358         })?;
 359         self.cur_parent = next;
 360         return Ok(Some(ret))
 361     }
 362 }
 363
 364 impl<'a, 'b> de::Deserializer for MapVisitor<'a, 'b> {
 365     type Error = Error;
 366
 367     fn deserialize<V>(self, visitor: V) -> Result<V::Value, Error>
 368         where V: de::Visitor,
 369     {
 370         if self.array  {
 371             visitor.visit_seq(self)
 372         } else {
 373             visitor.visit_map(self)
 374         }
 375     }
 376
 377     // `None` is interpreted as a missing field so be sure to implement `Some`
 378     // as a present field.
 379     fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Error>
 380         where V: de::Visitor
 381     {
 382         visitor.visit_some(self)
 383     }
 384
 385     forward_to_deserialize! {
 386         bool u8 u16 u32 u64 i8 i16 i32 i64 f32 f64 char str string seq
 387         seq_fixed_size bytes byte_buf map struct unit newtype_struct
 388         struct_field ignored_any unit_struct tuple_struct tuple enum
 389     }
 390 }
 391
 392 struct StrDeserializer<'a> {
 393     key: Cow<'a, str>,
 394 }
 395
 396 impl<'a> StrDeserializer<'a> {
 397     fn new(key: Cow<'a, str>) -> StrDeserializer<'a> {
 398         StrDeserializer {
 399             key: key,
 400         }
 401     }
 402 }
 403
 404 impl<'a> de::Deserializer for StrDeserializer<'a> {
 405     type Error = Error;
 406
 407     fn deserialize<V>(self, visitor: V) -> Result<V::Value, Error>
 408         where V: de::Visitor,
 409     {
 410         match self.key {
 411             Cow::Borrowed(s) => visitor.visit_str(s),
 412             Cow::Owned(s) => visitor.visit_string(s),
 413         }
 414     }
 415
 416     forward_to_deserialize! {
 417         bool u8 u16 u32 u64 i8 i16 i32 i64 f32 f64 char str string seq
 418         seq_fixed_size bytes byte_buf map struct option unit newtype_struct
 419         struct_field ignored_any unit_struct tuple_struct tuple enum
 420     }
 421 }
 422
 423 struct ValueDeserializer<'a> {
 424     value: Value<'a>,
 425 }
 426
 427 impl<'a> ValueDeserializer<'a> {
 428     fn new(value: Value<'a>) -> ValueDeserializer<'a> {
 429         ValueDeserializer {
 430             value: value,
 431         }
 432     }
 433 }
 434
 435 impl<'a> de::Deserializer for ValueDeserializer<'a> {
 436     type Error = Error;
 437
 438     fn deserialize<V>(self, visitor: V) -> Result<V::Value, Error>
 439         where V: de::Visitor,
 440     {
 441         match self.value {
 442             Value::Integer(i) => visitor.visit_i64(i),
 443             Value::Boolean(b) => visitor.visit_bool(b),
 444             Value::Float(f) => visitor.visit_f64(f),
 445             Value::String(Cow::Borrowed(s)) => visitor.visit_str(s),
 446             Value::String(Cow::Owned(s)) => visitor.visit_string(s),
 447             Value::Datetime(s) => visitor.visit_map(DatetimeDeserializer {
 448                 date: s,
 449                 visited: false,
 450             }),
 451             Value::Array(values) => {
 452                 let mut s = de::value::SeqDeserializer::new(values.into_iter());
 453                 let ret = visitor.visit_seq(&mut s)?;
 454                 s.end()?;
 455                 Ok(ret)
 456             }
 457             Value::InlineTable(values) => {
 458                 visitor.visit_map(InlineTableDeserializer {
 459                     values: values.into_iter(),
 460                     next_value: None,
 461                 })
 462             }
 463         }
 464     }
 465
 466     fn deserialize_struct<V>(self,
 467                              name: &'static str,
 468                              fields: &'static [&'static str],
 469                              visitor: V) -> Result<V::Value, Error>
 470         where V: de::Visitor,
 471     {
 472         if name == SERDE_STRUCT_NAME && fields == &[SERDE_STRUCT_FIELD_NAME] {
 473             if let Value::Datetime(ref s) = self.value {
 474                 return visitor.visit_map(DatetimeDeserializer {
 475                     date: s,
 476                     visited: false,
 477                 })
 478             }
 479         }
 480
 481         self.deserialize(visitor)
 482     }
 483
 484     // `None` is interpreted as a missing field so be sure to implement `Some`
 485     // as a present field.
 486     fn deserialize_option<V>(self, visitor: V) -> Result<V::Value, Error>
 487         where V: de::Visitor
 488     {
 489         visitor.visit_some(self)
 490     }
 491
 492     forward_to_deserialize! {
 493         bool u8 u16 u32 u64 i8 i16 i32 i64 f32 f64 char str string seq
 494         seq_fixed_size bytes byte_buf map unit newtype_struct
 495         struct_field ignored_any unit_struct tuple_struct tuple enum
 496     }
 497 }
 498
 499 impl<'a> de::value::ValueDeserializer<Error> for Value<'a> {
 500     type Deserializer = ValueDeserializer<'a>;
 501
 502     fn into_deserializer(self) -> Self::Deserializer {
 503         ValueDeserializer::new(self)
 504     }
 505 }
 506
 507 struct DatetimeDeserializer<'a> {
 508     visited: bool,
 509     date: &'a str,
 510 }
 511
 512 impl<'a> de::MapVisitor for DatetimeDeserializer<'a> {
 513     type Error = Error;
 514
 515     fn visit_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Error>
 516         where K: de::DeserializeSeed,
 517     {
 518         if self.visited {
 519             return Ok(None)
 520         }
 521         self.visited = true;
 522         seed.deserialize(DatetimeFieldDeserializer).map(Some)
 523     }
 524
 525     fn visit_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Error>
 526         where V: de::DeserializeSeed,
 527     {
 528         seed.deserialize(StrDeserializer::new(self.date.into()))
 529     }
 530 }
 531
 532 struct DatetimeFieldDeserializer;
 533
 534 impl de::Deserializer for DatetimeFieldDeserializer {
 535     type Error = Error;
 536
 537     fn deserialize<V>(self, visitor: V) -> Result<V::Value, Error>
 538         where V: de::Visitor,
 539     {
 540         visitor.visit_str(SERDE_STRUCT_FIELD_NAME)
 541     }
 542
 543     forward_to_deserialize! {
 544         bool u8 u16 u32 u64 i8 i16 i32 i64 f32 f64 char str string seq
 545         seq_fixed_size bytes byte_buf map struct option unit newtype_struct
 546         struct_field ignored_any unit_struct tuple_struct tuple enum
 547     }
 548 }
 549
 550 struct InlineTableDeserializer<'a> {
 551     values: vec::IntoIter<(Cow<'a, str>, Value<'a>)>,
 552     next_value: Option<Value<'a>>,
 553 }
 554
 555 impl<'a> de::MapVisitor for InlineTableDeserializer<'a> {
 556     type Error = Error;
 557
 558     fn visit_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, Error>
 559         where K: de::DeserializeSeed,
 560     {
 561         let (key, value) = match self.values.next() {
 562             Some(pair) => pair,
 563             None => return Ok(None),
 564         };
 565         self.next_value = Some(value);
 566         seed.deserialize(StrDeserializer::new(key)).map(Some)
 567     }
 568
 569     fn visit_value_seed<V>(&mut self, seed: V) -> Result<V::Value, Error>
 570         where V: de::DeserializeSeed,
 571     {
 572         let value = self.next_value.take().unwrap();
 573         seed.deserialize(ValueDeserializer::new(value))
 574     }
 575 }
 576
 577 impl<'a> Deserializer<'a> {
 578     /// Creates a new deserializer which will be deserializing the string
 579     /// provided.
 580     pub fn new(input: &'a str) -> Deserializer<'a> {
 581         Deserializer {
 582             tokens: Tokenizer::new(input),
 583             input: input,
 584             require_newline_after_table: true,
 585         }
 586     }
 587
 588     /// The `Deserializer::end` method should be called after a value has been
 589     /// fully deserialized.  This allows the `Deserializer` to validate that the
 590     /// input stream is at the end or that it only has trailing
 591     /// whitespace/comments.
 592     pub fn end(&mut self) -> Result<(), Error> {
 593         Ok(())
 594     }
 595
 596     /// Historical versions of toml-rs accidentally allowed a newline after a
 597     /// table definition, but the TOML spec requires a newline after a table
 598     /// definition header.
 599     ///
 600     /// This option can be set to `false` (the default is `true`) to emulate
 601     /// this behavior for backwards compatibility with older toml-rs versions.
 602     pub fn set_require_newline_after_table(&mut self, require: bool) {
 603         self.require_newline_after_table = require;
 604     }
 605
 606     fn line(&mut self) -> Result<Option<Line<'a>>, Error> {
 607         loop {
 608             self.eat_whitespace()?;
 609             if self.eat_comment()? {
 610                 continue
 611             }
 612             if self.eat(Token::Newline)? {
 613                 continue
 614             }
 615             break
 616         }
 617
 618         match self.peek()? {
 619             Some(Token::LeftBracket) => self.table_header().map(Some),
 620             Some(_) => self.key_value().map(Some),
 621             None => Ok(None),
 622         }
 623     }
 624
 625     fn table_header(&mut self) -> Result<Line<'a>, Error> {
 626         let start = self.tokens.current();
 627         self.expect(Token::LeftBracket)?;
 628         let array = self.eat(Token::LeftBracket)?;
 629         let ret = Header::new(self.tokens.clone(),
 630                               array,
 631                               self.require_newline_after_table);
 632         if self.require_newline_after_table {
 633             self.tokens.skip_to_newline();
 634         } else {
 635             loop {
 636                 match self.next()? {
 637                     Some(Token::RightBracket) => {
 638                         if array {
 639                             self.eat(Token::RightBracket)?;
 640                         }
 641                         break
 642                     }
 643                     Some(Token::Newline) |
 644                     None => break,
 645                     _ => {}
 646                 }
 647             }
 648             self.eat_whitespace()?;
 649         }
 650         Ok(Line::Table { at: start, header: ret, array: array })
 651     }
 652
 653     fn key_value(&mut self) -> Result<Line<'a>, Error> {
 654         let key = self.table_key()?;
 655         self.eat_whitespace()?;
 656         self.expect(Token::Equals)?;
 657         self.eat_whitespace()?;
 658
 659         let value = self.value()?;
 660         self.eat_whitespace()?;
 661         if !self.eat_comment()? {
 662             self.eat_newline_or_eof()?;
 663         }
 664
 665         Ok(Line::KeyValue(key, value))
 666     }
 667
 668     fn value(&mut self) -> Result<Value<'a>, Error> {
 669         let at = self.tokens.current();
 670         let value = match self.next()? {
 671             Some(Token::String { val, .. }) => Value::String(val),
 672             Some(Token::Keylike("true")) => Value::Boolean(true),
 673             Some(Token::Keylike("false")) => Value::Boolean(false),
 674             Some(Token::Keylike(key)) => self.number_or_date(key)?,
 675             Some(Token::Plus) => self.number_leading_plus()?,
 676             Some(Token::LeftBrace) => self.inline_table().map(Value::InlineTable)?,
 677             Some(Token::LeftBracket) => self.array().map(Value::Array)?,
 678             Some(token) => {
 679                 return Err(self.error(at, ErrorKind::Wanted {
 680                     expected: "a value",
 681                     found: token.describe(),
 682                 }))
 683             }
 684             None => return Err(self.eof()),
 685         };
 686         Ok(value)
 687     }
 688
 689     fn number_or_date(&mut self, s: &'a str) -> Result<Value<'a>, Error> {
 690         if s.contains("T") || (s.len() > 1 && s[1..].contains("-")) &&
 691            !s.contains("e-") {
 692             self.datetime(s, false).map(Value::Datetime)
 693         } else if self.eat(Token::Colon)? {
 694             self.datetime(s, true).map(Value::Datetime)
 695         } else {
 696             self.number(s)
 697         }
 698     }
 699
 700     fn number(&mut self, s: &'a str) -> Result<Value<'a>, Error> {
 701         if s.contains("e") || s.contains("E") {
 702             self.float(s, None).map(Value::Float)
 703         } else if self.eat(Token::Period)? {
 704             let at = self.tokens.current();
 705             match self.next()? {
 706                 Some(Token::Keylike(after)) => {
 707                     self.float(s, Some(after)).map(Value::Float)
 708                 }
 709                 _ => Err(self.error(at, ErrorKind::NumberInvalid)),
 710             }
 711         } else {
 712             self.integer(s).map(Value::Integer)
 713         }
 714     }
 715
 716     fn number_leading_plus(&mut self) -> Result<Value<'a>, Error> {
 717         let start = self.tokens.current();
 718         match self.next()? {
 719             Some(Token::Keylike(s)) => self.number(s),
 720             _ => Err(self.error(start, ErrorKind::NumberInvalid)),
 721         }
 722     }
 723
 724     fn integer(&self, s: &'a str) -> Result<i64, Error> {
 725         let (prefix, suffix) = self.parse_integer(s, true, false)?;
 726         let start = self.tokens.substr_offset(s);
 727         if suffix != "" {
 728             return Err(self.error(start, ErrorKind::NumberInvalid))
 729         }
 730         prefix.replace("_", "").trim_left_matches("+").parse().map_err(|_e| {
 731             self.error(start, ErrorKind::NumberInvalid)
 732         })
 733     }
 734
 735     fn parse_integer(&self,
 736                      s: &'a str,
 737                      allow_sign: bool,
 738                      allow_leading_zeros: bool)
 739                      -> Result<(&'a str, &'a str), Error> {
 740         let start = self.tokens.substr_offset(s);
 741
 742         let mut first = true;
 743         let mut first_zero = false;
 744         let mut underscore = false;
 745         let mut end = s.len();
 746         for (i, c) in s.char_indices() {
 747             let at = i + start;
 748             if i == 0 && (c == '+' || c == '-') && allow_sign {
 749                 continue
 750             }
 751
 752             match c {
 753                 '0' if first => first_zero = true,
 754                 '0' ... '9' if !first && first_zero && !allow_leading_zeros => {
 755                     return Err(self.error(at, ErrorKind::NumberInvalid))
 756                 }
 757                 '0' ... '9' => underscore = false,
 758                 '_' if first => {
 759                     return Err(self.error(at, ErrorKind::NumberInvalid))
 760                 }
 761                 '_' if !underscore => underscore = true,
 762                 _ => {
 763                     end = i;
 764                     break
 765                 }
 766
 767             }
 768             first = false;
 769         }
 770         if first || underscore {
 771             return Err(self.error(start, ErrorKind::NumberInvalid))
 772         }
 773         Ok((&s[..end], &s[end..]))
 774     }
 775
 776     fn float(&mut self, s: &'a str, after_decimal: Option<&'a str>)
 777              -> Result<f64, Error> {
 778         let (integral, mut suffix) = self.parse_integer(s, true, false)?;
 779         let start = self.tokens.substr_offset(integral);
 780
 781         let mut fraction = None;
 782         if let Some(after) = after_decimal {
 783             if suffix != "" {
 784                 return Err(self.error(start, ErrorKind::NumberInvalid))
 785             }
 786             let (a, b) = self.parse_integer(&after, false, true)?;
 787             fraction = Some(a);
 788             suffix = b;
 789         }
 790
 791         let mut exponent = None;
 792         if suffix.starts_with("e") || suffix.starts_with("E") {
 793             let (a, b) = if suffix.len() == 1 {
 794                 self.eat(Token::Plus)?;
 795                 match self.next()? {
 796                     Some(Token::Keylike(s)) => {
 797                         self.parse_integer(s, false, false)?
 798                     }
 799                     _ => return Err(self.error(start, ErrorKind::NumberInvalid)),
 800                 }
 801             } else {
 802                 self.parse_integer(&suffix[1..], true, false)?
 803             };
 804             if b != "" {
 805                 return Err(self.error(start, ErrorKind::NumberInvalid))
 806             }
 807             exponent = Some(a);
 808         }
 809
 810         let mut number = integral.trim_left_matches("+")
 811                                  .chars()
 812                                  .filter(|c| *c != '_')
 813                                  .collect::<String>();
 814         if let Some(fraction) = fraction {
 815             number.push_str(".");
 816             number.extend(fraction.chars().filter(|c| *c != '_'));
 817         }
 818         if let Some(exponent) = exponent {
 819             number.push_str("E");
 820             number.extend(exponent.chars().filter(|c| *c != '_'));
 821         }
 822         number.parse().map_err(|_e| {
 823             self.error(start, ErrorKind::NumberInvalid)
 824         })
 825     }
 826
 827     fn datetime(&mut self, date: &'a str, colon_eaten: bool)
 828                 -> Result<&'a str, Error> {
 829         let start = self.tokens.substr_offset(date);
 830         if colon_eaten || self.eat(Token::Colon)? {
 831             // minutes
 832             match self.next()? {
 833                 Some(Token::Keylike(_)) => {}
 834                 _ => return Err(self.error(start, ErrorKind::DateInvalid)),
 835             }
 836             // Seconds
 837             self.expect(Token::Colon)?;
 838             match self.next()? {
 839                 Some(Token::Keylike(_)) => {}
 840                 _ => return Err(self.error(start, ErrorKind::DateInvalid)),
 841             }
 842             // Fractional seconds
 843             if self.eat(Token::Period)? {
 844                 match self.next()? {
 845                     Some(Token::Keylike(_)) => {}
 846                     _ => return Err(self.error(start, ErrorKind::DateInvalid)),
 847                 }
 848             }
 849
 850             // offset
 851             if self.eat(Token::Plus)? {
 852                 match self.next()? {
 853                     Some(Token::Keylike(_)) => {}
 854                     _ => return Err(self.error(start, ErrorKind::DateInvalid)),
 855                 }
 856             }
 857             if self.eat(Token::Colon)? {
 858                 match self.next()? {
 859                     Some(Token::Keylike(_)) => {}
 860                     _ => return Err(self.error(start, ErrorKind::DateInvalid)),
 861                 }
 862             }
 863         }
 864         let end = self.tokens.current();
 865         Ok(&self.tokens.input()[start..end])
 866     }
 867
 868     // TODO(#140): shouldn't buffer up this entire table in memory, it'd be
 869     // great to defer parsing everything until later.
 870     fn inline_table(&mut self) -> Result<Vec<(Cow<'a, str>, Value<'a>)>, Error> {
 871         let mut ret = Vec::new();
 872         self.eat_whitespace()?;
 873         if self.eat(Token::RightBrace)? {
 874             return Ok(ret)
 875         }
 876         loop {
 877             let key = self.table_key()?;
 878             self.eat_whitespace()?;
 879             self.expect(Token::Equals)?;
 880             self.eat_whitespace()?;
 881             ret.push((key, self.value()?));
 882
 883             self.eat_whitespace()?;
 884             if self.eat(Token::RightBrace)? {
 885                 return Ok(ret)
 886             }
 887             self.expect(Token::Comma)?;
 888             self.eat_whitespace()?;
 889         }
 890     }
 891
 892     // TODO(#140): shouldn't buffer up this entire array in memory, it'd be
 893     // great to defer parsing everything until later.
 894     fn array(&mut self) -> Result<Vec<Value<'a>>, Error> {
 895         let mut ret = Vec::new();
 896
 897         let intermediate = |me: &mut Deserializer| {
 898             loop {
 899                 me.eat_whitespace()?;
 900                 if !me.eat(Token::Newline)? && !me.eat_comment()? {
 901                     break
 902                 }
 903             }
 904             Ok(())
 905         };
 906
 907         loop {
 908             intermediate(self)?;
 909             if self.eat(Token::RightBracket)? {
 910                 return Ok(ret)
 911             }
 912             let at = self.tokens.current();
 913             let value = self.value()?;
 914             if let Some(last) = ret.last() {
 915                 if !value.same_type(last) {
 916                     return Err(self.error(at, ErrorKind::MixedArrayType))
 917                 }
 918             }
 919             ret.push(value);
 920             intermediate(self)?;
 921             if !self.eat(Token::Comma)? {
 922                 break
 923             }
 924         }
 925         intermediate(self)?;
 926         self.expect(Token::RightBracket)?;
 927         Ok(ret)
 928     }
 929
 930     fn table_key(&mut self) -> Result<Cow<'a, str>, Error> {
 931         self.tokens.table_key().map_err(|e| self.token_error(e))
 932     }
 933
 934     fn eat_whitespace(&mut self) -> Result<(), Error> {
 935         self.tokens.eat_whitespace().map_err(|e| self.token_error(e))
 936     }
 937
 938     fn eat_comment(&mut self) -> Result<bool, Error> {
 939         self.tokens.eat_comment().map_err(|e| self.token_error(e))
 940     }
 941
 942     fn eat_newline_or_eof(&mut self) -> Result<(), Error> {
 943         self.tokens.eat_newline_or_eof().map_err(|e| self.token_error(e))
 944     }
 945
 946     fn eat(&mut self, expected: Token<'a>) -> Result<bool, Error> {
 947         self.tokens.eat(expected).map_err(|e| self.token_error(e))
 948     }
 949
 950     fn expect(&mut self, expected: Token<'a>) -> Result<(), Error> {
 951         self.tokens.expect(expected).map_err(|e| self.token_error(e))
 952     }
 953
 954     fn next(&mut self) -> Result<Option<Token<'a>>, Error> {
 955         self.tokens.next().map_err(|e| self.token_error(e))
 956     }
 957
 958     fn peek(&mut self) -> Result<Option<Token<'a>>, Error> {
 959         self.tokens.peek().map_err(|e| self.token_error(e))
 960     }
 961
 962     fn eof(&self) -> Error {
 963         self.error(self.input.len(), ErrorKind::UnexpectedEof)
 964     }
 965
 966     fn token_error(&self, error: TokenError) -> Error {
 967         match error {
 968             TokenError::InvalidCharInString(at, ch) => {
 969                 self.error(at, ErrorKind::InvalidCharInString(ch))
 970             }
 971             TokenError::InvalidEscape(at, ch) => {
 972                 self.error(at, ErrorKind::InvalidEscape(ch))
 973             }
 974             TokenError::InvalidEscapeValue(at, v) => {
 975                 self.error(at, ErrorKind::InvalidEscapeValue(v))
 976             }
 977             TokenError::InvalidHexEscape(at, ch) => {
 978                 self.error(at, ErrorKind::InvalidHexEscape(ch))
 979             }
 980             TokenError::NewlineInString(at) => {
 981                 self.error(at, ErrorKind::NewlineInString)
 982             }
 983             TokenError::Unexpected(at, ch) => {
 984                 self.error(at, ErrorKind::Unexpected(ch))
 985             }
 986             TokenError::UnterminatedString(at) => {
 987                 self.error(at, ErrorKind::UnterminatedString)
 988             }
 989             TokenError::NewlineInTableKey(at) => {
 990                 self.error(at, ErrorKind::NewlineInTableKey)
 991             }
 992             TokenError::Wanted { at, expected, found } => {
 993                 self.error(at, ErrorKind::Wanted { expected: expected, found: found })
 994             }
 995             TokenError::EmptyTableKey(at) => {
 996                 self.error(at, ErrorKind::EmptyTableKey)
 997             }
 998         }
 999     }
1000
1001     fn error(&self, at: usize, kind: ErrorKind) -> Error {
1002         let mut err = Error::from_kind(kind);
1003         let (line, col) = self.to_linecol(at);
1004         err.inner.line = Some(line);
1005         err.inner.col = col;
1006         return err
1007     }
1008
1009     /// Converts a byte offset from an error message to a (line, column) pair
1010     ///
1011     /// All indexes are 0-based.
1012     fn to_linecol(&self, offset: usize) -> (usize, usize) {
1013         let mut cur = 0;
1014         for (i, line) in self.input.lines().enumerate() {
1015             if cur + line.len() + 1 > offset {
1016                 return (i, offset - cur)
1017             }
1018             cur += line.len() + 1;
1019         }
1020         (self.input.lines().count(), 0)
1021     }
1022 }
1023
1024 impl Error {
1025     fn from_kind(kind: ErrorKind) -> Error {
1026         Error {
1027             inner: Box::new(ErrorInner {
1028                 kind: kind,
1029                 line: None,
1030                 col: 0,
1031                 message: String::new(),
1032                 key: Vec::new(),
1033             }),
1034         }
1035     }
1036
1037     fn custom(s: String) -> Error {
1038         Error {
1039             inner: Box::new(ErrorInner {
1040                 kind: ErrorKind::Custom,
1041                 line: None,
1042                 col: 0,
1043                 message: s,
1044                 key: Vec::new(),
1045             }),
1046         }
1047     }
1048
1049     /// Do not call this method, it may be removed at any time, it's just an
1050     /// internal implementation detail.
1051     #[doc(hidden)]
1052     pub fn add_key_context(&mut self, key: &str) {
1053         self.inner.key.insert(0, key.to_string());
1054     }
1055 }
1056
1057 impl fmt::Display for Error {
1058     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1059         match self.inner.kind {
1060             ErrorKind::UnexpectedEof => "unexpected eof encountered".fmt(f)?,
1061             ErrorKind::InvalidCharInString(c) => {
1062                 write!(f, "invalid character in string: `{}`",
1063                        c.escape_default().collect::<String>())?
1064             }
1065             ErrorKind::InvalidEscape(c) => {
1066                 write!(f, "invalid escape character in string: `{}`",
1067                        c.escape_default().collect::<String>())?
1068             }
1069             ErrorKind::InvalidHexEscape(c) => {
1070                 write!(f, "invalid hex escape character in string: `{}`",
1071                        c.escape_default().collect::<String>())?
1072             }
1073             ErrorKind::InvalidEscapeValue(c) => {
1074                 write!(f, "invalid escape value: `{}`", c)?
1075             }
1076             ErrorKind::NewlineInString => "newline in string found".fmt(f)?,
1077             ErrorKind::Unexpected(ch) => {
1078                 write!(f, "unexpected character found: `{}`",
1079                        ch.escape_default().collect::<String>())?
1080             }
1081             ErrorKind::UnterminatedString => "unterminated string".fmt(f)?,
1082             ErrorKind::NewlineInTableKey => "found newline in table key".fmt(f)?,
1083             ErrorKind::Wanted { expected, found } => {
1084                 write!(f, "expected {}, found {}", expected, found)?
1085             }
1086             ErrorKind::NumberInvalid => "invalid number".fmt(f)?,
1087             ErrorKind::DateInvalid => "invalid date".fmt(f)?,
1088             ErrorKind::MixedArrayType => "mixed types in an array".fmt(f)?,
1089             ErrorKind::DuplicateTable(ref s) => {
1090                 write!(f, "redefinition of table `{}`", s)?;
1091             }
1092             ErrorKind::RedefineAsArray => "table redefined as array".fmt(f)?,
1093             ErrorKind::EmptyTableKey => "empty table key found".fmt(f)?,
1094             ErrorKind::Custom => self.inner.message.fmt(f)?,
1095             ErrorKind::__Nonexhaustive => panic!(),
1096         }
1097
1098         if self.inner.key.len() > 0 {
1099             write!(f, " for key `")?;
1100             for (i, k) in self.inner.key.iter().enumerate() {
1101                 if i > 0 {
1102                     write!(f, ".")?;
1103                 }
1104                 write!(f, "{}", k)?;
1105             }
1106             write!(f, "`")?;
1107         }
1108
1109         if let Some(line) = self.inner.line {
1110             write!(f, " at line {}", line + 1)?;
1111         }
1112
1113         Ok(())
1114     }
1115 }
1116
1117 impl error::Error for Error {
1118     fn description(&self) -> &str {
1119         match self.inner.kind {
1120             ErrorKind::UnexpectedEof => "unexpected eof encountered",
1121             ErrorKind::InvalidCharInString(_) => "invalid char in string",
1122             ErrorKind::InvalidEscape(_) => "invalid escape in string",
1123             ErrorKind::InvalidHexEscape(_) => "invalid hex escape in string",
1124             ErrorKind::InvalidEscapeValue(_) => "invalid escape value in string",
1125             ErrorKind::NewlineInString => "newline in string found",
1126             ErrorKind::Unexpected(_) => "unexpected or invalid character",
1127             ErrorKind::UnterminatedString => "unterminated string",
1128             ErrorKind::NewlineInTableKey => "found newline in table key",
1129             ErrorKind::Wanted { .. } => "expected a token but found another",
1130             ErrorKind::NumberInvalid => "invalid number",
1131             ErrorKind::DateInvalid => "invalid date",
1132             ErrorKind::MixedArrayType => "mixed types in an array",
1133             ErrorKind::DuplicateTable(_) => "duplicate table",
1134             ErrorKind::RedefineAsArray => "table redefined as array",
1135             ErrorKind::EmptyTableKey => "empty table key found",
1136             ErrorKind::Custom => "a custom error",
1137             ErrorKind::__Nonexhaustive => panic!(),
1138         }
1139     }
1140 }
1141
1142 impl de::Error for Error {
1143     fn custom<T: fmt::Display>(msg: T) -> Error {
1144         Error::custom(msg.to_string())
1145     }
1146 }
1147
1148 enum Line<'a> {
1149     Table { at: usize, header: Header<'a>, array: bool },
1150     KeyValue(Cow<'a, str>, Value<'a>),
1151 }
1152
1153 struct Header<'a> {
1154     first: bool,
1155     array: bool,
1156     require_newline_after_table: bool,
1157     tokens: Tokenizer<'a>,
1158 }
1159
1160 impl<'a> Header<'a> {
1161     fn new(tokens: Tokenizer<'a>,
1162            array: bool,
1163            require_newline_after_table: bool) -> Header<'a> {
1164         Header {
1165             first: true,
1166             array: array,
1167             tokens: tokens,
1168             require_newline_after_table: require_newline_after_table,
1169         }
1170     }
1171
1172     fn next(&mut self) -> Result<Option<Cow<'a, str>>, TokenError> {
1173         self.tokens.eat_whitespace()?;
1174
1175         if self.first || self.tokens.eat(Token::Period)? {
1176             self.first = false;
1177             self.tokens.eat_whitespace()?;
1178             self.tokens.table_key().map(Some)
1179         } else {
1180             self.tokens.expect(Token::RightBracket)?;
1181             if self.array {
1182                 self.tokens.expect(Token::RightBracket)?;
1183             }
1184
1185             self.tokens.eat_whitespace()?;
1186             if self.require_newline_after_table {
1187                 if !self.tokens.eat_comment()? {
1188                     self.tokens.eat_newline_or_eof()?;
1189                 }
1190             }
1191             Ok(None)
1192         }
1193     }
1194 }
1195
1196 enum Value<'a> {
1197     Integer(i64),
1198     Float(f64),
1199     Boolean(bool),
1200     String(Cow<'a, str>),
1201     Datetime(&'a str),
1202     Array(Vec<Value<'a>>),
1203     InlineTable(Vec<(Cow<'a, str>, Value<'a>)>),
1204 }
1205
1206 impl<'a> Value<'a> {
1207     fn same_type(&self, other: &Value<'a>) -> bool {
1208         match (self, other) {
1209             (&Value::String(..), &Value::String(..)) |
1210             (&Value::Integer(..), &Value::Integer(..)) |
1211             (&Value::Float(..), &Value::Float(..)) |
1212             (&Value::Boolean(..), &Value::Boolean(..)) |
1213             (&Value::Datetime(..), &Value::Datetime(..)) |
1214             (&Value::Array(..), &Value::Array(..)) |
1215             (&Value::InlineTable(..), &Value::InlineTable(..)) => true,
1216
1217             _ => false,
1218         }
1219     }
1220 }