src/backup/catalog.rs

   1 use std::convert::TryFrom;
   2 use std::ffi::{CStr, CString, OsStr};
   3 use std::fmt;
   4 use std::io::{Read, Write, Seek, SeekFrom};
   5 use std::os::unix::ffi::OsStrExt;
   6
   7 use anyhow::{bail, format_err, Error};
   8
   9 use pathpatterns::{MatchList, MatchType};
  10 use proxmox::tools::io::ReadExt;
  11
  12 use crate::backup::file_formats::PROXMOX_CATALOG_FILE_MAGIC_1_0;
  13 use crate::pxar::catalog::BackupCatalogWriter;
  14
  15 #[repr(u8)]
  16 #[derive(Copy,Clone,PartialEq)]
  17 pub(crate) enum CatalogEntryType {
  18     Directory = b'd',
  19     File = b'f',
  20     Symlink = b'l',
  21     Hardlink = b'h',
  22     BlockDevice = b'b',
  23     CharDevice = b'c',
  24     Fifo = b'p', // Fifo,Pipe
  25     Socket = b's',
  26 }
  27
  28 impl TryFrom<u8> for CatalogEntryType {
  29     type Error=Error;
  30
  31     fn try_from(value: u8) -> Result<Self, Error> {
  32         Ok(match value {
  33             b'd' => CatalogEntryType::Directory,
  34             b'f' => CatalogEntryType::File,
  35             b'l' => CatalogEntryType::Symlink,
  36             b'h' => CatalogEntryType::Hardlink,
  37             b'b' => CatalogEntryType::BlockDevice,
  38             b'c' => CatalogEntryType::CharDevice,
  39             b'p' => CatalogEntryType::Fifo,
  40             b's' => CatalogEntryType::Socket,
  41             _ => bail!("invalid CatalogEntryType value '{}'", char::from(value)),
  42         })
  43     }
  44 }
  45
  46 impl From<&DirEntryAttribute> for CatalogEntryType {
  47     fn from(value: &DirEntryAttribute) -> Self {
  48         match value {
  49             DirEntryAttribute::Directory { .. } => CatalogEntryType::Directory,
  50             DirEntryAttribute::File { .. } => CatalogEntryType::File,
  51             DirEntryAttribute::Symlink => CatalogEntryType::Symlink,
  52             DirEntryAttribute::Hardlink => CatalogEntryType::Hardlink,
  53             DirEntryAttribute::BlockDevice => CatalogEntryType::BlockDevice,
  54             DirEntryAttribute::CharDevice => CatalogEntryType::CharDevice,
  55             DirEntryAttribute::Fifo => CatalogEntryType::Fifo,
  56             DirEntryAttribute::Socket => CatalogEntryType::Socket,
  57         }
  58     }
  59 }
  60
  61 impl fmt::Display for CatalogEntryType {
  62     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  63         write!(f, "{}", char::from(*self as u8))
  64     }
  65 }
  66
  67 /// Represents a named directory entry
  68 ///
  69 /// The ``attr`` property contain the exact type with type specific
  70 /// attributes.
  71 #[derive(Clone, PartialEq)]
  72 pub struct DirEntry {
  73     pub name: Vec<u8>,
  74     pub attr: DirEntryAttribute,
  75 }
  76
  77 /// Used to specific additional attributes inside DirEntry
  78 #[derive(Clone, Debug, PartialEq)]
  79 pub enum DirEntryAttribute {
  80     Directory { start: u64 },
  81     File { size: u64, mtime: i64 },
  82     Symlink,
  83     Hardlink,
  84     BlockDevice,
  85     CharDevice,
  86     Fifo,
  87     Socket,
  88 }
  89
  90 impl DirEntry {
  91
  92     fn new(etype: CatalogEntryType, name: Vec<u8>, start: u64, size: u64, mtime: i64) -> Self {
  93         match etype {
  94             CatalogEntryType::Directory => {
  95                 DirEntry { name, attr: DirEntryAttribute::Directory { start } }
  96             }
  97             CatalogEntryType::File => {
  98                 DirEntry { name, attr: DirEntryAttribute::File { size, mtime } }
  99             }
 100             CatalogEntryType::Symlink => {
 101                 DirEntry { name, attr: DirEntryAttribute::Symlink }
 102             }
 103             CatalogEntryType::Hardlink => {
 104                 DirEntry { name, attr: DirEntryAttribute::Hardlink }
 105             }
 106             CatalogEntryType::BlockDevice => {
 107                 DirEntry { name, attr: DirEntryAttribute::BlockDevice }
 108             }
 109             CatalogEntryType::CharDevice => {
 110                 DirEntry { name, attr: DirEntryAttribute::CharDevice }
 111             }
 112             CatalogEntryType::Fifo => {
 113                 DirEntry { name, attr: DirEntryAttribute::Fifo }
 114             }
 115             CatalogEntryType::Socket => {
 116                 DirEntry { name, attr: DirEntryAttribute::Socket }
 117             }
 118         }
 119     }
 120
 121     /// Get file mode bits for this entry to be used with the `MatchList` api.
 122     pub fn get_file_mode(&self) -> Option<u32> {
 123         Some(
 124             match self.attr {
 125                 DirEntryAttribute::Directory { .. } => pxar::mode::IFDIR,
 126                 DirEntryAttribute::File { .. } => pxar::mode::IFREG,
 127                 DirEntryAttribute::Symlink => pxar::mode::IFLNK,
 128                 DirEntryAttribute::Hardlink => return None,
 129                 DirEntryAttribute::BlockDevice => pxar::mode::IFBLK,
 130                 DirEntryAttribute::CharDevice => pxar::mode::IFCHR,
 131                 DirEntryAttribute::Fifo => pxar::mode::IFIFO,
 132                 DirEntryAttribute::Socket => pxar::mode::IFSOCK,
 133             }
 134             as u32
 135         )
 136     }
 137
 138     /// Check if DirEntry is a directory
 139     pub fn is_directory(&self) -> bool {
 140         matches!(self.attr, DirEntryAttribute::Directory { .. })
 141     }
 142
 143     /// Check if DirEntry is a symlink
 144     pub fn is_symlink(&self) -> bool {
 145         matches!(self.attr, DirEntryAttribute::Symlink { .. })
 146     }
 147 }
 148
 149 struct DirInfo {
 150     name: CString,
 151     entries: Vec<DirEntry>,
 152 }
 153
 154 impl DirInfo {
 155
 156     fn new(name: CString) -> Self {
 157         DirInfo { name, entries: Vec::new() }
 158     }
 159
 160     fn new_rootdir() -> Self {
 161         DirInfo::new(CString::new(b"/".to_vec()).unwrap())
 162     }
 163
 164     fn encode_entry<W: Write>(
 165         writer: &mut W,
 166         entry: &DirEntry,
 167         pos: u64,
 168     ) -> Result<(), Error> {
 169         match entry {
 170             DirEntry { name, attr: DirEntryAttribute::Directory { start } } => {
 171                 writer.write_all(&[CatalogEntryType::Directory as u8])?;
 172                 catalog_encode_u64(writer, name.len() as u64)?;
 173                 writer.write_all(name)?;
 174                 catalog_encode_u64(writer, pos - start)?;
 175             }
 176             DirEntry { name, attr: DirEntryAttribute::File { size, mtime } } => {
 177                 writer.write_all(&[CatalogEntryType::File as u8])?;
 178                 catalog_encode_u64(writer, name.len() as u64)?;
 179                 writer.write_all(name)?;
 180                 catalog_encode_u64(writer, *size)?;
 181                 catalog_encode_i64(writer, *mtime)?;
 182             }
 183             DirEntry { name, attr: DirEntryAttribute::Symlink } => {
 184                 writer.write_all(&[CatalogEntryType::Symlink as u8])?;
 185                 catalog_encode_u64(writer, name.len() as u64)?;
 186                 writer.write_all(name)?;
 187             }
 188             DirEntry { name, attr: DirEntryAttribute::Hardlink } => {
 189                 writer.write_all(&[CatalogEntryType::Hardlink as u8])?;
 190                 catalog_encode_u64(writer, name.len() as u64)?;
 191                 writer.write_all(name)?;
 192             }
 193             DirEntry { name, attr: DirEntryAttribute::BlockDevice } => {
 194                 writer.write_all(&[CatalogEntryType::BlockDevice as u8])?;
 195                 catalog_encode_u64(writer, name.len() as u64)?;
 196                 writer.write_all(name)?;
 197             }
 198             DirEntry { name, attr: DirEntryAttribute::CharDevice } => {
 199                 writer.write_all(&[CatalogEntryType::CharDevice as u8])?;
 200                 catalog_encode_u64(writer, name.len() as u64)?;
 201                 writer.write_all(name)?;
 202             }
 203             DirEntry { name, attr: DirEntryAttribute::Fifo } => {
 204                 writer.write_all(&[CatalogEntryType::Fifo as u8])?;
 205                 catalog_encode_u64(writer, name.len() as u64)?;
 206                 writer.write_all(name)?;
 207             }
 208             DirEntry { name, attr: DirEntryAttribute::Socket } => {
 209                 writer.write_all(&[CatalogEntryType::Socket as u8])?;
 210                 catalog_encode_u64(writer, name.len() as u64)?;
 211                 writer.write_all(name)?;
 212             }
 213         }
 214         Ok(())
 215     }
 216
 217     fn encode(self, start: u64) -> Result<(CString, Vec<u8>), Error> {
 218         let mut table = Vec::new();
 219         catalog_encode_u64(&mut table, self.entries.len() as u64)?;
 220         for entry in self.entries {
 221             Self::encode_entry(&mut table, &entry, start)?;
 222         }
 223
 224         let mut data = Vec::new();
 225         catalog_encode_u64(&mut data, table.len() as u64)?;
 226         data.extend_from_slice(&table);
 227
 228         Ok((self.name, data))
 229     }
 230
 231     fn parse<C: FnMut(CatalogEntryType, &[u8], u64, u64, i64) -> Result<bool, Error>>(
 232         data: &[u8],
 233         mut callback: C,
 234     ) -> Result<(), Error> {
 235
 236         let mut cursor = data;
 237
 238         let entries = catalog_decode_u64(&mut cursor)?;
 239
 240         let mut name_buf = vec![0u8; 4096];
 241
 242         for _ in 0..entries {
 243
 244             let mut buf = [ 0u8 ];
 245             cursor.read_exact(&mut buf)?;
 246             let etype = CatalogEntryType::try_from(buf[0])?;
 247
 248             let name_len = catalog_decode_u64(&mut cursor)? as usize;
 249             if name_len >= name_buf.len() {
 250                 bail!("directory entry name too long ({} >= {})", name_len, name_buf.len());
 251             }
 252             let name = &mut name_buf[0..name_len];
 253             cursor.read_exact(name)?;
 254
 255             let cont = match etype {
 256                 CatalogEntryType::Directory => {
 257                     let offset = catalog_decode_u64(&mut cursor)?;
 258                     callback(etype, name, offset, 0, 0)?
 259                 }
 260                 CatalogEntryType::File => {
 261                     let size = catalog_decode_u64(&mut cursor)?;
 262                     let mtime = catalog_decode_i64(&mut cursor)?;
 263                     callback(etype, name, 0, size, mtime)?
 264                 }
 265                 _ => {
 266                     callback(etype, name, 0, 0, 0)?
 267                 }
 268             };
 269             if !cont {
 270                 return Ok(());
 271             }
 272         }
 273
 274         if !cursor.is_empty() {
 275             bail!("unable to parse whole catalog data block");
 276         }
 277
 278         Ok(())
 279     }
 280 }
 281
 282 /// Write small catalog files
 283 ///
 284 /// A Catalogs simply contains list of files and directories
 285 /// (directory tree). They are use to find content without having to
 286 /// search the real archive (which may be large). For files, they
 287 /// include the last modification time and file size.
 288 pub struct CatalogWriter<W> {
 289     writer: W,
 290     dirstack: Vec<DirInfo>,
 291     pos: u64,
 292 }
 293
 294 impl <W: Write> CatalogWriter<W> {
 295
 296     /// Create a new  CatalogWriter instance
 297     pub fn new(writer: W) -> Result<Self, Error> {
 298         let mut me = Self { writer, dirstack: vec![ DirInfo::new_rootdir() ], pos: 0 };
 299         me.write_all(&PROXMOX_CATALOG_FILE_MAGIC_1_0)?;
 300         Ok(me)
 301     }
 302
 303     fn write_all(&mut self, data: &[u8]) -> Result<(), Error> {
 304         self.writer.write_all(data)?;
 305         self.pos += u64::try_from(data.len())?;
 306         Ok(())
 307     }
 308
 309     /// Finish writing, flush all data
 310     ///
 311     /// This need to be called before drop.
 312     pub fn finish(&mut self) -> Result<(), Error> {
 313         if self.dirstack.len() != 1 {
 314             bail!("unable to finish catalog at level {}", self.dirstack.len());
 315         }
 316
 317         let dir = self.dirstack.pop().unwrap();
 318
 319         let start = self.pos;
 320         let (_, data) = dir.encode(start)?;
 321         self.write_all(&data)?;
 322
 323         self.write_all(&start.to_le_bytes())?;
 324
 325         self.writer.flush()?;
 326
 327         Ok(())
 328     }
 329 }
 330
 331 impl <W: Write> BackupCatalogWriter for CatalogWriter<W> {
 332
 333     fn start_directory(&mut self, name: &CStr) -> Result<(), Error> {
 334         let new = DirInfo::new(name.to_owned());
 335         self.dirstack.push(new);
 336         Ok(())
 337     }
 338
 339     fn end_directory(&mut self) -> Result<(), Error> {
 340         let (start, name) = match self.dirstack.pop() {
 341             Some(dir) => {
 342                 let start = self.pos;
 343                 let (name, data) = dir.encode(start)?;
 344                 self.write_all(&data)?;
 345                 (start, name)
 346             }
 347             None => {
 348                 bail!("got unexpected end_directory level 0");
 349             }
 350         };
 351
 352         let current = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 353         let name = name.to_bytes().to_vec();
 354         current.entries.push(DirEntry { name, attr: DirEntryAttribute::Directory { start } });
 355
 356         Ok(())
 357     }
 358
 359     fn add_file(&mut self, name: &CStr, size: u64, mtime: i64) -> Result<(), Error> {
 360         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 361         let name = name.to_bytes().to_vec();
 362         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::File { size, mtime } });
 363         Ok(())
 364     }
 365
 366     fn add_symlink(&mut self, name: &CStr) -> Result<(), Error> {
 367         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 368         let name = name.to_bytes().to_vec();
 369         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::Symlink });
 370         Ok(())
 371     }
 372
 373     fn add_hardlink(&mut self, name: &CStr) -> Result<(), Error> {
 374         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 375         let name = name.to_bytes().to_vec();
 376         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::Hardlink });
 377         Ok(())
 378     }
 379
 380     fn add_block_device(&mut self, name: &CStr) -> Result<(), Error> {
 381         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 382         let name = name.to_bytes().to_vec();
 383         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::BlockDevice });
 384         Ok(())
 385     }
 386
 387     fn add_char_device(&mut self, name: &CStr) -> Result<(), Error> {
 388         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 389         let name = name.to_bytes().to_vec();
 390         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::CharDevice });
 391         Ok(())
 392     }
 393
 394     fn add_fifo(&mut self, name: &CStr) -> Result<(), Error> {
 395         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 396         let name = name.to_bytes().to_vec();
 397         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::Fifo });
 398         Ok(())
 399     }
 400
 401     fn add_socket(&mut self, name: &CStr) -> Result<(), Error> {
 402         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 403         let name = name.to_bytes().to_vec();
 404         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::Socket });
 405         Ok(())
 406     }
 407 }
 408
 409 /// Read Catalog files
 410 pub struct CatalogReader<R> {
 411     reader: R,
 412 }
 413
 414 impl <R: Read + Seek> CatalogReader<R> {
 415
 416     /// Create a new CatalogReader instance
 417     pub fn new(reader: R) -> Self {
 418         Self { reader }
 419     }
 420
 421     /// Print whole catalog to stdout
 422     pub fn dump(&mut self) -> Result<(), Error> {
 423
 424         let root = self.root()?;
 425         match root {
 426             DirEntry { attr: DirEntryAttribute::Directory { start }, .. }=> {
 427                 self.dump_dir(std::path::Path::new("./"), start)
 428             }
 429             _ => unreachable!(),
 430         }
 431     }
 432
 433     /// Get the root DirEntry
 434     pub fn root(&mut self) ->  Result<DirEntry, Error>  {
 435         // Root dir is special
 436         self.reader.seek(SeekFrom::Start(0))?;
 437         let mut magic = [ 0u8; 8];
 438         self.reader.read_exact(&mut magic)?;
 439         if magic != PROXMOX_CATALOG_FILE_MAGIC_1_0 {
 440             bail!("got unexpected magic number for catalog");
 441         }
 442         self.reader.seek(SeekFrom::End(-8))?;
 443         let start = unsafe { self.reader.read_le_value::<u64>()? };
 444         Ok(DirEntry { name: b"".to_vec(), attr: DirEntryAttribute::Directory { start } })
 445     }
 446
 447     /// Read all directory entries
 448     pub fn read_dir(
 449         &mut self,
 450         parent: &DirEntry,
 451     ) -> Result<Vec<DirEntry>, Error>  {
 452
 453         let start = match parent.attr {
 454             DirEntryAttribute::Directory { start } => start,
 455             _ => bail!("parent is not a directory - internal error"),
 456         };
 457
 458         let data = self.read_raw_dirinfo_block(start)?;
 459
 460         let mut entry_list = Vec::new();
 461
 462         DirInfo::parse(&data, |etype, name, offset, size, mtime| {
 463             let entry = DirEntry::new(etype, name.to_vec(), start - offset, size, mtime);
 464             entry_list.push(entry);
 465             Ok(true)
 466         })?;
 467
 468         Ok(entry_list)
 469     }
 470
 471     /// Lookup a DirEntry from an absolute path
 472     pub fn lookup_recursive(
 473         &mut self,
 474         path: &[u8],
 475     ) -> Result<DirEntry, Error> {
 476         let mut current = self.root()?;
 477         if path == b"/" {
 478             return Ok(current);
 479         }
 480
 481         let components = if !path.is_empty() && path[0] == b'/' {
 482             &path[1..]
 483         } else {
 484             path
 485         }.split(|c| *c == b'/');
 486
 487         for comp in components {
 488             if let Some(entry) = self.lookup(&current, comp)? {
 489                 current = entry;
 490             } else {
 491                 bail!("path {:?} not found in catalog", String::from_utf8_lossy(&path));
 492             }
 493         }
 494         Ok(current)
 495     }
 496
 497     /// Lockup a DirEntry inside a parent directory
 498     pub fn lookup(
 499         &mut self,
 500         parent: &DirEntry,
 501         filename: &[u8],
 502     ) -> Result<Option<DirEntry>, Error>  {
 503
 504         let start = match parent.attr {
 505             DirEntryAttribute::Directory { start } => start,
 506             _ => bail!("parent is not a directory - internal error"),
 507         };
 508
 509         let data = self.read_raw_dirinfo_block(start)?;
 510
 511         let mut item = None;
 512         DirInfo::parse(&data, |etype, name, offset, size, mtime| {
 513             if name != filename {
 514                 return Ok(true);
 515             }
 516
 517             let entry = DirEntry::new(etype, name.to_vec(), start - offset, size, mtime);
 518             item = Some(entry);
 519             Ok(false) // stop parsing
 520         })?;
 521
 522         Ok(item)
 523     }
 524
 525     /// Read the raw directory info block from current reader position.
 526     fn read_raw_dirinfo_block(&mut self, start: u64) ->  Result<Vec<u8>, Error>  {
 527         self.reader.seek(SeekFrom::Start(start))?;
 528         let size = catalog_decode_u64(&mut self.reader)?;
 529         if size < 1 { bail!("got small directory size {}", size) };
 530         let data = self.reader.read_exact_allocated(size as usize)?;
 531         Ok(data)
 532     }
 533
 534     /// Print the content of a directory to stdout
 535     pub fn dump_dir(&mut self, prefix: &std::path::Path, start: u64) -> Result<(), Error> {
 536
 537         let data = self.read_raw_dirinfo_block(start)?;
 538
 539         DirInfo::parse(&data, |etype, name, offset, size, mtime| {
 540
 541             let mut path = std::path::PathBuf::from(prefix);
 542             let name: &OsStr = OsStrExt::from_bytes(name);
 543             path.push(name);
 544
 545             match etype {
 546                 CatalogEntryType::Directory => {
 547                     println!("{} {:?}", etype, path);
 548                     if offset > start {
 549                         bail!("got wrong directory offset ({} > {})", offset, start);
 550                     }
 551                     let pos = start - offset;
 552                     self.dump_dir(&path, pos)?;
 553                 }
 554                 CatalogEntryType::File => {
 555                     let mut mtime_string = mtime.to_string();
 556                     if let Ok(s) = proxmox::tools::time::strftime_local("%FT%TZ", mtime as i64) {
 557                         mtime_string = s;
 558                     }
 559
 560                     println!(
 561                         "{} {:?} {} {}",
 562                         etype,
 563                         path,
 564                         size,
 565                         mtime_string,
 566                     );
 567                 }
 568                 _ => {
 569                     println!("{} {:?}", etype, path);
 570                 }
 571             }
 572
 573             Ok(true)
 574         })
 575     }
 576
 577     /// Finds all entries matching the given match patterns and calls the
 578     /// provided callback on them.
 579     pub fn find(
 580         &mut self,
 581         parent: &DirEntry,
 582         file_path: &mut Vec<u8>,
 583         match_list: &impl MatchList, //&[MatchEntry],
 584         callback: &mut dyn FnMut(&[u8]) -> Result<(), Error>,
 585     ) -> Result<(), Error> {
 586         let file_len = file_path.len();
 587         for e in self.read_dir(parent)?  {
 588             let is_dir = e.is_directory();
 589             file_path.truncate(file_len);
 590             if !e.name.starts_with(b"/") {
 591                 file_path.reserve(e.name.len() + 1);
 592                 file_path.push(b'/');
 593             }
 594             file_path.extend(&e.name);
 595             match match_list.matches(&file_path, e.get_file_mode()) {
 596                 Some(MatchType::Exclude) => continue,
 597                 Some(MatchType::Include) => callback(&file_path)?,
 598                 None => (),
 599             }
 600             if is_dir {
 601                 self.find(&e, file_path, match_list, callback)?;
 602             }
 603         }
 604         file_path.truncate(file_len);
 605
 606         Ok(())
 607     }
 608 }
 609
 610 /// Serialize i64 as short, variable length byte sequence
 611 ///
 612 /// Stores 7 bits per byte, Bit 8 indicates the end of the sequence (when not set).
 613 /// If the value is negative, we end with a zero byte (0x00).
 614 #[allow(clippy::neg_multiply)]
 615 pub fn catalog_encode_i64<W: Write>(writer: &mut W, v: i64) -> Result<(), Error> {
 616     let mut enc = Vec::new();
 617
 618     let mut d = if v < 0 {
 619         (-1 * (v + 1)) as u64 + 1 // also handles i64::MIN
 620     } else {
 621         v as u64
 622     };
 623
 624     loop {
 625         if d < 128 {
 626             if v < 0 {
 627                 enc.push(128 | d as u8);
 628                 enc.push(0u8);
 629             } else {
 630                 enc.push(d as u8);
 631             }
 632             break;
 633         }
 634         enc.push((128 | (d & 127)) as u8);
 635         d >>= 7;
 636     }
 637     writer.write_all(&enc)?;
 638
 639     Ok(())
 640 }
 641
 642 /// Deserialize i64 from variable length byte sequence
 643 ///
 644 /// We currently read maximal 11 bytes, which give a maximum of 70 bits + sign.
 645 /// this method is compatible with catalog_encode_u64 iff the
 646 /// value encoded is <= 2^63 (values > 2^63 cannot be represented in an i64)
 647 #[allow(clippy::neg_multiply)]
 648 pub fn catalog_decode_i64<R: Read>(reader: &mut R) -> Result<i64, Error> {
 649
 650     let mut v: u64 = 0;
 651     let mut buf = [0u8];
 652
 653     for i in 0..11 { // only allow 11 bytes (70 bits + sign marker)
 654         if buf.is_empty() {
 655             bail!("decode_i64 failed - unexpected EOB");
 656         }
 657         reader.read_exact(&mut buf)?;
 658
 659         let t = buf[0];
 660
 661         if t == 0 {
 662             if v == 0 {
 663                 return Ok(0);
 664             }
 665             return Ok(((v - 1) as i64 * -1) - 1); // also handles i64::MIN
 666         } else if t < 128 {
 667             v |= (t as u64) << (i*7);
 668             return Ok(v as i64);
 669         } else {
 670             v |= ((t & 127) as u64) << (i*7);
 671         }
 672     }
 673
 674     bail!("decode_i64 failed - missing end marker");
 675 }
 676
 677 /// Serialize u64 as short, variable length byte sequence
 678 ///
 679 /// Stores 7 bits per byte, Bit 8 indicates the end of the sequence (when not set).
 680 pub fn catalog_encode_u64<W: Write>(writer: &mut W, v: u64) -> Result<(), Error> {
 681     let mut enc = Vec::new();
 682
 683     let mut d = v;
 684     loop {
 685         if d < 128 {
 686             enc.push(d as u8);
 687             break;
 688         }
 689         enc.push((128 | (d & 127)) as u8);
 690         d >>= 7;
 691     }
 692     writer.write_all(&enc)?;
 693
 694     Ok(())
 695 }
 696
 697 /// Deserialize u64 from variable length byte sequence
 698 ///
 699 /// We currently read maximal 10 bytes, which give a maximum of 70 bits,
 700 /// but we currently only encode up to 64 bits
 701 pub fn catalog_decode_u64<R: Read>(reader: &mut R) -> Result<u64, Error> {
 702
 703     let mut v: u64 = 0;
 704     let mut buf = [0u8];
 705
 706     for i in 0..10 { // only allow 10 bytes (70 bits)
 707         if buf.is_empty() {
 708             bail!("decode_u64 failed - unexpected EOB");
 709         }
 710         reader.read_exact(&mut buf)?;
 711         let t = buf[0];
 712         if t < 128 {
 713             v |= (t as u64) << (i*7);
 714             return Ok(v);
 715         } else {
 716             v |= ((t & 127) as u64) << (i*7);
 717         }
 718     }
 719
 720     bail!("decode_u64 failed - missing end marker");
 721 }
 722
 723 #[test]
 724 fn test_catalog_u64_encoder() {
 725
 726     fn test_encode_decode(value: u64) {
 727
 728         let mut data = Vec::new();
 729         catalog_encode_u64(&mut data, value).unwrap();
 730
 731         //println!("ENCODE {} {:?}", value, data);
 732
 733         let slice = &mut &data[..];
 734         let decoded = catalog_decode_u64(slice).unwrap();
 735
 736         //println!("DECODE {}", decoded);
 737
 738         assert!(decoded == value);
 739     }
 740
 741     test_encode_decode(u64::MIN);
 742     test_encode_decode(126);
 743     test_encode_decode((1<<12)-1);
 744     test_encode_decode((1<<20)-1);
 745     test_encode_decode((1<<50)-1);
 746     test_encode_decode(u64::MAX);
 747 }
 748
 749 #[test]
 750 fn test_catalog_i64_encoder() {
 751
 752     fn test_encode_decode(value: i64) {
 753
 754         let mut data = Vec::new();
 755         catalog_encode_i64(&mut data, value).unwrap();
 756
 757         let slice = &mut &data[..];
 758         let decoded = catalog_decode_i64(slice).unwrap();
 759
 760         assert!(decoded == value);
 761     }
 762
 763     test_encode_decode(0);
 764     test_encode_decode(-0);
 765     test_encode_decode(126);
 766     test_encode_decode(-126);
 767     test_encode_decode((1<<12)-1);
 768     test_encode_decode(-(1<<12)-1);
 769     test_encode_decode((1<<20)-1);
 770     test_encode_decode(-(1<<20)-1);
 771     test_encode_decode(i64::MIN);
 772     test_encode_decode(i64::MAX);
 773 }
 774
 775 #[test]
 776 fn test_catalog_i64_compatibility() {
 777
 778     fn test_encode_decode(value: u64) {
 779
 780         let mut data = Vec::new();
 781         catalog_encode_u64(&mut data, value).unwrap();
 782
 783         let slice = &mut &data[..];
 784         let decoded = catalog_decode_i64(slice).unwrap() as u64;
 785
 786         assert!(decoded == value);
 787     }
 788
 789     test_encode_decode(u64::MIN);
 790     test_encode_decode(126);
 791     test_encode_decode((1<<12)-1);
 792     test_encode_decode((1<<20)-1);
 793     test_encode_decode((1<<50)-1);
 794     test_encode_decode(u64::MAX);
 795 }