pbs-datastore/src/catalog.rs

   1 use std::convert::TryFrom;
   2 use std::ffi::{CStr, CString, OsStr};
   3 use std::fmt;
   4 use std::io::{Read, Write, Seek, SeekFrom};
   5 use std::os::unix::ffi::OsStrExt;
   6
   7 use anyhow::{bail, format_err, Error};
   8 use serde::{Deserialize, Serialize};
   9
  10 use pathpatterns::{MatchList, MatchType};
  11
  12 use proxmox_io::ReadExt;
  13 use proxmox_schema::api;
  14
  15 use crate::file_formats::PROXMOX_CATALOG_FILE_MAGIC_1_0;
  16
  17 /// Trait for writing file list catalogs.
  18 ///
  19 /// A file list catalog simply stores a directory tree. Such catalogs may be used as index to do a
  20 /// fast search for files.
  21 pub trait BackupCatalogWriter {
  22     fn start_directory(&mut self, name: &CStr) -> Result<(), Error>;
  23     fn end_directory(&mut self) -> Result<(), Error>;
  24     fn add_file(&mut self, name: &CStr, size: u64, mtime: i64) -> Result<(), Error>;
  25     fn add_symlink(&mut self, name: &CStr) -> Result<(), Error>;
  26     fn add_hardlink(&mut self, name: &CStr) -> Result<(), Error>;
  27     fn add_block_device(&mut self, name: &CStr) -> Result<(), Error>;
  28     fn add_char_device(&mut self, name: &CStr) -> Result<(), Error>;
  29     fn add_fifo(&mut self, name: &CStr) -> Result<(), Error>;
  30     fn add_socket(&mut self, name: &CStr) -> Result<(), Error>;
  31 }
  32
  33 #[repr(u8)]
  34 #[derive(Copy,Clone,PartialEq)]
  35 pub enum CatalogEntryType {
  36     Directory = b'd',
  37     File = b'f',
  38     Symlink = b'l',
  39     Hardlink = b'h',
  40     BlockDevice = b'b',
  41     CharDevice = b'c',
  42     Fifo = b'p', // Fifo,Pipe
  43     Socket = b's',
  44 }
  45
  46 impl TryFrom<u8> for CatalogEntryType {
  47     type Error=Error;
  48
  49     fn try_from(value: u8) -> Result<Self, Error> {
  50         Ok(match value {
  51             b'd' => CatalogEntryType::Directory,
  52             b'f' => CatalogEntryType::File,
  53             b'l' => CatalogEntryType::Symlink,
  54             b'h' => CatalogEntryType::Hardlink,
  55             b'b' => CatalogEntryType::BlockDevice,
  56             b'c' => CatalogEntryType::CharDevice,
  57             b'p' => CatalogEntryType::Fifo,
  58             b's' => CatalogEntryType::Socket,
  59             _ => bail!("invalid CatalogEntryType value '{}'", char::from(value)),
  60         })
  61     }
  62 }
  63
  64 impl From<&DirEntryAttribute> for CatalogEntryType {
  65     fn from(value: &DirEntryAttribute) -> Self {
  66         match value {
  67             DirEntryAttribute::Directory { .. } => CatalogEntryType::Directory,
  68             DirEntryAttribute::File { .. } => CatalogEntryType::File,
  69             DirEntryAttribute::Symlink => CatalogEntryType::Symlink,
  70             DirEntryAttribute::Hardlink => CatalogEntryType::Hardlink,
  71             DirEntryAttribute::BlockDevice => CatalogEntryType::BlockDevice,
  72             DirEntryAttribute::CharDevice => CatalogEntryType::CharDevice,
  73             DirEntryAttribute::Fifo => CatalogEntryType::Fifo,
  74             DirEntryAttribute::Socket => CatalogEntryType::Socket,
  75         }
  76     }
  77 }
  78
  79 impl fmt::Display for CatalogEntryType {
  80     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
  81         write!(f, "{}", char::from(*self as u8))
  82     }
  83 }
  84
  85 /// Represents a named directory entry
  86 ///
  87 /// The ``attr`` property contain the exact type with type specific
  88 /// attributes.
  89 #[derive(Clone, PartialEq)]
  90 pub struct DirEntry {
  91     pub name: Vec<u8>,
  92     pub attr: DirEntryAttribute,
  93 }
  94
  95 /// Used to specific additional attributes inside DirEntry
  96 #[derive(Clone, Debug, PartialEq)]
  97 pub enum DirEntryAttribute {
  98     Directory { start: u64 },
  99     File { size: u64, mtime: i64 },
 100     Symlink,
 101     Hardlink,
 102     BlockDevice,
 103     CharDevice,
 104     Fifo,
 105     Socket,
 106 }
 107
 108 impl DirEntry {
 109
 110     fn new(etype: CatalogEntryType, name: Vec<u8>, start: u64, size: u64, mtime: i64) -> Self {
 111         match etype {
 112             CatalogEntryType::Directory => {
 113                 DirEntry { name, attr: DirEntryAttribute::Directory { start } }
 114             }
 115             CatalogEntryType::File => {
 116                 DirEntry { name, attr: DirEntryAttribute::File { size, mtime } }
 117             }
 118             CatalogEntryType::Symlink => {
 119                 DirEntry { name, attr: DirEntryAttribute::Symlink }
 120             }
 121             CatalogEntryType::Hardlink => {
 122                 DirEntry { name, attr: DirEntryAttribute::Hardlink }
 123             }
 124             CatalogEntryType::BlockDevice => {
 125                 DirEntry { name, attr: DirEntryAttribute::BlockDevice }
 126             }
 127             CatalogEntryType::CharDevice => {
 128                 DirEntry { name, attr: DirEntryAttribute::CharDevice }
 129             }
 130             CatalogEntryType::Fifo => {
 131                 DirEntry { name, attr: DirEntryAttribute::Fifo }
 132             }
 133             CatalogEntryType::Socket => {
 134                 DirEntry { name, attr: DirEntryAttribute::Socket }
 135             }
 136         }
 137     }
 138
 139     /// Get file mode bits for this entry to be used with the `MatchList` api.
 140     pub fn get_file_mode(&self) -> Option<u32> {
 141         Some(
 142             match self.attr {
 143                 DirEntryAttribute::Directory { .. } => pxar::mode::IFDIR,
 144                 DirEntryAttribute::File { .. } => pxar::mode::IFREG,
 145                 DirEntryAttribute::Symlink => pxar::mode::IFLNK,
 146                 DirEntryAttribute::Hardlink => return None,
 147                 DirEntryAttribute::BlockDevice => pxar::mode::IFBLK,
 148                 DirEntryAttribute::CharDevice => pxar::mode::IFCHR,
 149                 DirEntryAttribute::Fifo => pxar::mode::IFIFO,
 150                 DirEntryAttribute::Socket => pxar::mode::IFSOCK,
 151             }
 152             as u32
 153         )
 154     }
 155
 156     /// Check if DirEntry is a directory
 157     pub fn is_directory(&self) -> bool {
 158         matches!(self.attr, DirEntryAttribute::Directory { .. })
 159     }
 160
 161     /// Check if DirEntry is a symlink
 162     pub fn is_symlink(&self) -> bool {
 163         matches!(self.attr, DirEntryAttribute::Symlink { .. })
 164     }
 165 }
 166
 167 struct DirInfo {
 168     name: CString,
 169     entries: Vec<DirEntry>,
 170 }
 171
 172 impl DirInfo {
 173
 174     fn new(name: CString) -> Self {
 175         DirInfo { name, entries: Vec::new() }
 176     }
 177
 178     fn new_rootdir() -> Self {
 179         DirInfo::new(CString::new(b"/".to_vec()).unwrap())
 180     }
 181
 182     fn encode_entry<W: Write>(
 183         writer: &mut W,
 184         entry: &DirEntry,
 185         pos: u64,
 186     ) -> Result<(), Error> {
 187         match entry {
 188             DirEntry { name, attr: DirEntryAttribute::Directory { start } } => {
 189                 writer.write_all(&[CatalogEntryType::Directory as u8])?;
 190                 catalog_encode_u64(writer, name.len() as u64)?;
 191                 writer.write_all(name)?;
 192                 catalog_encode_u64(writer, pos - start)?;
 193             }
 194             DirEntry { name, attr: DirEntryAttribute::File { size, mtime } } => {
 195                 writer.write_all(&[CatalogEntryType::File as u8])?;
 196                 catalog_encode_u64(writer, name.len() as u64)?;
 197                 writer.write_all(name)?;
 198                 catalog_encode_u64(writer, *size)?;
 199                 catalog_encode_i64(writer, *mtime)?;
 200             }
 201             DirEntry { name, attr: DirEntryAttribute::Symlink } => {
 202                 writer.write_all(&[CatalogEntryType::Symlink as u8])?;
 203                 catalog_encode_u64(writer, name.len() as u64)?;
 204                 writer.write_all(name)?;
 205             }
 206             DirEntry { name, attr: DirEntryAttribute::Hardlink } => {
 207                 writer.write_all(&[CatalogEntryType::Hardlink as u8])?;
 208                 catalog_encode_u64(writer, name.len() as u64)?;
 209                 writer.write_all(name)?;
 210             }
 211             DirEntry { name, attr: DirEntryAttribute::BlockDevice } => {
 212                 writer.write_all(&[CatalogEntryType::BlockDevice as u8])?;
 213                 catalog_encode_u64(writer, name.len() as u64)?;
 214                 writer.write_all(name)?;
 215             }
 216             DirEntry { name, attr: DirEntryAttribute::CharDevice } => {
 217                 writer.write_all(&[CatalogEntryType::CharDevice as u8])?;
 218                 catalog_encode_u64(writer, name.len() as u64)?;
 219                 writer.write_all(name)?;
 220             }
 221             DirEntry { name, attr: DirEntryAttribute::Fifo } => {
 222                 writer.write_all(&[CatalogEntryType::Fifo as u8])?;
 223                 catalog_encode_u64(writer, name.len() as u64)?;
 224                 writer.write_all(name)?;
 225             }
 226             DirEntry { name, attr: DirEntryAttribute::Socket } => {
 227                 writer.write_all(&[CatalogEntryType::Socket as u8])?;
 228                 catalog_encode_u64(writer, name.len() as u64)?;
 229                 writer.write_all(name)?;
 230             }
 231         }
 232         Ok(())
 233     }
 234
 235     fn encode(self, start: u64) -> Result<(CString, Vec<u8>), Error> {
 236         let mut table = Vec::new();
 237         catalog_encode_u64(&mut table, self.entries.len() as u64)?;
 238         for entry in self.entries {
 239             Self::encode_entry(&mut table, &entry, start)?;
 240         }
 241
 242         let mut data = Vec::new();
 243         catalog_encode_u64(&mut data, table.len() as u64)?;
 244         data.extend_from_slice(&table);
 245
 246         Ok((self.name, data))
 247     }
 248
 249     fn parse<C: FnMut(CatalogEntryType, &[u8], u64, u64, i64) -> Result<bool, Error>>(
 250         data: &[u8],
 251         mut callback: C,
 252     ) -> Result<(), Error> {
 253
 254         let mut cursor = data;
 255
 256         let entries = catalog_decode_u64(&mut cursor)?;
 257
 258         let mut name_buf = vec![0u8; 4096];
 259
 260         for _ in 0..entries {
 261
 262             let mut buf = [ 0u8 ];
 263             cursor.read_exact(&mut buf)?;
 264             let etype = CatalogEntryType::try_from(buf[0])?;
 265
 266             let name_len = catalog_decode_u64(&mut cursor)? as usize;
 267             if name_len >= name_buf.len() {
 268                 bail!("directory entry name too long ({} >= {})", name_len, name_buf.len());
 269             }
 270             let name = &mut name_buf[0..name_len];
 271             cursor.read_exact(name)?;
 272
 273             let cont = match etype {
 274                 CatalogEntryType::Directory => {
 275                     let offset = catalog_decode_u64(&mut cursor)?;
 276                     callback(etype, name, offset, 0, 0)?
 277                 }
 278                 CatalogEntryType::File => {
 279                     let size = catalog_decode_u64(&mut cursor)?;
 280                     let mtime = catalog_decode_i64(&mut cursor)?;
 281                     callback(etype, name, 0, size, mtime)?
 282                 }
 283                 _ => {
 284                     callback(etype, name, 0, 0, 0)?
 285                 }
 286             };
 287             if !cont {
 288                 return Ok(());
 289             }
 290         }
 291
 292         if !cursor.is_empty() {
 293             bail!("unable to parse whole catalog data block");
 294         }
 295
 296         Ok(())
 297     }
 298 }
 299
 300 /// Write small catalog files
 301 ///
 302 /// A Catalogs simply contains list of files and directories
 303 /// (directory tree). They are use to find content without having to
 304 /// search the real archive (which may be large). For files, they
 305 /// include the last modification time and file size.
 306 pub struct CatalogWriter<W> {
 307     writer: W,
 308     dirstack: Vec<DirInfo>,
 309     pos: u64,
 310 }
 311
 312 impl <W: Write> CatalogWriter<W> {
 313
 314     /// Create a new  CatalogWriter instance
 315     pub fn new(writer: W) -> Result<Self, Error> {
 316         let mut me = Self { writer, dirstack: vec![ DirInfo::new_rootdir() ], pos: 0 };
 317         me.write_all(&PROXMOX_CATALOG_FILE_MAGIC_1_0)?;
 318         Ok(me)
 319     }
 320
 321     fn write_all(&mut self, data: &[u8]) -> Result<(), Error> {
 322         self.writer.write_all(data)?;
 323         self.pos += u64::try_from(data.len())?;
 324         Ok(())
 325     }
 326
 327     /// Finish writing, flush all data
 328     ///
 329     /// This need to be called before drop.
 330     pub fn finish(&mut self) -> Result<(), Error> {
 331         if self.dirstack.len() != 1 {
 332             bail!("unable to finish catalog at level {}", self.dirstack.len());
 333         }
 334
 335         let dir = self.dirstack.pop().unwrap();
 336
 337         let start = self.pos;
 338         let (_, data) = dir.encode(start)?;
 339         self.write_all(&data)?;
 340
 341         self.write_all(&start.to_le_bytes())?;
 342
 343         self.writer.flush()?;
 344
 345         Ok(())
 346     }
 347 }
 348
 349 impl <W: Write> BackupCatalogWriter for CatalogWriter<W> {
 350
 351     fn start_directory(&mut self, name: &CStr) -> Result<(), Error> {
 352         let new = DirInfo::new(name.to_owned());
 353         self.dirstack.push(new);
 354         Ok(())
 355     }
 356
 357     fn end_directory(&mut self) -> Result<(), Error> {
 358         let (start, name) = match self.dirstack.pop() {
 359             Some(dir) => {
 360                 let start = self.pos;
 361                 let (name, data) = dir.encode(start)?;
 362                 self.write_all(&data)?;
 363                 (start, name)
 364             }
 365             None => {
 366                 bail!("got unexpected end_directory level 0");
 367             }
 368         };
 369
 370         let current = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 371         let name = name.to_bytes().to_vec();
 372         current.entries.push(DirEntry { name, attr: DirEntryAttribute::Directory { start } });
 373
 374         Ok(())
 375     }
 376
 377     fn add_file(&mut self, name: &CStr, size: u64, mtime: i64) -> Result<(), Error> {
 378         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 379         let name = name.to_bytes().to_vec();
 380         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::File { size, mtime } });
 381         Ok(())
 382     }
 383
 384     fn add_symlink(&mut self, name: &CStr) -> Result<(), Error> {
 385         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 386         let name = name.to_bytes().to_vec();
 387         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::Symlink });
 388         Ok(())
 389     }
 390
 391     fn add_hardlink(&mut self, name: &CStr) -> Result<(), Error> {
 392         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 393         let name = name.to_bytes().to_vec();
 394         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::Hardlink });
 395         Ok(())
 396     }
 397
 398     fn add_block_device(&mut self, name: &CStr) -> Result<(), Error> {
 399         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 400         let name = name.to_bytes().to_vec();
 401         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::BlockDevice });
 402         Ok(())
 403     }
 404
 405     fn add_char_device(&mut self, name: &CStr) -> Result<(), Error> {
 406         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 407         let name = name.to_bytes().to_vec();
 408         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::CharDevice });
 409         Ok(())
 410     }
 411
 412     fn add_fifo(&mut self, name: &CStr) -> Result<(), Error> {
 413         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 414         let name = name.to_bytes().to_vec();
 415         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::Fifo });
 416         Ok(())
 417     }
 418
 419     fn add_socket(&mut self, name: &CStr) -> Result<(), Error> {
 420         let dir = self.dirstack.last_mut().ok_or_else(|| format_err!("outside root"))?;
 421         let name = name.to_bytes().to_vec();
 422         dir.entries.push(DirEntry { name, attr: DirEntryAttribute::Socket });
 423         Ok(())
 424     }
 425 }
 426
 427 /// Read Catalog files
 428 pub struct CatalogReader<R> {
 429     reader: R,
 430 }
 431
 432 impl <R: Read + Seek> CatalogReader<R> {
 433
 434     /// Create a new CatalogReader instance
 435     pub fn new(reader: R) -> Self {
 436         Self { reader }
 437     }
 438
 439     /// Print whole catalog to stdout
 440     pub fn dump(&mut self) -> Result<(), Error> {
 441
 442         let root = self.root()?;
 443         match root {
 444             DirEntry { attr: DirEntryAttribute::Directory { start }, .. }=> {
 445                 self.dump_dir(std::path::Path::new("./"), start)
 446             }
 447             _ => unreachable!(),
 448         }
 449     }
 450
 451     /// Get the root DirEntry
 452     pub fn root(&mut self) ->  Result<DirEntry, Error>  {
 453         // Root dir is special
 454         self.reader.seek(SeekFrom::Start(0))?;
 455         let mut magic = [ 0u8; 8];
 456         self.reader.read_exact(&mut magic)?;
 457         if magic != PROXMOX_CATALOG_FILE_MAGIC_1_0 {
 458             bail!("got unexpected magic number for catalog");
 459         }
 460         self.reader.seek(SeekFrom::End(-8))?;
 461         let start = unsafe { self.reader.read_le_value::<u64>()? };
 462         Ok(DirEntry { name: b"".to_vec(), attr: DirEntryAttribute::Directory { start } })
 463     }
 464
 465     /// Read all directory entries
 466     pub fn read_dir(
 467         &mut self,
 468         parent: &DirEntry,
 469     ) -> Result<Vec<DirEntry>, Error>  {
 470
 471         let start = match parent.attr {
 472             DirEntryAttribute::Directory { start } => start,
 473             _ => bail!("parent is not a directory - internal error"),
 474         };
 475
 476         let data = self.read_raw_dirinfo_block(start)?;
 477
 478         let mut entry_list = Vec::new();
 479
 480         DirInfo::parse(&data, |etype, name, offset, size, mtime| {
 481             let entry = DirEntry::new(etype, name.to_vec(), start - offset, size, mtime);
 482             entry_list.push(entry);
 483             Ok(true)
 484         })?;
 485
 486         Ok(entry_list)
 487     }
 488
 489     /// Lookup a DirEntry from an absolute path
 490     pub fn lookup_recursive(
 491         &mut self,
 492         path: &[u8],
 493     ) -> Result<DirEntry, Error> {
 494         let mut current = self.root()?;
 495         if path == b"/" {
 496             return Ok(current);
 497         }
 498
 499         let components = if !path.is_empty() && path[0] == b'/' {
 500             &path[1..]
 501         } else {
 502             path
 503         }.split(|c| *c == b'/');
 504
 505         for comp in components {
 506             if let Some(entry) = self.lookup(&current, comp)? {
 507                 current = entry;
 508             } else {
 509                 bail!("path {:?} not found in catalog", String::from_utf8_lossy(&path));
 510             }
 511         }
 512         Ok(current)
 513     }
 514
 515     /// Lockup a DirEntry inside a parent directory
 516     pub fn lookup(
 517         &mut self,
 518         parent: &DirEntry,
 519         filename: &[u8],
 520     ) -> Result<Option<DirEntry>, Error>  {
 521
 522         let start = match parent.attr {
 523             DirEntryAttribute::Directory { start } => start,
 524             _ => bail!("parent is not a directory - internal error"),
 525         };
 526
 527         let data = self.read_raw_dirinfo_block(start)?;
 528
 529         let mut item = None;
 530         DirInfo::parse(&data, |etype, name, offset, size, mtime| {
 531             if name != filename {
 532                 return Ok(true);
 533             }
 534
 535             let entry = DirEntry::new(etype, name.to_vec(), start - offset, size, mtime);
 536             item = Some(entry);
 537             Ok(false) // stop parsing
 538         })?;
 539
 540         Ok(item)
 541     }
 542
 543     /// Read the raw directory info block from current reader position.
 544     fn read_raw_dirinfo_block(&mut self, start: u64) ->  Result<Vec<u8>, Error>  {
 545         self.reader.seek(SeekFrom::Start(start))?;
 546         let size = catalog_decode_u64(&mut self.reader)?;
 547         if size < 1 { bail!("got small directory size {}", size) };
 548         let data = self.reader.read_exact_allocated(size as usize)?;
 549         Ok(data)
 550     }
 551
 552     /// Print the content of a directory to stdout
 553     pub fn dump_dir(&mut self, prefix: &std::path::Path, start: u64) -> Result<(), Error> {
 554
 555         let data = self.read_raw_dirinfo_block(start)?;
 556
 557         DirInfo::parse(&data, |etype, name, offset, size, mtime| {
 558
 559             let mut path = std::path::PathBuf::from(prefix);
 560             let name: &OsStr = OsStrExt::from_bytes(name);
 561             path.push(name);
 562
 563             match etype {
 564                 CatalogEntryType::Directory => {
 565                     println!("{} {:?}", etype, path);
 566                     if offset > start {
 567                         bail!("got wrong directory offset ({} > {})", offset, start);
 568                     }
 569                     let pos = start - offset;
 570                     self.dump_dir(&path, pos)?;
 571                 }
 572                 CatalogEntryType::File => {
 573                     let mut mtime_string = mtime.to_string();
 574                     if let Ok(s) = proxmox_time::strftime_local("%FT%TZ", mtime as i64) {
 575                         mtime_string = s;
 576                     }
 577
 578                     println!(
 579                         "{} {:?} {} {}",
 580                         etype,
 581                         path,
 582                         size,
 583                         mtime_string,
 584                     );
 585                 }
 586                 _ => {
 587                     println!("{} {:?}", etype, path);
 588                 }
 589             }
 590
 591             Ok(true)
 592         })
 593     }
 594
 595     /// Finds all entries matching the given match patterns and calls the
 596     /// provided callback on them.
 597     pub fn find(
 598         &mut self,
 599         parent: &DirEntry,
 600         file_path: &mut Vec<u8>,
 601         match_list: &impl MatchList, //&[MatchEntry],
 602         callback: &mut dyn FnMut(&[u8]) -> Result<(), Error>,
 603     ) -> Result<(), Error> {
 604         let file_len = file_path.len();
 605         for e in self.read_dir(parent)?  {
 606             let is_dir = e.is_directory();
 607             file_path.truncate(file_len);
 608             if !e.name.starts_with(b"/") {
 609                 file_path.reserve(e.name.len() + 1);
 610                 file_path.push(b'/');
 611             }
 612             file_path.extend(&e.name);
 613             match match_list.matches(&file_path, e.get_file_mode()) {
 614                 Some(MatchType::Exclude) => continue,
 615                 Some(MatchType::Include) => callback(&file_path)?,
 616                 None => (),
 617             }
 618             if is_dir {
 619                 self.find(&e, file_path, match_list, callback)?;
 620             }
 621         }
 622         file_path.truncate(file_len);
 623
 624         Ok(())
 625     }
 626
 627     /// Returns the list of content of the given path
 628     pub fn list_dir_contents(&mut self, path: &[u8]) -> Result<Vec<ArchiveEntry>, Error> {
 629         let dir = self.lookup_recursive(path)?;
 630         let mut res = vec![];
 631         let mut path = path.to_vec();
 632         if !path.is_empty() && path[0] == b'/' {
 633             path.remove(0);
 634         }
 635
 636         for direntry in self.read_dir(&dir)? {
 637             let mut components = path.clone();
 638             components.push(b'/');
 639             components.extend(&direntry.name);
 640             let mut entry = ArchiveEntry::new(&components, Some(&direntry.attr));
 641             if let DirEntryAttribute::File { size, mtime } = direntry.attr {
 642                 entry.size = size.into();
 643                 entry.mtime = mtime.into();
 644             }
 645             res.push(entry);
 646         }
 647
 648         Ok(res)
 649     }
 650 }
 651
 652 /// Serialize i64 as short, variable length byte sequence
 653 ///
 654 /// Stores 7 bits per byte, Bit 8 indicates the end of the sequence (when not set).
 655 /// If the value is negative, we end with a zero byte (0x00).
 656 #[allow(clippy::neg_multiply)]
 657 pub fn catalog_encode_i64<W: Write>(writer: &mut W, v: i64) -> Result<(), Error> {
 658     let mut enc = Vec::new();
 659
 660     let mut d = if v < 0 {
 661         (-1 * (v + 1)) as u64 + 1 // also handles i64::MIN
 662     } else {
 663         v as u64
 664     };
 665
 666     loop {
 667         if d < 128 {
 668             if v < 0 {
 669                 enc.push(128 | d as u8);
 670                 enc.push(0u8);
 671             } else {
 672                 enc.push(d as u8);
 673             }
 674             break;
 675         }
 676         enc.push((128 | (d & 127)) as u8);
 677         d >>= 7;
 678     }
 679     writer.write_all(&enc)?;
 680
 681     Ok(())
 682 }
 683
 684 /// Deserialize i64 from variable length byte sequence
 685 ///
 686 /// We currently read maximal 11 bytes, which give a maximum of 70 bits + sign.
 687 /// this method is compatible with catalog_encode_u64 iff the
 688 /// value encoded is <= 2^63 (values > 2^63 cannot be represented in an i64)
 689 #[allow(clippy::neg_multiply)]
 690 pub fn catalog_decode_i64<R: Read>(reader: &mut R) -> Result<i64, Error> {
 691
 692     let mut v: u64 = 0;
 693     let mut buf = [0u8];
 694
 695     for i in 0..11 { // only allow 11 bytes (70 bits + sign marker)
 696         if buf.is_empty() {
 697             bail!("decode_i64 failed - unexpected EOB");
 698         }
 699         reader.read_exact(&mut buf)?;
 700
 701         let t = buf[0];
 702
 703         if t == 0 {
 704             if v == 0 {
 705                 return Ok(0);
 706             }
 707             return Ok(((v - 1) as i64 * -1) - 1); // also handles i64::MIN
 708         } else if t < 128 {
 709             v |= (t as u64) << (i*7);
 710             return Ok(v as i64);
 711         } else {
 712             v |= ((t & 127) as u64) << (i*7);
 713         }
 714     }
 715
 716     bail!("decode_i64 failed - missing end marker");
 717 }
 718
 719 /// Serialize u64 as short, variable length byte sequence
 720 ///
 721 /// Stores 7 bits per byte, Bit 8 indicates the end of the sequence (when not set).
 722 pub fn catalog_encode_u64<W: Write>(writer: &mut W, v: u64) -> Result<(), Error> {
 723     let mut enc = Vec::new();
 724
 725     let mut d = v;
 726     loop {
 727         if d < 128 {
 728             enc.push(d as u8);
 729             break;
 730         }
 731         enc.push((128 | (d & 127)) as u8);
 732         d >>= 7;
 733     }
 734     writer.write_all(&enc)?;
 735
 736     Ok(())
 737 }
 738
 739 /// Deserialize u64 from variable length byte sequence
 740 ///
 741 /// We currently read maximal 10 bytes, which give a maximum of 70 bits,
 742 /// but we currently only encode up to 64 bits
 743 pub fn catalog_decode_u64<R: Read>(reader: &mut R) -> Result<u64, Error> {
 744
 745     let mut v: u64 = 0;
 746     let mut buf = [0u8];
 747
 748     for i in 0..10 { // only allow 10 bytes (70 bits)
 749         if buf.is_empty() {
 750             bail!("decode_u64 failed - unexpected EOB");
 751         }
 752         reader.read_exact(&mut buf)?;
 753         let t = buf[0];
 754         if t < 128 {
 755             v |= (t as u64) << (i*7);
 756             return Ok(v);
 757         } else {
 758             v |= ((t & 127) as u64) << (i*7);
 759         }
 760     }
 761
 762     bail!("decode_u64 failed - missing end marker");
 763 }
 764
 765 #[test]
 766 fn test_catalog_u64_encoder() {
 767
 768     fn test_encode_decode(value: u64) {
 769
 770         let mut data = Vec::new();
 771         catalog_encode_u64(&mut data, value).unwrap();
 772
 773         //println!("ENCODE {} {:?}", value, data);
 774
 775         let slice = &mut &data[..];
 776         let decoded = catalog_decode_u64(slice).unwrap();
 777
 778         //println!("DECODE {}", decoded);
 779
 780         assert!(decoded == value);
 781     }
 782
 783     test_encode_decode(u64::MIN);
 784     test_encode_decode(126);
 785     test_encode_decode((1<<12)-1);
 786     test_encode_decode((1<<20)-1);
 787     test_encode_decode((1<<50)-1);
 788     test_encode_decode(u64::MAX);
 789 }
 790
 791 #[test]
 792 fn test_catalog_i64_encoder() {
 793
 794     fn test_encode_decode(value: i64) {
 795
 796         let mut data = Vec::new();
 797         catalog_encode_i64(&mut data, value).unwrap();
 798
 799         let slice = &mut &data[..];
 800         let decoded = catalog_decode_i64(slice).unwrap();
 801
 802         assert!(decoded == value);
 803     }
 804
 805     test_encode_decode(0);
 806     test_encode_decode(-0);
 807     test_encode_decode(126);
 808     test_encode_decode(-126);
 809     test_encode_decode((1<<12)-1);
 810     test_encode_decode(-(1<<12)-1);
 811     test_encode_decode((1<<20)-1);
 812     test_encode_decode(-(1<<20)-1);
 813     test_encode_decode(i64::MIN);
 814     test_encode_decode(i64::MAX);
 815 }
 816
 817 #[test]
 818 fn test_catalog_i64_compatibility() {
 819
 820     fn test_encode_decode(value: u64) {
 821
 822         let mut data = Vec::new();
 823         catalog_encode_u64(&mut data, value).unwrap();
 824
 825         let slice = &mut &data[..];
 826         let decoded = catalog_decode_i64(slice).unwrap() as u64;
 827
 828         assert!(decoded == value);
 829     }
 830
 831     test_encode_decode(u64::MIN);
 832     test_encode_decode(126);
 833     test_encode_decode((1<<12)-1);
 834     test_encode_decode((1<<20)-1);
 835     test_encode_decode((1<<50)-1);
 836     test_encode_decode(u64::MAX);
 837 }
 838
 839 /// An entry in a hierarchy of files for restore and listing.
 840 #[api]
 841 #[derive(Serialize, Deserialize)]
 842 pub struct ArchiveEntry {
 843     /// Base64-encoded full path to the file, including the filename
 844     pub filepath: String,
 845     /// Displayable filename text for UIs
 846     pub text: String,
 847     /// File or directory type of this entry
 848     #[serde(rename = "type")]
 849     pub entry_type: String,
 850     /// Is this entry a leaf node, or does it have children (i.e. a directory)?
 851     pub leaf: bool,
 852     /// The file size, if entry_type is 'f' (file)
 853     #[serde(skip_serializing_if="Option::is_none")]
 854     pub size: Option<u64>,
 855     /// The file "last modified" time stamp, if entry_type is 'f' (file)
 856     #[serde(skip_serializing_if="Option::is_none")]
 857     pub mtime: Option<i64>,
 858 }
 859
 860 impl ArchiveEntry {
 861     pub fn new(filepath: &[u8], entry_type: Option<&DirEntryAttribute>) -> Self {
 862         let size = match entry_type {
 863             Some(DirEntryAttribute::File { size, .. }) => Some(*size),
 864             _ => None,
 865         };
 866         Self::new_with_size(filepath, entry_type, size)
 867     }
 868
 869     pub fn new_with_size(
 870         filepath: &[u8],
 871         entry_type: Option<&DirEntryAttribute>,
 872         size: Option<u64>,
 873     ) -> Self {
 874         Self {
 875             filepath: base64::encode(filepath),
 876             text: String::from_utf8_lossy(filepath.split(|x| *x == b'/').last().unwrap())
 877                 .to_string(),
 878             entry_type: match entry_type {
 879                 Some(entry_type) => CatalogEntryType::from(entry_type).to_string(),
 880                 None => "v".to_owned(),
 881             },
 882             leaf: !matches!(entry_type, None | Some(DirEntryAttribute::Directory { .. })),
 883             size,
 884             mtime: match entry_type {
 885                 Some(DirEntryAttribute::File { mtime, .. }) => Some(*mtime),
 886                 _ => None,
 887             },
 888         }
 889     }
 890 }