src/pxar/encoder.rs

   1 //! *pxar* format encoder.
   2 //!
   3 //! This module contain the code to generate *pxar* archive files.
   4 use std::collections::{HashMap, HashSet};
   5 use std::ffi::{CStr, CString};
   6 use std::io::Write;
   7 use std::os::unix::ffi::OsStrExt;
   8 use std::os::unix::io::AsRawFd;
   9 use std::os::unix::io::RawFd;
  10 use std::path::{Path, PathBuf};
  11
  12 use endian_trait::Endian;
  13 use failure::*;
  14 use nix::errno::Errno;
  15 use nix::fcntl::OFlag;
  16 use nix::sys::stat::FileStat;
  17 use nix::sys::stat::Mode;
  18 use nix::NixPath;
  19
  20 use proxmox::tools::vec;
  21
  22 use super::binary_search_tree::*;
  23 use super::catalog::BackupCatalogWriter;
  24 use super::flags;
  25 use super::format_definition::*;
  26 use super::helper::*;
  27 use super::match_pattern::{MatchPattern, MatchPatternSlice, MatchType};
  28 use crate::tools::acl;
  29 use crate::tools::fs;
  30 use crate::tools::xattr;
  31
  32 /// The format requires to build sorted directory lookup tables in
  33 /// memory, so we restrict the number of allowed entries to limit
  34 /// maximum memory usage.
  35 pub const MAX_DIRECTORY_ENTRIES: usize = 256 * 1024;
  36
  37 #[derive(Eq, PartialEq, Hash)]
  38 struct HardLinkInfo {
  39     st_dev: u64,
  40     st_ino: u64,
  41 }
  42
  43 pub struct Encoder<'a, W: Write, C: BackupCatalogWriter> {
  44     base_path: PathBuf,
  45     relative_path: PathBuf,
  46     writer: &'a mut W,
  47     writer_pos: usize,
  48     catalog: Option<&'a mut C>,
  49     _size: usize,
  50     file_copy_buffer: Vec<u8>,
  51     device_set: Option<HashSet<u64>>,
  52     verbose: bool,
  53     // Flags set by the user
  54     feature_flags: u64,
  55     // Flags signaling features supported by the filesystem
  56     fs_feature_flags: u64,
  57     hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
  58 }
  59
  60 impl<'a, W: Write, C: BackupCatalogWriter> Encoder<'a, W, C> {
  61     // used for error reporting
  62     fn full_path(&self) -> PathBuf {
  63         self.base_path.join(&self.relative_path)
  64     }
  65
  66     /// Create archive, write result data to ``writer``.
  67     ///
  68     /// The ``device_set`` can be use used to limit included mount points.
  69     ///
  70     /// - ``None``: include all mount points
  71     /// - ``Some(set)``: only include devices listed in this set (the
  72     ///   root path device is automathically added to this list, so
  73     ///   you can pass an empty set if you want to archive a single
  74     ///   mount point.)
  75     pub fn encode(
  76         path: PathBuf,
  77         dir: &mut nix::dir::Dir,
  78         writer: &'a mut W,
  79         catalog: Option<&'a mut C>,
  80         device_set: Option<HashSet<u64>>,
  81         verbose: bool,
  82         skip_lost_and_found: bool, // fixme: should be a feature flag ??
  83         feature_flags: u64,
  84         mut excludes: Vec<MatchPattern>,
  85     ) -> Result<(), Error> {
  86         const FILE_COPY_BUFFER_SIZE: usize = 1024 * 1024;
  87
  88         let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
  89         unsafe {
  90             file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE);
  91         }
  92
  93         // todo: use scandirat??
  94
  95         let dir_fd = dir.as_raw_fd();
  96         let stat = nix::sys::stat::fstat(dir_fd)
  97             .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
  98
  99         if !is_directory(&stat) {
 100             bail!("got unexpected file type {:?} (not a directory)", path);
 101         }
 102
 103         let mut device_set = device_set.clone();
 104         if let Some(ref mut set) = device_set {
 105             set.insert(stat.st_dev);
 106         }
 107
 108         let magic = detect_fs_type(dir_fd)?;
 109
 110         if is_virtual_file_system(magic) {
 111             bail!("backup virtual file systems is disabled!");
 112         }
 113
 114         let fs_feature_flags = flags::feature_flags_from_magic(magic);
 115
 116         let mut me = Self {
 117             base_path: path,
 118             relative_path: PathBuf::new(),
 119             writer,
 120             writer_pos: 0,
 121             catalog,
 122             _size: 0,
 123             file_copy_buffer,
 124             device_set,
 125             verbose,
 126             feature_flags,
 127             fs_feature_flags,
 128             hardlinks: HashMap::new(),
 129         };
 130
 131         if verbose {
 132             println!("{:?}", me.full_path());
 133         }
 134
 135         if skip_lost_and_found {
 136             excludes.push(MatchPattern::from_line(b"**/lost+found").unwrap().unwrap());
 137         }
 138         let mut exclude_slices = Vec::new();
 139         for excl in &excludes {
 140             exclude_slices.push(excl.as_slice());
 141         }
 142
 143         me.encode_dir(dir, &stat, magic, exclude_slices)?;
 144
 145         Ok(())
 146     }
 147
 148     fn write(&mut self, buf: &[u8]) -> Result<(), Error> {
 149         self.writer.write_all(buf)?;
 150         self.writer_pos += buf.len();
 151         Ok(())
 152     }
 153
 154     fn write_item<T: Endian>(&mut self, item: T) -> Result<(), Error> {
 155         let data = item.to_le();
 156
 157         let buffer = unsafe {
 158             std::slice::from_raw_parts(&data as *const T as *const u8, std::mem::size_of::<T>())
 159         };
 160
 161         self.write(buffer)?;
 162
 163         Ok(())
 164     }
 165
 166     fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
 167         self.writer.write_all(&self.file_copy_buffer[..size])?;
 168         self.writer_pos += size;
 169         Ok(())
 170     }
 171
 172     fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
 173         let size = size + (std::mem::size_of::<PxarHeader>() as u64);
 174         self.write_item(PxarHeader { size, htype })?;
 175
 176         Ok(())
 177     }
 178
 179     fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
 180         let buffer = name.to_bytes_with_nul();
 181         self.write_header(PXAR_FILENAME, buffer.len() as u64)?;
 182         self.write(buffer)?;
 183
 184         Ok(())
 185     }
 186
 187     fn create_entry(&self, stat: &FileStat) -> Result<PxarEntry, Error> {
 188         let mode = if is_symlink(&stat) {
 189             (libc::S_IFLNK | 0o777) as u64
 190         } else {
 191             (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
 192         };
 193
 194         let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
 195         if mtime < 0 {
 196             bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
 197         }
 198
 199         let entry = PxarEntry {
 200             mode,
 201             flags: 0,
 202             uid: stat.st_uid,
 203             gid: stat.st_gid,
 204             mtime: mtime as u64,
 205         };
 206
 207         Ok(entry)
 208     }
 209
 210     fn read_chattr(&self, fd: RawFd, entry: &mut PxarEntry) -> Result<(), Error> {
 211         let mut attr: usize = 0;
 212
 213         let res = unsafe { fs::read_attr_fd(fd, &mut attr) };
 214         if let Err(err) = res {
 215             if let nix::Error::Sys(errno) = err {
 216                 if errno_is_unsupported(errno) {
 217                     return Ok(());
 218                 };
 219             }
 220             bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
 221         }
 222
 223         let flags = flags::feature_flags_from_chattr(attr as u32);
 224         entry.flags |= flags;
 225
 226         Ok(())
 227     }
 228
 229     fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut PxarEntry) -> Result<(), Error> {
 230         use proxmox::sys::linux::magic::*;
 231
 232         if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC {
 233             return Ok(());
 234         }
 235
 236         let mut attr: u32 = 0;
 237
 238         let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr) };
 239         if let Err(err) = res {
 240             if let nix::Error::Sys(errno) = err {
 241                 if errno_is_unsupported(errno) {
 242                     return Ok(());
 243                 };
 244             }
 245             bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
 246         }
 247
 248         let flags = flags::feature_flags_from_fat_attr(attr);
 249         entry.flags |= flags;
 250
 251         Ok(())
 252     }
 253
 254     /// True if all of the given feature flags are set in the Encoder, false otherwise
 255     fn has_features(&self, feature_flags: u64) -> bool {
 256         (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
 257     }
 258
 259     /// True if at least one of the given feature flags is set in the Encoder, false otherwise
 260     fn has_some_features(&self, feature_flags: u64) -> bool {
 261         (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
 262     }
 263
 264     fn read_xattrs(
 265         &self,
 266         fd: RawFd,
 267         stat: &FileStat,
 268     ) -> Result<(Vec<PxarXAttr>, Option<PxarFCaps>), Error> {
 269         let mut xattrs = Vec::new();
 270         let mut fcaps = None;
 271
 272         let flags = flags::WITH_XATTRS | flags::WITH_FCAPS;
 273         if !self.has_some_features(flags) {
 274             return Ok((xattrs, fcaps));
 275         }
 276         // Should never be called on symlinks, just in case check anyway
 277         if is_symlink(&stat) {
 278             return Ok((xattrs, fcaps));
 279         }
 280
 281         let xattr_names = match xattr::flistxattr(fd) {
 282             Ok(names) => names,
 283             // Do not bail if the underlying endpoint does not supports xattrs
 284             Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
 285             // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
 286             Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
 287             Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 288         };
 289
 290         for name in xattr_names.split(|c| *c == b'\0') {
 291             // Only extract the relevant extended attributes
 292             if !xattr::is_valid_xattr_name(&name) {
 293                 continue;
 294             }
 295
 296             let value = match xattr::fgetxattr(fd, name) {
 297                 Ok(value) => value,
 298                 // Vanished between flistattr and getxattr, this is ok, silently ignore
 299                 Err(Errno::ENODATA) => continue,
 300                 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 301             };
 302
 303             if xattr::is_security_capability(&name) {
 304                 if self.has_features(flags::WITH_FCAPS) {
 305                     // fcaps are stored in own format within the archive
 306                     fcaps = Some(PxarFCaps { data: value });
 307                 }
 308             } else if self.has_features(flags::WITH_XATTRS) {
 309                 xattrs.push(PxarXAttr {
 310                     name: name.to_vec(),
 311                     value,
 312                 });
 313             }
 314         }
 315         xattrs.sort();
 316
 317         Ok((xattrs, fcaps))
 318     }
 319
 320     fn read_acl(
 321         &self,
 322         fd: RawFd,
 323         stat: &FileStat,
 324         acl_type: acl::ACLType,
 325     ) -> Result<PxarACL, Error> {
 326         let ret = PxarACL {
 327             users: Vec::new(),
 328             groups: Vec::new(),
 329             group_obj: None,
 330             default: None,
 331         };
 332
 333         if !self.has_features(flags::WITH_ACL) {
 334             return Ok(ret);
 335         }
 336         if is_symlink(&stat) {
 337             return Ok(ret);
 338         }
 339         if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
 340             bail!("ACL_TYPE_DEFAULT only defined for directories.");
 341         }
 342
 343         // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
 344         // to create a path for acl_get_file(). acl_get_fd() only allows to get
 345         // ACL_TYPE_ACCESS attributes.
 346         let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
 347         let acl = match acl::ACL::get_file(&proc_path, acl_type) {
 348             Ok(acl) => acl,
 349             // Don't bail if underlying endpoint does not support acls
 350             Err(Errno::EOPNOTSUPP) => return Ok(ret),
 351             // Don't bail if the endpoint cannot carry acls
 352             Err(Errno::EBADF) => return Ok(ret),
 353             // Don't bail if there is no data
 354             Err(Errno::ENODATA) => return Ok(ret),
 355             Err(err) => bail!("error while reading ACL - {}", err),
 356         };
 357
 358         self.process_acl(acl, acl_type)
 359     }
 360
 361     fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 362         let mut acl_user = Vec::new();
 363         let mut acl_group = Vec::new();
 364         let mut acl_group_obj = None;
 365         let mut acl_default = None;
 366         let mut user_obj_permissions = None;
 367         let mut group_obj_permissions = None;
 368         let mut other_permissions = None;
 369         let mut mask_permissions = None;
 370
 371         for entry in &mut acl.entries() {
 372             let tag = entry.get_tag_type()?;
 373             let permissions = entry.get_permissions()?;
 374             match tag {
 375                 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
 376                 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
 377                 acl::ACL_OTHER => other_permissions = Some(permissions),
 378                 acl::ACL_MASK => mask_permissions = Some(permissions),
 379                 acl::ACL_USER => {
 380                     acl_user.push(PxarACLUser {
 381                         uid: entry.get_qualifier()?,
 382                         permissions,
 383                     });
 384                 }
 385                 acl::ACL_GROUP => {
 386                     acl_group.push(PxarACLGroup {
 387                         gid: entry.get_qualifier()?,
 388                         permissions,
 389                     });
 390                 }
 391                 _ => bail!("Unexpected ACL tag encountered!"),
 392             }
 393         }
 394
 395         acl_user.sort();
 396         acl_group.sort();
 397
 398         match acl_type {
 399             acl::ACL_TYPE_ACCESS => {
 400                 // The mask permissions are mapped to the stat group permissions
 401                 // in case that the ACL group permissions were set.
 402                 // Only in that case we need to store the group permissions,
 403                 // in the other cases they are identical to the stat group permissions.
 404                 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
 405                     acl_group_obj = Some(PxarACLGroupObj { permissions: gop });
 406                 }
 407             }
 408             acl::ACL_TYPE_DEFAULT => {
 409                 if user_obj_permissions != None
 410                     || group_obj_permissions != None
 411                     || other_permissions != None
 412                     || mask_permissions != None
 413                 {
 414                     acl_default = Some(PxarACLDefault {
 415                         // The value is set to UINT64_MAX as placeholder if one
 416                         // of the permissions is not set
 417                         user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
 418                         group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
 419                         other_permissions: other_permissions.unwrap_or(std::u64::MAX),
 420                         mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
 421                     });
 422                 }
 423             }
 424             _ => bail!("Unexpected ACL type encountered"),
 425         }
 426
 427         Ok(PxarACL {
 428             users: acl_user,
 429             groups: acl_group,
 430             group_obj: acl_group_obj,
 431             default: acl_default,
 432         })
 433     }
 434
 435     /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
 436     fn read_quota_project_id(
 437         &self,
 438         fd: RawFd,
 439         magic: i64,
 440         stat: &FileStat,
 441     ) -> Result<Option<PxarQuotaProjID>, Error> {
 442         if !(is_directory(&stat) || is_reg_file(&stat)) {
 443             return Ok(None);
 444         }
 445         if !self.has_features(flags::WITH_QUOTA_PROJID) {
 446             return Ok(None);
 447         }
 448
 449         use proxmox::sys::linux::magic::*;
 450
 451         match magic {
 452             EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC | ZFS_SUPER_MAGIC => {
 453                 let mut fsxattr = fs::FSXAttr::default();
 454                 let res = unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) };
 455
 456                 // On some FUSE filesystems it can happen that ioctl is not supported.
 457                 // For these cases projid is set to 0 while the error is ignored.
 458                 if let Err(err) = res {
 459                     let errno = err.as_errno().ok_or_else(|| {
 460                         format_err!(
 461                             "error while reading quota project id for {:#?}",
 462                             self.full_path()
 463                         )
 464                     })?;
 465                     if errno_is_unsupported(errno) {
 466                         return Ok(None);
 467                     } else {
 468                         bail!(
 469                             "error while reading quota project id for {:#?} - {}",
 470                             self.full_path(),
 471                             errno
 472                         );
 473                     }
 474                 }
 475
 476                 let projid = fsxattr.fsx_projid as u64;
 477                 if projid == 0 {
 478                     Ok(None)
 479                 } else {
 480                     Ok(Some(PxarQuotaProjID { projid }))
 481                 }
 482             }
 483             _ => Ok(None),
 484         }
 485     }
 486
 487     fn write_entry(&mut self, entry: PxarEntry) -> Result<(), Error> {
 488         self.write_header(PXAR_ENTRY, std::mem::size_of::<PxarEntry>() as u64)?;
 489         self.write_item(entry)?;
 490
 491         Ok(())
 492     }
 493
 494     fn write_xattr(&mut self, xattr: PxarXAttr) -> Result<(), Error> {
 495         let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
 496         self.write_header(PXAR_XATTR, size as u64)?;
 497         self.write(xattr.name.as_slice())?;
 498         self.write(&[0])?;
 499         self.write(xattr.value.as_slice())?;
 500
 501         Ok(())
 502     }
 503
 504     fn write_fcaps(&mut self, fcaps: Option<PxarFCaps>) -> Result<(), Error> {
 505         if let Some(fcaps) = fcaps {
 506             let size = fcaps.data.len();
 507             self.write_header(PXAR_FCAPS, size as u64)?;
 508             self.write(fcaps.data.as_slice())?;
 509         }
 510
 511         Ok(())
 512     }
 513
 514     fn write_acl_user(&mut self, acl_user: PxarACLUser) -> Result<(), Error> {
 515         self.write_header(PXAR_ACL_USER, std::mem::size_of::<PxarACLUser>() as u64)?;
 516         self.write_item(acl_user)?;
 517
 518         Ok(())
 519     }
 520
 521     fn write_acl_group(&mut self, acl_group: PxarACLGroup) -> Result<(), Error> {
 522         self.write_header(PXAR_ACL_GROUP, std::mem::size_of::<PxarACLGroup>() as u64)?;
 523         self.write_item(acl_group)?;
 524
 525         Ok(())
 526     }
 527
 528     fn write_acl_group_obj(&mut self, acl_group_obj: PxarACLGroupObj) -> Result<(), Error> {
 529         self.write_header(
 530             PXAR_ACL_GROUP_OBJ,
 531             std::mem::size_of::<PxarACLGroupObj>() as u64,
 532         )?;
 533         self.write_item(acl_group_obj)?;
 534
 535         Ok(())
 536     }
 537
 538     fn write_acl_default(&mut self, acl_default: PxarACLDefault) -> Result<(), Error> {
 539         self.write_header(
 540             PXAR_ACL_DEFAULT,
 541             std::mem::size_of::<PxarACLDefault>() as u64,
 542         )?;
 543         self.write_item(acl_default)?;
 544
 545         Ok(())
 546     }
 547
 548     fn write_acl_default_user(&mut self, acl_default_user: PxarACLUser) -> Result<(), Error> {
 549         self.write_header(
 550             PXAR_ACL_DEFAULT_USER,
 551             std::mem::size_of::<PxarACLUser>() as u64,
 552         )?;
 553         self.write_item(acl_default_user)?;
 554
 555         Ok(())
 556     }
 557
 558     fn write_acl_default_group(&mut self, acl_default_group: PxarACLGroup) -> Result<(), Error> {
 559         self.write_header(
 560             PXAR_ACL_DEFAULT_GROUP,
 561             std::mem::size_of::<PxarACLGroup>() as u64,
 562         )?;
 563         self.write_item(acl_default_group)?;
 564
 565         Ok(())
 566     }
 567
 568     fn write_quota_project_id(&mut self, projid: PxarQuotaProjID) -> Result<(), Error> {
 569         self.write_header(
 570             PXAR_QUOTA_PROJID,
 571             std::mem::size_of::<PxarQuotaProjID>() as u64,
 572         )?;
 573         self.write_item(projid)?;
 574
 575         Ok(())
 576     }
 577
 578     fn write_goodbye_table(
 579         &mut self,
 580         goodbye_offset: usize,
 581         goodbye_items: &mut [PxarGoodbyeItem],
 582     ) -> Result<(), Error> {
 583         goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
 584
 585         let item_count = goodbye_items.len();
 586
 587         let goodbye_table_size = (item_count + 1) * std::mem::size_of::<PxarGoodbyeItem>();
 588
 589         self.write_header(PXAR_GOODBYE, goodbye_table_size as u64)?;
 590
 591         if self.file_copy_buffer.len() < goodbye_table_size {
 592             let need = goodbye_table_size - self.file_copy_buffer.len();
 593             self.file_copy_buffer.reserve(need);
 594             unsafe {
 595                 self.file_copy_buffer
 596                     .set_len(self.file_copy_buffer.capacity());
 597             }
 598         }
 599
 600         let buffer = &mut self.file_copy_buffer;
 601
 602         copy_binary_search_tree(item_count, |s, d| {
 603             let item = &goodbye_items[s];
 604             let offset = d * std::mem::size_of::<PxarGoodbyeItem>();
 605             let dest =
 606                 crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
 607             dest.offset = u64::to_le(item.offset);
 608             dest.size = u64::to_le(item.size);
 609             dest.hash = u64::to_le(item.hash);
 610         });
 611
 612         // append PxarGoodbyeTail as last item
 613         let offset = item_count * std::mem::size_of::<PxarGoodbyeItem>();
 614         let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
 615         dest.offset = u64::to_le(goodbye_offset as u64);
 616         dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<PxarHeader>()) as u64);
 617         dest.hash = u64::to_le(PXAR_GOODBYE_TAIL_MARKER);
 618
 619         self.flush_copy_buffer(goodbye_table_size)?;
 620
 621         Ok(())
 622     }
 623
 624     fn encode_dir(
 625         &mut self,
 626         dir: &mut nix::dir::Dir,
 627         dir_stat: &FileStat,
 628         magic: i64,
 629         match_pattern: Vec<MatchPatternSlice>,
 630     ) -> Result<(), Error> {
 631         //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
 632
 633         let mut name_list = Vec::new();
 634
 635         let rawfd = dir.as_raw_fd();
 636
 637         let dir_start_pos = self.writer_pos;
 638
 639         let is_root = dir_start_pos == 0;
 640
 641         let mut dir_entry = self.create_entry(&dir_stat)?;
 642
 643         self.read_chattr(rawfd, &mut dir_entry)?;
 644         self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
 645
 646         // for each node in the directory tree, the filesystem features are
 647         // checked based on the fs magic number.
 648         self.fs_feature_flags = flags::feature_flags_from_magic(magic);
 649
 650         let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
 651         let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
 652         let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
 653         let projid = self.read_quota_project_id(rawfd, magic, &dir_stat)?;
 654
 655         self.write_entry(dir_entry)?;
 656         for xattr in xattrs {
 657             self.write_xattr(xattr)?;
 658         }
 659         self.write_fcaps(fcaps)?;
 660
 661         for user in acl_access.users {
 662             self.write_acl_user(user)?;
 663         }
 664         for group in acl_access.groups {
 665             self.write_acl_group(group)?;
 666         }
 667         if let Some(group_obj) = acl_access.group_obj {
 668             self.write_acl_group_obj(group_obj)?;
 669         }
 670
 671         for default_user in acl_default.users {
 672             self.write_acl_default_user(default_user)?;
 673         }
 674         for default_group in acl_default.groups {
 675             self.write_acl_default_group(default_group)?;
 676         }
 677         if let Some(default) = acl_default.default {
 678             self.write_acl_default(default)?;
 679         }
 680         if let Some(projid) = projid {
 681             self.write_quota_project_id(projid)?;
 682         }
 683
 684         let include_children;
 685         if is_virtual_file_system(magic) {
 686             include_children = false;
 687         } else if let Some(set) = &self.device_set {
 688             include_children = set.contains(&dir_stat.st_dev);
 689         } else {
 690             include_children = true;
 691         }
 692
 693         // Expand the exclude match pattern inherited from the parent by local entries, if present
 694         let mut local_match_pattern = match_pattern.clone();
 695         let (pxar_exclude, excludes) = match MatchPattern::from_file(rawfd, ".pxarexclude") {
 696             Ok(Some((excludes, buffer, stat))) => {
 697                 (Some((buffer, stat)), excludes)
 698             }
 699             Ok(None) => (None, Vec::new()),
 700             Err(err) => bail!("error while reading exclude file - {}", err),
 701         };
 702         for excl in &excludes {
 703             local_match_pattern.push(excl.as_slice());
 704         }
 705
 706         if include_children {
 707             // Exclude patterns passed via the CLI are stored as '.pxarexclude-cli'
 708             // in the root directory of the archive.
 709             if is_root && !match_pattern.is_empty() {
 710                 let filename = CString::new(".pxarexclude-cli")?;
 711                 name_list.push((filename, *dir_stat, match_pattern.clone()));
 712             }
 713
 714             for entry in dir.iter() {
 715                 let entry = entry
 716                     .map_err(|err| format_err!("readir {:?} failed - {}", self.full_path(), err))?;
 717                 let filename = entry.file_name().to_owned();
 718
 719                 let name = filename.to_bytes_with_nul();
 720                 if name == b".\0" || name == b"..\0" {
 721                     continue;
 722                 }
 723                 // Do not store a ".pxarexclude-cli" file found in the archive root,
 724                 // as this would confilict with new cli passed exclude patterns,
 725                 // if present.
 726                 if is_root && name == b".pxarexclude-cli\0" {
 727                     eprintln!("skip existing '.pxarexclude-cli' in archive root.");
 728                     continue;
 729                 }
 730
 731                 let stat = match nix::sys::stat::fstatat(
 732                     rawfd,
 733                     filename.as_ref(),
 734                     nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW,
 735                 ) {
 736                     Ok(stat) => stat,
 737                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 738                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 739                         self.report_vanished_file(&self.full_path().join(filename_osstr))?;
 740                         continue;
 741                     }
 742                     Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
 743                 };
 744
 745                 match MatchPatternSlice::match_filename_exclude(
 746                     &filename,
 747                     is_directory(&stat),
 748                     &local_match_pattern,
 749                 )? {
 750                     (MatchType::Positive, _) => {
 751                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 752                         eprintln!(
 753                             "matched by .pxarexclude entry - skipping: {:?}",
 754                             self.full_path().join(filename_osstr)
 755                         );
 756                     }
 757                     (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
 758                 }
 759
 760                 if name_list.len() > MAX_DIRECTORY_ENTRIES {
 761                     bail!(
 762                         "too many directory items in {:?} (> {})",
 763                         self.full_path(),
 764                         MAX_DIRECTORY_ENTRIES
 765                     );
 766                 }
 767             }
 768         } else {
 769             eprintln!("skip mount point: {:?}", self.full_path());
 770         }
 771
 772         name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
 773
 774         let mut goodbye_items = Vec::with_capacity(name_list.len());
 775
 776         for (filename, stat, exclude_list) in name_list {
 777             let start_pos = self.writer_pos;
 778
 779             if filename.as_bytes() == b".pxarexclude" {
 780                 if let Some((ref content, ref stat)) = pxar_exclude {
 781                     let filefd = match nix::fcntl::openat(
 782                         rawfd,
 783                         filename.as_ref(),
 784                         OFlag::O_NOFOLLOW,
 785                         Mode::empty(),
 786                     ) {
 787                         Ok(filefd) => filefd,
 788                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 789                             self.report_vanished_file(&self.full_path())?;
 790                             continue;
 791                         }
 792                         Err(err) => {
 793                             let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 794                             bail!(
 795                                 "open file {:?} failed - {}",
 796                                 self.full_path().join(filename_osstr),
 797                                 err
 798                             );
 799                         }
 800                     };
 801
 802                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 803                         detect_fs_type(filefd)?
 804                     } else {
 805                         magic
 806                     };
 807
 808                     self.write_filename(&filename)?;
 809                     if let Some(ref mut catalog) = self.catalog {
 810                         catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
 811                     }
 812                     self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
 813                     continue;
 814                 }
 815             }
 816
 817             if is_root && filename.as_bytes() == b".pxarexclude-cli" {
 818                 // '.pxarexclude-cli' is used to store the exclude MatchPatterns
 819                 // passed via the cli in the root directory of the archive.
 820                 self.write_filename(&filename)?;
 821                 let content = MatchPatternSlice::to_bytes(&exclude_list);
 822                 if let Some(ref mut catalog) = self.catalog {
 823                     catalog.add_file(&filename, content.len() as u64, 0)?;
 824                 }
 825                 self.encode_pxar_exclude_cli(stat.st_uid, stat.st_gid, 0, &content)?;
 826                 continue;
 827             }
 828
 829             self.relative_path
 830                 .push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
 831
 832             if self.verbose {
 833                 println!("{:?}", self.full_path());
 834             }
 835
 836             if is_directory(&stat) {
 837                 let mut dir = match nix::dir::Dir::openat(
 838                     rawfd,
 839                     filename.as_ref(),
 840                     OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW,
 841                     Mode::empty(),
 842                 ) {
 843                     Ok(dir) => dir,
 844                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 845                         self.report_vanished_file(&self.full_path())?;
 846                         self.relative_path.pop();
 847                         continue;
 848                     }
 849                     Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
 850                 };
 851
 852                 let child_magic = if dir_stat.st_dev != stat.st_dev {
 853                     detect_fs_type(dir.as_raw_fd())?
 854                 } else {
 855                     magic
 856                 };
 857
 858                 self.write_filename(&filename)?;
 859                 if let Some(ref mut catalog) = self.catalog {
 860                     catalog.start_directory(&filename)?;
 861                 }
 862                 self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
 863                 if let Some(ref mut catalog) = self.catalog {
 864                     catalog.end_directory()?;
 865                 }
 866             } else if is_reg_file(&stat) {
 867                 let mut hardlink_target = None;
 868
 869                 if stat.st_nlink > 1 {
 870                     let link_info = HardLinkInfo {
 871                         st_dev: stat.st_dev,
 872                         st_ino: stat.st_ino,
 873                     };
 874                     hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
 875                         let mut target = v.clone().into_os_string();
 876                         target.push("\0"); // add Nul byte
 877                         (target, (start_pos as u64) - offset)
 878                     });
 879                     if hardlink_target == None {
 880                         self.hardlinks
 881                             .insert(link_info, (self.relative_path.clone(), start_pos as u64));
 882                     }
 883                 }
 884
 885                 if let Some((target, offset)) = hardlink_target {
 886                     if let Some(ref mut catalog) = self.catalog {
 887                         catalog.add_hardlink(&filename)?;
 888                     }
 889                     self.write_filename(&filename)?;
 890                     self.encode_hardlink(target.as_bytes(), offset)?;
 891                 } else {
 892                     let filefd = match nix::fcntl::openat(
 893                         rawfd,
 894                         filename.as_ref(),
 895                         OFlag::O_NOFOLLOW,
 896                         Mode::empty(),
 897                     ) {
 898                         Ok(filefd) => filefd,
 899                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 900                             self.report_vanished_file(&self.full_path())?;
 901                             self.relative_path.pop();
 902                             continue;
 903                         }
 904                         Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
 905                     };
 906
 907                     if let Some(ref mut catalog) = self.catalog {
 908                         catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
 909                     }
 910                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 911                         detect_fs_type(filefd)?
 912                     } else {
 913                         magic
 914                     };
 915
 916                     self.write_filename(&filename)?;
 917                     let res = self.encode_file(filefd, &stat, child_magic);
 918                     let _ = nix::unistd::close(filefd); // ignore close errors
 919                     res?;
 920                 }
 921             } else if is_symlink(&stat) {
 922                 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
 923
 924                 let res = filename.with_nix_path(|cstr| unsafe {
 925                     libc::readlinkat(
 926                         rawfd,
 927                         cstr.as_ptr(),
 928                         buffer.as_mut_ptr() as *mut libc::c_char,
 929                         buffer.len() - 1,
 930                     )
 931                 })?;
 932
 933                 match Errno::result(res) {
 934                     Ok(len) => {
 935                         if let Some(ref mut catalog) = self.catalog {
 936                             catalog.add_symlink(&filename)?;
 937                         }
 938                         buffer[len as usize] = 0u8; // add Nul byte
 939                         self.write_filename(&filename)?;
 940                         self.encode_symlink(&buffer[..((len + 1) as usize)], &stat)?
 941                     }
 942                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 943                         self.report_vanished_file(&self.full_path())?;
 944                         self.relative_path.pop();
 945                         continue;
 946                     }
 947                     Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
 948                 }
 949             } else if is_block_dev(&stat) || is_char_dev(&stat) {
 950                 if self.has_features(flags::WITH_DEVICE_NODES) {
 951                     if let Some(ref mut catalog) = self.catalog {
 952                         if is_block_dev(&stat) {
 953                             catalog.add_block_device(&filename)?;
 954                         } else {
 955                             catalog.add_char_device(&filename)?;
 956                         }
 957                     }
 958                     self.write_filename(&filename)?;
 959                     self.encode_device(&stat)?;
 960                 } else {
 961                     eprintln!("skip device node: {:?}", self.full_path());
 962                     self.relative_path.pop();
 963                     continue;
 964                 }
 965             } else if is_fifo(&stat) {
 966                 if self.has_features(flags::WITH_FIFOS) {
 967                     if let Some(ref mut catalog) = self.catalog {
 968                         catalog.add_fifo(&filename)?;
 969                     }
 970                     self.write_filename(&filename)?;
 971                     self.encode_special(&stat)?;
 972                 } else {
 973                     eprintln!("skip fifo: {:?}", self.full_path());
 974                     self.relative_path.pop();
 975                     continue;
 976                 }
 977             } else if is_socket(&stat) {
 978                 if self.has_features(flags::WITH_SOCKETS) {
 979                     if let Some(ref mut catalog) = self.catalog {
 980                         catalog.add_socket(&filename)?;
 981                     }
 982                     self.write_filename(&filename)?;
 983                     self.encode_special(&stat)?;
 984                 } else {
 985                     eprintln!("skip socket: {:?}", self.full_path());
 986                     self.relative_path.pop();
 987                     continue;
 988                 }
 989             } else {
 990                 bail!(
 991                     "unsupported file type (mode {:o} {:?})",
 992                     stat.st_mode,
 993                     self.full_path()
 994                 );
 995             }
 996
 997             let end_pos = self.writer_pos;
 998
 999             goodbye_items.push(PxarGoodbyeItem {
1000                 offset: start_pos as u64,
1001                 size: (end_pos - start_pos) as u64,
1002                 hash: compute_goodbye_hash(filename.to_bytes()),
1003             });
1004
1005             self.relative_path.pop();
1006         }
1007
1008         //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
1009
1010         // fixup goodby item offsets
1011         let goodbye_start = self.writer_pos as u64;
1012         for item in &mut goodbye_items {
1013             item.offset = goodbye_start - item.offset;
1014         }
1015
1016         let goodbye_offset = self.writer_pos - dir_start_pos;
1017
1018         self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
1019
1020         //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
1021         Ok(())
1022     }
1023
1024     fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64) -> Result<(), Error> {
1025         //println!("encode_file: {:?}", self.full_path());
1026
1027         let mut entry = self.create_entry(&stat)?;
1028
1029         self.read_chattr(filefd, &mut entry)?;
1030         self.read_fat_attr(filefd, magic, &mut entry)?;
1031         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1032         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1033         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1034
1035         self.write_entry(entry)?;
1036         for xattr in xattrs {
1037             self.write_xattr(xattr)?;
1038         }
1039         self.write_fcaps(fcaps)?;
1040         for user in acl_access.users {
1041             self.write_acl_user(user)?;
1042         }
1043         for group in acl_access.groups {
1044             self.write_acl_group(group)?;
1045         }
1046         if let Some(group_obj) = acl_access.group_obj {
1047             self.write_acl_group_obj(group_obj)?;
1048         }
1049         if let Some(projid) = projid {
1050             self.write_quota_project_id(projid)?;
1051         }
1052
1053         let include_payload;
1054         if is_virtual_file_system(magic) {
1055             include_payload = false;
1056         } else if let Some(ref set) = &self.device_set {
1057             include_payload = set.contains(&stat.st_dev);
1058         } else {
1059             include_payload = true;
1060         }
1061
1062         if !include_payload {
1063             eprintln!("skip content: {:?}", self.full_path());
1064             self.write_header(PXAR_PAYLOAD, 0)?;
1065             return Ok(());
1066         }
1067
1068         let size = stat.st_size as u64;
1069
1070         self.write_header(PXAR_PAYLOAD, size)?;
1071
1072         let mut pos: u64 = 0;
1073         loop {
1074             let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
1075                 Ok(n) => n,
1076                 Err(nix::Error::Sys(Errno::EINTR)) => continue, /* try again */
1077                 Err(err) => bail!("read {:?} failed - {}", self.full_path(), err),
1078             };
1079             if n == 0 { // EOF
1080                 if pos != size {
1081                     // Note:: casync format cannot handle that
1082                     bail!(
1083                         "detected shrinked file {:?} ({} < {})",
1084                         self.full_path(),
1085                         pos,
1086                         size
1087                     );
1088                 }
1089                 break;
1090             }
1091
1092             let mut next = pos + (n as u64);
1093
1094             if next > size {
1095                 next = size;
1096             }
1097
1098             let count = (next - pos) as usize;
1099
1100             self.flush_copy_buffer(count)?;
1101
1102             pos = next;
1103
1104             if pos >= size {
1105                 break;
1106             }
1107         }
1108
1109         Ok(())
1110     }
1111
1112     fn encode_device(&mut self, stat: &FileStat) -> Result<(), Error> {
1113         let entry = self.create_entry(&stat)?;
1114
1115         self.write_entry(entry)?;
1116
1117         let major = unsafe { libc::major(stat.st_rdev) } as u64;
1118         let minor = unsafe { libc::minor(stat.st_rdev) } as u64;
1119
1120         //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
1121
1122         self.write_header(PXAR_DEVICE, std::mem::size_of::<PxarDevice>() as u64)?;
1123         self.write_item(PxarDevice { major, minor })?;
1124
1125         Ok(())
1126     }
1127
1128     // FIFO or Socket
1129     fn encode_special(&mut self, stat: &FileStat) -> Result<(), Error> {
1130         let entry = self.create_entry(&stat)?;
1131
1132         self.write_entry(entry)?;
1133
1134         Ok(())
1135     }
1136
1137     fn encode_symlink(&mut self, target: &[u8], stat: &FileStat) -> Result<(), Error> {
1138         //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
1139
1140         let entry = self.create_entry(&stat)?;
1141         self.write_entry(entry)?;
1142
1143         self.write_header(PXAR_SYMLINK, target.len() as u64)?;
1144         self.write(target)?;
1145
1146         Ok(())
1147     }
1148
1149     fn encode_hardlink(&mut self, target: &[u8], offset: u64) -> Result<(), Error> {
1150         //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
1151
1152         // Note: HARDLINK replaces an ENTRY.
1153         self.write_header(PXAR_FORMAT_HARDLINK, (target.len() as u64) + 8)?;
1154         self.write_item(offset)?;
1155         self.write(target)?;
1156
1157         Ok(())
1158     }
1159
1160     fn encode_pxar_exclude(
1161         &mut self,
1162         filefd: RawFd,
1163         stat: &FileStat,
1164         magic: i64,
1165         content: &[u8],
1166     ) -> Result<(), Error> {
1167         let mut entry = self.create_entry(&stat)?;
1168
1169         self.read_chattr(filefd, &mut entry)?;
1170         self.read_fat_attr(filefd, magic, &mut entry)?;
1171         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1172         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1173         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1174
1175         self.write_entry(entry)?;
1176         for xattr in xattrs {
1177             self.write_xattr(xattr)?;
1178         }
1179         self.write_fcaps(fcaps)?;
1180         for user in acl_access.users {
1181             self.write_acl_user(user)?;
1182         }
1183         for group in acl_access.groups {
1184             self.write_acl_group(group)?;
1185         }
1186         if let Some(group_obj) = acl_access.group_obj {
1187             self.write_acl_group_obj(group_obj)?;
1188         }
1189         if let Some(projid) = projid {
1190             self.write_quota_project_id(projid)?;
1191         }
1192
1193         let include_payload;
1194         if is_virtual_file_system(magic) {
1195             include_payload = false;
1196         } else if let Some(set) = &self.device_set {
1197             include_payload = set.contains(&stat.st_dev);
1198         } else {
1199             include_payload = true;
1200         }
1201
1202         if !include_payload {
1203             eprintln!("skip content: {:?}", self.full_path());
1204             self.write_header(PXAR_PAYLOAD, 0)?;
1205             return Ok(());
1206         }
1207
1208         let size = content.len();
1209         self.write_header(PXAR_PAYLOAD, size as u64)?;
1210         self.writer.write_all(content)?;
1211         self.writer_pos += size;
1212
1213         Ok(())
1214     }
1215
1216     /// Encodes the excude match patterns passed via cli as file in the archive.
1217     fn encode_pxar_exclude_cli(
1218         &mut self,
1219         uid: u32,
1220         gid: u32,
1221         mtime: u64,
1222         content: &[u8],
1223     ) -> Result<(), Error> {
1224         let entry = PxarEntry {
1225             mode: (libc::S_IFREG | 0o600) as u64,
1226             flags: 0,
1227             uid,
1228             gid,
1229             mtime,
1230         };
1231         self.write_entry(entry)?;
1232         let size = content.len();
1233         self.write_header(PXAR_PAYLOAD, size as u64)?;
1234         self.writer.write_all(content)?;
1235         self.writer_pos += size;
1236
1237         Ok(())
1238     }
1239
1240     // the report_XXX method may raise and error - depending on encoder configuration
1241
1242     fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
1243         eprintln!("WARNING: detected vanished file {:?}", path);
1244
1245         Ok(())
1246     }
1247 }
1248
1249 fn errno_is_unsupported(errno: Errno) -> bool {
1250     match errno {
1251         Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => true,
1252         _ => false,
1253     }
1254 }
1255
1256 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
1257     let mut fs_stat = std::mem::MaybeUninit::uninit();
1258     let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
1259     Errno::result(res)?;
1260     let fs_stat = unsafe { fs_stat.assume_init() };
1261
1262     Ok(fs_stat.f_type)
1263 }
1264
1265 #[inline(always)]
1266 pub fn is_temporary_file_system(magic: i64) -> bool {
1267     use proxmox::sys::linux::magic::*;
1268     magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
1269 }
1270
1271 pub fn is_virtual_file_system(magic: i64) -> bool {
1272     use proxmox::sys::linux::magic::*;
1273
1274     match magic {
1275         BINFMTFS_MAGIC |
1276         CGROUP2_SUPER_MAGIC |
1277         CGROUP_SUPER_MAGIC |
1278         CONFIGFS_MAGIC |
1279         DEBUGFS_MAGIC |
1280         DEVPTS_SUPER_MAGIC |
1281         EFIVARFS_MAGIC |
1282         FUSE_CTL_SUPER_MAGIC |
1283         HUGETLBFS_MAGIC |
1284         MQUEUE_MAGIC |
1285         NFSD_MAGIC |
1286         PROC_SUPER_MAGIC |
1287         PSTOREFS_MAGIC |
1288         RPCAUTH_GSSMAGIC |
1289         SECURITYFS_MAGIC |
1290         SELINUX_MAGIC |
1291         SMACK_MAGIC |
1292         SYSFS_MAGIC => true,
1293         _ => false
1294     }
1295 }