src/pxar/encoder.rs

   1 //! *pxar* format encoder.
   2 //!
   3 //! This module contain the code to generate *pxar* archive files.
   4 use std::collections::{HashMap, HashSet};
   5 use std::ffi::CStr;
   6 use std::io::Write;
   7 use std::os::unix::ffi::OsStrExt;
   8 use std::os::unix::io::AsRawFd;
   9 use std::os::unix::io::RawFd;
  10 use std::path::{Path, PathBuf};
  11
  12 use endian_trait::Endian;
  13 use failure::*;
  14 use nix::errno::Errno;
  15 use nix::fcntl::OFlag;
  16 use nix::sys::stat::FileStat;
  17 use nix::sys::stat::Mode;
  18 use nix::NixPath;
  19
  20 use proxmox::tools::vec;
  21
  22 use super::binary_search_tree::*;
  23 use super::catalog::BackupCatalogWriter;
  24 use super::flags;
  25 use super::format_definition::*;
  26 use super::helper::*;
  27 use super::match_pattern::{MatchPattern, MatchType};
  28 use crate::tools::acl;
  29 use crate::tools::fs;
  30 use crate::tools::xattr;
  31
  32 /// The format requires to build sorted directory lookup tables in
  33 /// memory, so we restrict the number of allowed entries to limit
  34 /// maximum memory usage.
  35 pub const MAX_DIRECTORY_ENTRIES: usize = 256 * 1024;
  36
  37 #[derive(Eq, PartialEq, Hash)]
  38 struct HardLinkInfo {
  39     st_dev: u64,
  40     st_ino: u64,
  41 }
  42
  43 pub struct Encoder<'a, W: Write, C: BackupCatalogWriter> {
  44     base_path: PathBuf,
  45     relative_path: PathBuf,
  46     writer: &'a mut W,
  47     writer_pos: usize,
  48     catalog: Option<&'a mut C>,
  49     _size: usize,
  50     file_copy_buffer: Vec<u8>,
  51     device_set: Option<HashSet<u64>>,
  52     verbose: bool,
  53     // Flags set by the user
  54     feature_flags: u64,
  55     // Flags signaling features supported by the filesystem
  56     fs_feature_flags: u64,
  57     hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
  58 }
  59
  60 impl<'a, W: Write, C: BackupCatalogWriter> Encoder<'a, W, C> {
  61     // used for error reporting
  62     fn full_path(&self) -> PathBuf {
  63         self.base_path.join(&self.relative_path)
  64     }
  65
  66     /// Create archive, write result data to ``writer``.
  67     ///
  68     /// The ``device_set`` can be use used to limit included mount points.
  69     ///
  70     /// - ``None``: include all mount points
  71     /// - ``Some(set)``: only include devices listed in this set (the
  72     ///   root path device is automathically added to this list, so
  73     ///   you can pass an empty set if you want to archive a single
  74     ///   mount point.)
  75     pub fn encode(
  76         path: PathBuf,
  77         dir: &mut nix::dir::Dir,
  78         writer: &'a mut W,
  79         catalog: Option<&'a mut C>,
  80         device_set: Option<HashSet<u64>>,
  81         verbose: bool,
  82         skip_lost_and_found: bool, // fixme: should be a feature flag ??
  83         feature_flags: u64,
  84     ) -> Result<(), Error> {
  85         const FILE_COPY_BUFFER_SIZE: usize = 1024 * 1024;
  86
  87         let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
  88         unsafe {
  89             file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE);
  90         }
  91
  92         // todo: use scandirat??
  93
  94         let dir_fd = dir.as_raw_fd();
  95         let stat = nix::sys::stat::fstat(dir_fd)
  96             .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
  97
  98         if !is_directory(&stat) {
  99             bail!("got unexpected file type {:?} (not a directory)", path);
 100         }
 101
 102         let mut device_set = device_set.clone();
 103         if let Some(ref mut set) = device_set {
 104             set.insert(stat.st_dev);
 105         }
 106
 107         let magic = detect_fs_type(dir_fd)?;
 108
 109         if is_virtual_file_system(magic) {
 110             bail!("backup virtual file systems is disabled!");
 111         }
 112
 113         let fs_feature_flags = flags::feature_flags_from_magic(magic);
 114
 115         let mut me = Self {
 116             base_path: path,
 117             relative_path: PathBuf::new(),
 118             writer: writer,
 119             writer_pos: 0,
 120             catalog,
 121             _size: 0,
 122             file_copy_buffer,
 123             device_set,
 124             verbose,
 125             feature_flags,
 126             fs_feature_flags,
 127             hardlinks: HashMap::new(),
 128         };
 129
 130         if verbose {
 131             println!("{:?}", me.full_path());
 132         }
 133
 134         let mut excludes = Vec::new();
 135         if skip_lost_and_found {
 136             excludes.push(MatchPattern::from_line(b"**/lost+found").unwrap().unwrap());
 137         }
 138         me.encode_dir(dir, &stat, magic, excludes)?;
 139
 140         Ok(())
 141     }
 142
 143     fn write(&mut self, buf: &[u8]) -> Result<(), Error> {
 144         self.writer.write_all(buf)?;
 145         self.writer_pos += buf.len();
 146         Ok(())
 147     }
 148
 149     fn write_item<T: Endian>(&mut self, item: T) -> Result<(), Error> {
 150         let data = item.to_le();
 151
 152         let buffer = unsafe {
 153             std::slice::from_raw_parts(&data as *const T as *const u8, std::mem::size_of::<T>())
 154         };
 155
 156         self.write(buffer)?;
 157
 158         Ok(())
 159     }
 160
 161     fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
 162         self.writer.write_all(&self.file_copy_buffer[..size])?;
 163         self.writer_pos += size;
 164         Ok(())
 165     }
 166
 167     fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
 168         let size = size + (std::mem::size_of::<PxarHeader>() as u64);
 169         self.write_item(PxarHeader { size, htype })?;
 170
 171         Ok(())
 172     }
 173
 174     fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
 175         let buffer = name.to_bytes_with_nul();
 176         self.write_header(PXAR_FILENAME, buffer.len() as u64)?;
 177         self.write(buffer)?;
 178
 179         Ok(())
 180     }
 181
 182     fn create_entry(&self, stat: &FileStat) -> Result<PxarEntry, Error> {
 183         let mode = if is_symlink(&stat) {
 184             (libc::S_IFLNK | 0o777) as u64
 185         } else {
 186             (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
 187         };
 188
 189         let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
 190         if mtime < 0 {
 191             bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
 192         }
 193
 194         let entry = PxarEntry {
 195             mode: mode,
 196             flags: 0,
 197             uid: stat.st_uid,
 198             gid: stat.st_gid,
 199             mtime: mtime as u64,
 200         };
 201
 202         Ok(entry)
 203     }
 204
 205     fn read_chattr(&self, fd: RawFd, entry: &mut PxarEntry) -> Result<(), Error> {
 206         let mut attr: usize = 0;
 207
 208         let res = unsafe { fs::read_attr_fd(fd, &mut attr) };
 209         if let Err(err) = res {
 210             if let nix::Error::Sys(errno) = err {
 211                 if errno_is_unsupported(errno) {
 212                     return Ok(());
 213                 };
 214             }
 215             bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
 216         }
 217
 218         let flags = flags::feature_flags_from_chattr(attr as u32);
 219         entry.flags = entry.flags | flags;
 220
 221         Ok(())
 222     }
 223
 224     fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut PxarEntry) -> Result<(), Error> {
 225         use proxmox::sys::linux::magic::*;
 226
 227         if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC {
 228             return Ok(());
 229         }
 230
 231         let mut attr: u32 = 0;
 232
 233         let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr) };
 234         if let Err(err) = res {
 235             if let nix::Error::Sys(errno) = err {
 236                 if errno_is_unsupported(errno) {
 237                     return Ok(());
 238                 };
 239             }
 240             bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
 241         }
 242
 243         let flags = flags::feature_flags_from_fat_attr(attr);
 244         entry.flags = entry.flags | flags;
 245
 246         Ok(())
 247     }
 248
 249     /// True if all of the given feature flags are set in the Encoder, false otherwise
 250     fn has_features(&self, feature_flags: u64) -> bool {
 251         (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
 252     }
 253
 254     /// True if at least one of the given feature flags is set in the Encoder, false otherwise
 255     fn has_some_features(&self, feature_flags: u64) -> bool {
 256         (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
 257     }
 258
 259     fn read_xattrs(
 260         &self,
 261         fd: RawFd,
 262         stat: &FileStat,
 263     ) -> Result<(Vec<PxarXAttr>, Option<PxarFCaps>), Error> {
 264         let mut xattrs = Vec::new();
 265         let mut fcaps = None;
 266
 267         let flags = flags::WITH_XATTRS | flags::WITH_FCAPS;
 268         if !self.has_some_features(flags) {
 269             return Ok((xattrs, fcaps));
 270         }
 271         // Should never be called on symlinks, just in case check anyway
 272         if is_symlink(&stat) {
 273             return Ok((xattrs, fcaps));
 274         }
 275
 276         let xattr_names = match xattr::flistxattr(fd) {
 277             Ok(names) => names,
 278             // Do not bail if the underlying endpoint does not supports xattrs
 279             Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
 280             // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
 281             Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
 282             Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 283         };
 284
 285         for name in xattr_names.split(|c| *c == b'\0') {
 286             // Only extract the relevant extended attributes
 287             if !xattr::is_valid_xattr_name(&name) {
 288                 continue;
 289             }
 290
 291             let value = match xattr::fgetxattr(fd, name) {
 292                 Ok(value) => value,
 293                 // Vanished between flistattr and getxattr, this is ok, silently ignore
 294                 Err(Errno::ENODATA) => continue,
 295                 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 296             };
 297
 298             if xattr::is_security_capability(&name) {
 299                 if self.has_features(flags::WITH_FCAPS) {
 300                     // fcaps are stored in own format within the archive
 301                     fcaps = Some(PxarFCaps { data: value });
 302                 }
 303             } else if self.has_features(flags::WITH_XATTRS) {
 304                 xattrs.push(PxarXAttr {
 305                     name: name.to_vec(),
 306                     value: value,
 307                 });
 308             }
 309         }
 310         xattrs.sort();
 311
 312         Ok((xattrs, fcaps))
 313     }
 314
 315     fn read_acl(
 316         &self,
 317         fd: RawFd,
 318         stat: &FileStat,
 319         acl_type: acl::ACLType,
 320     ) -> Result<PxarACL, Error> {
 321         let ret = PxarACL {
 322             users: Vec::new(),
 323             groups: Vec::new(),
 324             group_obj: None,
 325             default: None,
 326         };
 327
 328         if !self.has_features(flags::WITH_ACL) {
 329             return Ok(ret);
 330         }
 331         if is_symlink(&stat) {
 332             return Ok(ret);
 333         }
 334         if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
 335             bail!("ACL_TYPE_DEFAULT only defined for directories.");
 336         }
 337
 338         // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
 339         // to create a path for acl_get_file(). acl_get_fd() only allows to get
 340         // ACL_TYPE_ACCESS attributes.
 341         let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
 342         let acl = match acl::ACL::get_file(&proc_path, acl_type) {
 343             Ok(acl) => acl,
 344             // Don't bail if underlying endpoint does not support acls
 345             Err(Errno::EOPNOTSUPP) => return Ok(ret),
 346             // Don't bail if the endpoint cannot carry acls
 347             Err(Errno::EBADF) => return Ok(ret),
 348             // Don't bail if there is no data
 349             Err(Errno::ENODATA) => return Ok(ret),
 350             Err(err) => bail!("error while reading ACL - {}", err),
 351         };
 352
 353         self.process_acl(acl, acl_type)
 354     }
 355
 356     fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 357         let mut acl_user = Vec::new();
 358         let mut acl_group = Vec::new();
 359         let mut acl_group_obj = None;
 360         let mut acl_default = None;
 361         let mut user_obj_permissions = None;
 362         let mut group_obj_permissions = None;
 363         let mut other_permissions = None;
 364         let mut mask_permissions = None;
 365
 366         for entry in &mut acl.entries() {
 367             let tag = entry.get_tag_type()?;
 368             let permissions = entry.get_permissions()?;
 369             match tag {
 370                 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
 371                 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
 372                 acl::ACL_OTHER => other_permissions = Some(permissions),
 373                 acl::ACL_MASK => mask_permissions = Some(permissions),
 374                 acl::ACL_USER => {
 375                     acl_user.push(PxarACLUser {
 376                         uid: entry.get_qualifier()?,
 377                         permissions: permissions,
 378                     });
 379                 }
 380                 acl::ACL_GROUP => {
 381                     acl_group.push(PxarACLGroup {
 382                         gid: entry.get_qualifier()?,
 383                         permissions: permissions,
 384                     });
 385                 }
 386                 _ => bail!("Unexpected ACL tag encountered!"),
 387             }
 388         }
 389
 390         acl_user.sort();
 391         acl_group.sort();
 392
 393         match acl_type {
 394             acl::ACL_TYPE_ACCESS => {
 395                 // The mask permissions are mapped to the stat group permissions
 396                 // in case that the ACL group permissions were set.
 397                 // Only in that case we need to store the group permissions,
 398                 // in the other cases they are identical to the stat group permissions.
 399                 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
 400                     acl_group_obj = Some(PxarACLGroupObj { permissions: gop });
 401                 }
 402             }
 403             acl::ACL_TYPE_DEFAULT => {
 404                 if user_obj_permissions != None
 405                     || group_obj_permissions != None
 406                     || other_permissions != None
 407                     || mask_permissions != None
 408                 {
 409                     acl_default = Some(PxarACLDefault {
 410                         // The value is set to UINT64_MAX as placeholder if one
 411                         // of the permissions is not set
 412                         user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
 413                         group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
 414                         other_permissions: other_permissions.unwrap_or(std::u64::MAX),
 415                         mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
 416                     });
 417                 }
 418             }
 419             _ => bail!("Unexpected ACL type encountered"),
 420         }
 421
 422         Ok(PxarACL {
 423             users: acl_user,
 424             groups: acl_group,
 425             group_obj: acl_group_obj,
 426             default: acl_default,
 427         })
 428     }
 429
 430     /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
 431     fn read_quota_project_id(
 432         &self,
 433         fd: RawFd,
 434         magic: i64,
 435         stat: &FileStat,
 436     ) -> Result<Option<PxarQuotaProjID>, Error> {
 437         if !(is_directory(&stat) || is_reg_file(&stat)) {
 438             return Ok(None);
 439         }
 440         if !self.has_features(flags::WITH_QUOTA_PROJID) {
 441             return Ok(None);
 442         }
 443
 444         use proxmox::sys::linux::magic::*;
 445
 446         match magic {
 447             EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC | ZFS_SUPER_MAGIC => {
 448                 let mut fsxattr = fs::FSXAttr::default();
 449                 let res = unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) };
 450
 451                 // On some FUSE filesystems it can happen that ioctl is not supported.
 452                 // For these cases projid is set to 0 while the error is ignored.
 453                 if let Err(err) = res {
 454                     let errno = err.as_errno().ok_or_else(|| {
 455                         format_err!(
 456                             "error while reading quota project id for {:#?}",
 457                             self.full_path()
 458                         )
 459                     })?;
 460                     if errno_is_unsupported(errno) {
 461                         return Ok(None);
 462                     } else {
 463                         bail!(
 464                             "error while reading quota project id for {:#?} - {}",
 465                             self.full_path(),
 466                             errno
 467                         );
 468                     }
 469                 }
 470
 471                 let projid = fsxattr.fsx_projid as u64;
 472                 if projid == 0 {
 473                     return Ok(None);
 474                 } else {
 475                     return Ok(Some(PxarQuotaProjID { projid }));
 476                 }
 477             }
 478             _ => return Ok(None),
 479         }
 480     }
 481
 482     fn write_entry(&mut self, entry: PxarEntry) -> Result<(), Error> {
 483         self.write_header(PXAR_ENTRY, std::mem::size_of::<PxarEntry>() as u64)?;
 484         self.write_item(entry)?;
 485
 486         Ok(())
 487     }
 488
 489     fn write_xattr(&mut self, xattr: PxarXAttr) -> Result<(), Error> {
 490         let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
 491         self.write_header(PXAR_XATTR, size as u64)?;
 492         self.write(xattr.name.as_slice())?;
 493         self.write(&[0])?;
 494         self.write(xattr.value.as_slice())?;
 495
 496         Ok(())
 497     }
 498
 499     fn write_fcaps(&mut self, fcaps: Option<PxarFCaps>) -> Result<(), Error> {
 500         if let Some(fcaps) = fcaps {
 501             let size = fcaps.data.len();
 502             self.write_header(PXAR_FCAPS, size as u64)?;
 503             self.write(fcaps.data.as_slice())?;
 504         }
 505
 506         Ok(())
 507     }
 508
 509     fn write_acl_user(&mut self, acl_user: PxarACLUser) -> Result<(), Error> {
 510         self.write_header(PXAR_ACL_USER, std::mem::size_of::<PxarACLUser>() as u64)?;
 511         self.write_item(acl_user)?;
 512
 513         Ok(())
 514     }
 515
 516     fn write_acl_group(&mut self, acl_group: PxarACLGroup) -> Result<(), Error> {
 517         self.write_header(PXAR_ACL_GROUP, std::mem::size_of::<PxarACLGroup>() as u64)?;
 518         self.write_item(acl_group)?;
 519
 520         Ok(())
 521     }
 522
 523     fn write_acl_group_obj(&mut self, acl_group_obj: PxarACLGroupObj) -> Result<(), Error> {
 524         self.write_header(
 525             PXAR_ACL_GROUP_OBJ,
 526             std::mem::size_of::<PxarACLGroupObj>() as u64,
 527         )?;
 528         self.write_item(acl_group_obj)?;
 529
 530         Ok(())
 531     }
 532
 533     fn write_acl_default(&mut self, acl_default: PxarACLDefault) -> Result<(), Error> {
 534         self.write_header(
 535             PXAR_ACL_DEFAULT,
 536             std::mem::size_of::<PxarACLDefault>() as u64,
 537         )?;
 538         self.write_item(acl_default)?;
 539
 540         Ok(())
 541     }
 542
 543     fn write_acl_default_user(&mut self, acl_default_user: PxarACLUser) -> Result<(), Error> {
 544         self.write_header(
 545             PXAR_ACL_DEFAULT_USER,
 546             std::mem::size_of::<PxarACLUser>() as u64,
 547         )?;
 548         self.write_item(acl_default_user)?;
 549
 550         Ok(())
 551     }
 552
 553     fn write_acl_default_group(&mut self, acl_default_group: PxarACLGroup) -> Result<(), Error> {
 554         self.write_header(
 555             PXAR_ACL_DEFAULT_GROUP,
 556             std::mem::size_of::<PxarACLGroup>() as u64,
 557         )?;
 558         self.write_item(acl_default_group)?;
 559
 560         Ok(())
 561     }
 562
 563     fn write_quota_project_id(&mut self, projid: PxarQuotaProjID) -> Result<(), Error> {
 564         self.write_header(
 565             PXAR_QUOTA_PROJID,
 566             std::mem::size_of::<PxarQuotaProjID>() as u64,
 567         )?;
 568         self.write_item(projid)?;
 569
 570         Ok(())
 571     }
 572
 573     fn write_goodbye_table(
 574         &mut self,
 575         goodbye_offset: usize,
 576         goodbye_items: &mut [PxarGoodbyeItem],
 577     ) -> Result<(), Error> {
 578         goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
 579
 580         let item_count = goodbye_items.len();
 581
 582         let goodbye_table_size = (item_count + 1) * std::mem::size_of::<PxarGoodbyeItem>();
 583
 584         self.write_header(PXAR_GOODBYE, goodbye_table_size as u64)?;
 585
 586         if self.file_copy_buffer.len() < goodbye_table_size {
 587             let need = goodbye_table_size - self.file_copy_buffer.len();
 588             self.file_copy_buffer.reserve(need);
 589             unsafe {
 590                 self.file_copy_buffer
 591                     .set_len(self.file_copy_buffer.capacity());
 592             }
 593         }
 594
 595         let buffer = &mut self.file_copy_buffer;
 596
 597         copy_binary_search_tree(item_count, |s, d| {
 598             let item = &goodbye_items[s];
 599             let offset = d * std::mem::size_of::<PxarGoodbyeItem>();
 600             let dest =
 601                 crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
 602             dest.offset = u64::to_le(item.offset);
 603             dest.size = u64::to_le(item.size);
 604             dest.hash = u64::to_le(item.hash);
 605         });
 606
 607         // append PxarGoodbyeTail as last item
 608         let offset = item_count * std::mem::size_of::<PxarGoodbyeItem>();
 609         let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
 610         dest.offset = u64::to_le(goodbye_offset as u64);
 611         dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<PxarHeader>()) as u64);
 612         dest.hash = u64::to_le(PXAR_GOODBYE_TAIL_MARKER);
 613
 614         self.flush_copy_buffer(goodbye_table_size)?;
 615
 616         Ok(())
 617     }
 618
 619     fn encode_dir(
 620         &mut self,
 621         dir: &mut nix::dir::Dir,
 622         dir_stat: &FileStat,
 623         magic: i64,
 624         match_pattern: Vec<MatchPattern>,
 625     ) -> Result<(), Error> {
 626         //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
 627
 628         let mut name_list = vec![];
 629
 630         let rawfd = dir.as_raw_fd();
 631
 632         let dir_start_pos = self.writer_pos;
 633
 634         let mut dir_entry = self.create_entry(&dir_stat)?;
 635
 636         self.read_chattr(rawfd, &mut dir_entry)?;
 637         self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
 638
 639         // for each node in the directory tree, the filesystem features are
 640         // checked based on the fs magic number.
 641         self.fs_feature_flags = flags::feature_flags_from_magic(magic);
 642
 643         let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
 644         let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
 645         let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
 646         let projid = self.read_quota_project_id(rawfd, magic, &dir_stat)?;
 647
 648         self.write_entry(dir_entry)?;
 649         for xattr in xattrs {
 650             self.write_xattr(xattr)?;
 651         }
 652         self.write_fcaps(fcaps)?;
 653
 654         for user in acl_access.users {
 655             self.write_acl_user(user)?;
 656         }
 657         for group in acl_access.groups {
 658             self.write_acl_group(group)?;
 659         }
 660         if let Some(group_obj) = acl_access.group_obj {
 661             self.write_acl_group_obj(group_obj)?;
 662         }
 663
 664         for default_user in acl_default.users {
 665             self.write_acl_default_user(default_user)?;
 666         }
 667         for default_group in acl_default.groups {
 668             self.write_acl_default_group(default_group)?;
 669         }
 670         if let Some(default) = acl_default.default {
 671             self.write_acl_default(default)?;
 672         }
 673         if let Some(projid) = projid {
 674             self.write_quota_project_id(projid)?;
 675         }
 676
 677         let include_children;
 678         if is_virtual_file_system(magic) {
 679             include_children = false;
 680         } else {
 681             if let Some(set) = &self.device_set {
 682                 include_children = set.contains(&dir_stat.st_dev);
 683             } else {
 684                 include_children = true;
 685             }
 686         }
 687
 688         // Expand the exclude match pattern inherited from the parent by local entries, if present
 689         let mut local_match_pattern = match_pattern.clone();
 690         let pxar_exclude = match MatchPattern::from_file(rawfd, ".pxarexclude") {
 691             Ok(Some((mut excludes, buffer, stat))) => {
 692                 local_match_pattern.append(&mut excludes);
 693                 Some((buffer, stat))
 694             }
 695             Ok(None) => None,
 696             Err(err) => bail!("error while reading exclude file - {}", err),
 697         };
 698
 699         if include_children {
 700             for entry in dir.iter() {
 701                 let entry = entry
 702                     .map_err(|err| format_err!("readir {:?} failed - {}", self.full_path(), err))?;
 703                 let filename = entry.file_name().to_owned();
 704
 705                 let name = filename.to_bytes_with_nul();
 706                 if name == b".\0" || name == b"..\0" {
 707                     continue;
 708                 }
 709
 710                 let stat = match nix::sys::stat::fstatat(
 711                     rawfd,
 712                     filename.as_ref(),
 713                     nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW,
 714                 ) {
 715                     Ok(stat) => stat,
 716                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 717                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 718                         self.report_vanished_file(&self.full_path().join(filename_osstr))?;
 719                         continue;
 720                     }
 721                     Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
 722                 };
 723
 724                 match match_filename(&filename, &stat, &local_match_pattern)? {
 725                     (MatchType::Positive, _) => {
 726                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 727                         eprintln!(
 728                             "matched by .pxarexclude entry - skipping: {:?}",
 729                             self.full_path().join(filename_osstr)
 730                         );
 731                     }
 732                     (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
 733                 }
 734
 735                 if name_list.len() > MAX_DIRECTORY_ENTRIES {
 736                     bail!(
 737                         "too many directory items in {:?} (> {})",
 738                         self.full_path(),
 739                         MAX_DIRECTORY_ENTRIES
 740                     );
 741                 }
 742             }
 743         } else {
 744             eprintln!("skip mount point: {:?}", self.full_path());
 745         }
 746
 747         name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
 748
 749         let mut goodbye_items = vec![];
 750
 751         for (filename, stat, exclude_list) in name_list {
 752             let start_pos = self.writer_pos;
 753
 754             if filename.as_bytes() == b".pxarexclude" {
 755                 if let Some((ref content, ref stat)) = pxar_exclude {
 756                     let filefd = match nix::fcntl::openat(
 757                         rawfd,
 758                         filename.as_ref(),
 759                         OFlag::O_NOFOLLOW,
 760                         Mode::empty(),
 761                     ) {
 762                         Ok(filefd) => filefd,
 763                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 764                             self.report_vanished_file(&self.full_path())?;
 765                             continue;
 766                         }
 767                         Err(err) => {
 768                             let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 769                             bail!(
 770                                 "open file {:?} failed - {}",
 771                                 self.full_path().join(filename_osstr),
 772                                 err
 773                             );
 774                         }
 775                     };
 776
 777                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 778                         detect_fs_type(filefd)?
 779                     } else {
 780                         magic
 781                     };
 782
 783                     self.write_filename(&filename)?;
 784                     self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
 785                     continue;
 786                 }
 787             }
 788
 789             self.relative_path
 790                 .push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
 791
 792             if self.verbose {
 793                 println!("{:?}", self.full_path());
 794             }
 795
 796             if is_directory(&stat) {
 797                 let mut dir = match nix::dir::Dir::openat(
 798                     rawfd,
 799                     filename.as_ref(),
 800                     OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW,
 801                     Mode::empty(),
 802                 ) {
 803                     Ok(dir) => dir,
 804                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 805                         self.report_vanished_file(&self.full_path())?;
 806                         continue; // fixme!!
 807                     }
 808                     Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
 809                 };
 810
 811                 let child_magic = if dir_stat.st_dev != stat.st_dev {
 812                     detect_fs_type(dir.as_raw_fd())?
 813                 } else {
 814                     magic
 815                 };
 816
 817                 self.write_filename(&filename)?;
 818                 if let Some(ref mut catalog) = self.catalog {
 819                     catalog.start_directory(&filename)?;
 820                 }
 821                 self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
 822                 if let Some(ref mut catalog) = self.catalog {
 823                     catalog.end_directory()?;
 824                 }
 825             } else if is_reg_file(&stat) {
 826                 let mut hardlink_target = None;
 827
 828                 if stat.st_nlink > 1 {
 829                     let link_info = HardLinkInfo {
 830                         st_dev: stat.st_dev,
 831                         st_ino: stat.st_ino,
 832                     };
 833                     hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
 834                         let mut target = v.clone().into_os_string();
 835                         target.push("\0"); // add Nul byte
 836                         (target, (start_pos as u64) - offset)
 837                     });
 838                     if hardlink_target == None {
 839                         self.hardlinks
 840                             .insert(link_info, (self.relative_path.clone(), start_pos as u64));
 841                     }
 842                 }
 843
 844                 if let Some((target, offset)) = hardlink_target {
 845                     if let Some(ref mut catalog) = self.catalog {
 846                         catalog.add_hardlink(&filename)?;
 847                     }
 848                     self.write_filename(&filename)?;
 849                     self.encode_hardlink(target.as_bytes(), offset)?;
 850                 } else {
 851                     let filefd = match nix::fcntl::openat(
 852                         rawfd,
 853                         filename.as_ref(),
 854                         OFlag::O_NOFOLLOW,
 855                         Mode::empty(),
 856                     ) {
 857                         Ok(filefd) => filefd,
 858                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 859                             self.report_vanished_file(&self.full_path())?;
 860                             continue;
 861                         }
 862                         Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
 863                     };
 864
 865                     if let Some(ref mut catalog) = self.catalog {
 866                         catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
 867                     }
 868                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 869                         detect_fs_type(filefd)?
 870                     } else {
 871                         magic
 872                     };
 873
 874                     self.write_filename(&filename)?;
 875                     let res = self.encode_file(filefd, &stat, child_magic);
 876                     let _ = nix::unistd::close(filefd); // ignore close errors
 877                     res?;
 878                 }
 879             } else if is_symlink(&stat) {
 880                 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
 881
 882                 let res = filename.with_nix_path(|cstr| unsafe {
 883                     libc::readlinkat(
 884                         rawfd,
 885                         cstr.as_ptr(),
 886                         buffer.as_mut_ptr() as *mut libc::c_char,
 887                         buffer.len() - 1,
 888                     )
 889                 })?;
 890
 891                 match Errno::result(res) {
 892                     Ok(len) => {
 893                         if let Some(ref mut catalog) = self.catalog {
 894                             catalog.add_symlink(&filename)?;
 895                         }
 896                         buffer[len as usize] = 0u8; // add Nul byte
 897                         self.write_filename(&filename)?;
 898                         self.encode_symlink(&buffer[..((len + 1) as usize)], &stat)?
 899                     }
 900                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 901                         self.report_vanished_file(&self.full_path())?;
 902                         continue;
 903                     }
 904                     Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
 905                 }
 906             } else if is_block_dev(&stat) || is_char_dev(&stat) {
 907                 if self.has_features(flags::WITH_DEVICE_NODES) {
 908                     if let Some(ref mut catalog) = self.catalog {
 909                         if is_block_dev(&stat) {
 910                             catalog.add_block_device(&filename)?;
 911                         } else {
 912                             catalog.add_char_device(&filename)?;
 913                         }
 914                     }
 915                     self.write_filename(&filename)?;
 916                     self.encode_device(&stat)?;
 917                 } else {
 918                     eprintln!("skip device node: {:?}", self.full_path());
 919                 }
 920             } else if is_fifo(&stat) {
 921                 if self.has_features(flags::WITH_FIFOS) {
 922                     if let Some(ref mut catalog) = self.catalog {
 923                         catalog.add_fifo(&filename)?;
 924                     }
 925                     self.write_filename(&filename)?;
 926                     self.encode_special(&stat)?;
 927                 } else {
 928                     eprintln!("skip fifo: {:?}", self.full_path());
 929                 }
 930             } else if is_socket(&stat) {
 931                 if self.has_features(flags::WITH_SOCKETS) {
 932                     if let Some(ref mut catalog) = self.catalog {
 933                         catalog.add_socket(&filename)?;
 934                     }
 935                     self.write_filename(&filename)?;
 936                     self.encode_special(&stat)?;
 937                 } else {
 938                     eprintln!("skip socket: {:?}", self.full_path());
 939                 }
 940             } else {
 941                 bail!(
 942                     "unsupported file type (mode {:o} {:?})",
 943                     stat.st_mode,
 944                     self.full_path()
 945                 );
 946             }
 947
 948             let end_pos = self.writer_pos;
 949
 950             goodbye_items.push(PxarGoodbyeItem {
 951                 offset: start_pos as u64,
 952                 size: (end_pos - start_pos) as u64,
 953                 hash: compute_goodbye_hash(filename.to_bytes()),
 954             });
 955
 956             self.relative_path.pop();
 957         }
 958
 959         //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
 960
 961         // fixup goodby item offsets
 962         let goodbye_start = self.writer_pos as u64;
 963         for item in &mut goodbye_items {
 964             item.offset = goodbye_start - item.offset;
 965         }
 966
 967         let goodbye_offset = self.writer_pos - dir_start_pos;
 968
 969         self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
 970
 971         //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
 972         Ok(())
 973     }
 974
 975     fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64) -> Result<(), Error> {
 976         //println!("encode_file: {:?}", self.full_path());
 977
 978         let mut entry = self.create_entry(&stat)?;
 979
 980         self.read_chattr(filefd, &mut entry)?;
 981         self.read_fat_attr(filefd, magic, &mut entry)?;
 982         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
 983         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
 984         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
 985
 986         self.write_entry(entry)?;
 987         for xattr in xattrs {
 988             self.write_xattr(xattr)?;
 989         }
 990         self.write_fcaps(fcaps)?;
 991         for user in acl_access.users {
 992             self.write_acl_user(user)?;
 993         }
 994         for group in acl_access.groups {
 995             self.write_acl_group(group)?;
 996         }
 997         if let Some(group_obj) = acl_access.group_obj {
 998             self.write_acl_group_obj(group_obj)?;
 999         }
1000         if let Some(projid) = projid {
1001             self.write_quota_project_id(projid)?;
1002         }
1003
1004         let include_payload;
1005         if is_virtual_file_system(magic) {
1006             include_payload = false;
1007         } else {
1008             if let Some(ref set) = &self.device_set {
1009                 include_payload = set.contains(&stat.st_dev);
1010             } else {
1011                 include_payload = true;
1012             }
1013         }
1014
1015         if !include_payload {
1016             eprintln!("skip content: {:?}", self.full_path());
1017             self.write_header(PXAR_PAYLOAD, 0)?;
1018             return Ok(());
1019         }
1020
1021         let size = stat.st_size as u64;
1022
1023         self.write_header(PXAR_PAYLOAD, size)?;
1024
1025         let mut pos: u64 = 0;
1026         loop {
1027             let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
1028                 Ok(n) => n,
1029                 Err(nix::Error::Sys(Errno::EINTR)) => continue, /* try again */
1030                 Err(err) => bail!("read {:?} failed - {}", self.full_path(), err),
1031             };
1032             if n == 0 { // EOF
1033                 if pos != size {
1034                     // Note:: casync format cannot handle that
1035                     bail!(
1036                         "detected shrinked file {:?} ({} < {})",
1037                         self.full_path(),
1038                         pos,
1039                         size
1040                     );
1041                 }
1042                 break;
1043             }
1044
1045             let mut next = pos + (n as u64);
1046
1047             if next > size {
1048                 next = size;
1049             }
1050
1051             let count = (next - pos) as usize;
1052
1053             self.flush_copy_buffer(count)?;
1054
1055             pos = next;
1056
1057             if pos >= size {
1058                 break;
1059             }
1060         }
1061
1062         Ok(())
1063     }
1064
1065     fn encode_device(&mut self, stat: &FileStat) -> Result<(), Error> {
1066         let entry = self.create_entry(&stat)?;
1067
1068         self.write_entry(entry)?;
1069
1070         let major = unsafe { libc::major(stat.st_rdev) } as u64;
1071         let minor = unsafe { libc::minor(stat.st_rdev) } as u64;
1072
1073         //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
1074
1075         self.write_header(PXAR_DEVICE, std::mem::size_of::<PxarDevice>() as u64)?;
1076         self.write_item(PxarDevice { major, minor })?;
1077
1078         Ok(())
1079     }
1080
1081     // FIFO or Socket
1082     fn encode_special(&mut self, stat: &FileStat) -> Result<(), Error> {
1083         let entry = self.create_entry(&stat)?;
1084
1085         self.write_entry(entry)?;
1086
1087         Ok(())
1088     }
1089
1090     fn encode_symlink(&mut self, target: &[u8], stat: &FileStat) -> Result<(), Error> {
1091         //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
1092
1093         let entry = self.create_entry(&stat)?;
1094         self.write_entry(entry)?;
1095
1096         self.write_header(PXAR_SYMLINK, target.len() as u64)?;
1097         self.write(target)?;
1098
1099         Ok(())
1100     }
1101
1102     fn encode_hardlink(&mut self, target: &[u8], offset: u64) -> Result<(), Error> {
1103         //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
1104
1105         // Note: HARDLINK replaces an ENTRY.
1106         self.write_header(PXAR_FORMAT_HARDLINK, (target.len() as u64) + 8)?;
1107         self.write_item(offset)?;
1108         self.write(target)?;
1109
1110         Ok(())
1111     }
1112
1113     fn encode_pxar_exclude(
1114         &mut self,
1115         filefd: RawFd,
1116         stat: &FileStat,
1117         magic: i64,
1118         content: &[u8],
1119     ) -> Result<(), Error> {
1120         let mut entry = self.create_entry(&stat)?;
1121
1122         self.read_chattr(filefd, &mut entry)?;
1123         self.read_fat_attr(filefd, magic, &mut entry)?;
1124         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1125         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1126         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1127
1128         self.write_entry(entry)?;
1129         for xattr in xattrs {
1130             self.write_xattr(xattr)?;
1131         }
1132         self.write_fcaps(fcaps)?;
1133         for user in acl_access.users {
1134             self.write_acl_user(user)?;
1135         }
1136         for group in acl_access.groups {
1137             self.write_acl_group(group)?;
1138         }
1139         if let Some(group_obj) = acl_access.group_obj {
1140             self.write_acl_group_obj(group_obj)?;
1141         }
1142         if let Some(projid) = projid {
1143             self.write_quota_project_id(projid)?;
1144         }
1145
1146         let include_payload;
1147         if is_virtual_file_system(magic) {
1148             include_payload = false;
1149         } else {
1150             if let Some(set) = &self.device_set {
1151                 include_payload = set.contains(&stat.st_dev);
1152             } else {
1153                 include_payload = true;
1154             }
1155         }
1156
1157         if !include_payload {
1158             eprintln!("skip content: {:?}", self.full_path());
1159             self.write_header(PXAR_PAYLOAD, 0)?;
1160             return Ok(());
1161         }
1162
1163         let size = content.len();
1164         self.write_header(PXAR_PAYLOAD, size as u64)?;
1165         self.writer.write_all(content)?;
1166         self.writer_pos += size;
1167
1168         Ok(())
1169     }
1170
1171     // the report_XXX method may raise and error - depending on encoder configuration
1172
1173     fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
1174         eprintln!("WARNING: detected vanished file {:?}", path);
1175
1176         Ok(())
1177     }
1178 }
1179
1180 // If there is a match, an updated MatchPattern list to pass to the matched child is returned.
1181 fn match_filename(
1182     filename: &CStr,
1183     stat: &FileStat,
1184     match_pattern: &Vec<MatchPattern>,
1185 ) -> Result<(MatchType, Vec<MatchPattern>), Error> {
1186     let mut child_pattern = Vec::new();
1187     let mut match_state = MatchType::None;
1188
1189     for pattern in match_pattern {
1190         match pattern.matches_filename(filename, is_directory(&stat))? {
1191             MatchType::None => {}
1192             MatchType::Positive => match_state = MatchType::Positive,
1193             MatchType::Negative => match_state = MatchType::Negative,
1194             match_type => {
1195                 if match_state != MatchType::Positive && match_state != MatchType::Negative {
1196                     match_state = match_type;
1197                 }
1198                 child_pattern.push(pattern.get_rest_pattern());
1199             }
1200         }
1201     }
1202
1203     Ok((match_state, child_pattern))
1204 }
1205
1206 fn errno_is_unsupported(errno: Errno) -> bool {
1207     match errno {
1208         Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => true,
1209         _ => false,
1210     }
1211 }
1212
1213 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
1214     let mut fs_stat = std::mem::MaybeUninit::uninit();
1215     let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
1216     Errno::result(res)?;
1217     let fs_stat = unsafe { fs_stat.assume_init() };
1218
1219     Ok(fs_stat.f_type)
1220 }
1221
1222 #[inline(always)]
1223 pub fn is_temporary_file_system(magic: i64) -> bool {
1224     use proxmox::sys::linux::magic::*;
1225     magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
1226 }
1227
1228 pub fn is_virtual_file_system(magic: i64) -> bool {
1229     use proxmox::sys::linux::magic::*;
1230
1231     match magic {
1232         BINFMTFS_MAGIC |
1233         CGROUP2_SUPER_MAGIC |
1234         CGROUP_SUPER_MAGIC |
1235         CONFIGFS_MAGIC |
1236         DEBUGFS_MAGIC |
1237         DEVPTS_SUPER_MAGIC |
1238         EFIVARFS_MAGIC |
1239         FUSE_CTL_SUPER_MAGIC |
1240         HUGETLBFS_MAGIC |
1241         MQUEUE_MAGIC |
1242         NFSD_MAGIC |
1243         PROC_SUPER_MAGIC |
1244         PSTOREFS_MAGIC |
1245         RPCAUTH_GSSMAGIC |
1246         SECURITYFS_MAGIC |
1247         SELINUX_MAGIC |
1248         SMACK_MAGIC |
1249         SYSFS_MAGIC => true,
1250         _ => false
1251     }
1252 }