src/pxar/encoder.rs

   1 //! *pxar* format encoder.
   2 //!
   3 //! This module contain the code to generate *pxar* archive files.
   4
   5 use failure::*;
   6 use endian_trait::Endian;
   7 use std::collections::{HashSet, HashMap};
   8
   9 use super::flags;
  10 use super::format_definition::*;
  11 use super::binary_search_tree::*;
  12 use super::helper::*;
  13 use super::match_pattern::*;
  14 use crate::tools::fs;
  15 use crate::tools::acl;
  16 use crate::tools::xattr;
  17
  18 use std::io::Write;
  19 use std::os::unix::io::AsRawFd;
  20 use std::os::unix::ffi::OsStrExt;
  21 use std::os::unix::io::RawFd;
  22 use std::path::{Path, PathBuf};
  23
  24 use std::ffi::CStr;
  25
  26 use nix::NixPath;
  27 use nix::fcntl::OFlag;
  28 use nix::sys::stat::Mode;
  29 use nix::errno::Errno;
  30 use nix::sys::stat::FileStat;
  31
  32 use proxmox::tools::vec;
  33
  34 /// The format requires to build sorted directory lookup tables in
  35 /// memory, so we restrict the number of allowed entries to limit
  36 /// maximum memory usage.
  37 pub const MAX_DIRECTORY_ENTRIES: usize = 256*1024;
  38
  39 #[derive(Eq, PartialEq, Hash)]
  40 struct HardLinkInfo {
  41     st_dev: u64,
  42     st_ino: u64,
  43 }
  44
  45 pub struct Encoder<'a, W: Write> {
  46     base_path: PathBuf,
  47     relative_path: PathBuf,
  48     writer: &'a mut W,
  49     writer_pos: usize,
  50     _size: usize,
  51     file_copy_buffer: Vec<u8>,
  52     device_set: Option<HashSet<u64>>,
  53     verbose: bool,
  54     // Flags set by the user
  55     feature_flags: u64,
  56     // Flags signaling features supported by the filesystem
  57     fs_feature_flags: u64,
  58     hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
  59 }
  60
  61 impl <'a, W: Write> Encoder<'a, W> {
  62
  63     // used for error reporting
  64     fn full_path(&self) ->  PathBuf {
  65         self.base_path.join(&self.relative_path)
  66     }
  67
  68     /// Create archive, write result data to ``writer``.
  69     ///
  70     /// The ``device_set`` can be use used to limit included mount points.
  71     ///
  72     /// - ``None``: include all mount points
  73     /// - ``Some(set)``: only include devices listed in this set (the
  74     ///   root path device is automathically added to this list, so
  75     ///   you can pass an empty set if you want to archive a single
  76     ///   mount point.)
  77     pub fn encode(
  78         path: PathBuf,
  79         dir: &mut nix::dir::Dir,
  80         writer: &'a mut W,
  81         device_set: Option<HashSet<u64>>,
  82         verbose: bool,
  83         skip_lost_and_found: bool, // fixme: should be a feature flag ??
  84         feature_flags: u64,
  85     ) -> Result<(), Error> {
  86
  87         const FILE_COPY_BUFFER_SIZE: usize = 1024*1024;
  88
  89         let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
  90         unsafe { file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE); }
  91
  92
  93         // todo: use scandirat??
  94
  95         let dir_fd = dir.as_raw_fd();
  96         let stat = nix::sys::stat::fstat(dir_fd)
  97             .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
  98
  99         if !is_directory(&stat) {
 100             bail!("got unexpected file type {:?} (not a directory)", path);
 101         }
 102
 103         let mut device_set = device_set.clone();
 104         if let Some(ref mut set) = device_set {
 105             set.insert(stat.st_dev);
 106         }
 107
 108         let magic = detect_fs_type(dir_fd)?;
 109
 110         if is_virtual_file_system(magic) {
 111             bail!("backup virtual file systems is disabled!");
 112         }
 113
 114         let fs_feature_flags = flags::feature_flags_from_magic(magic);
 115
 116         let mut me = Self {
 117             base_path: path,
 118             relative_path: PathBuf::new(),
 119             writer: writer,
 120             writer_pos: 0,
 121             _size: 0,
 122             file_copy_buffer,
 123             device_set,
 124             verbose,
 125             feature_flags,
 126             fs_feature_flags,
 127             hardlinks: HashMap::new(),
 128         };
 129
 130         if verbose { println!("{:?}", me.full_path()); }
 131
 132         let mut excludes = Vec::new();
 133         if skip_lost_and_found {
 134             excludes.push(MatchPattern::from_line(b"**/lost+found").unwrap().unwrap());
 135         }
 136         me.encode_dir(dir, &stat, magic, excludes)?;
 137
 138         Ok(())
 139     }
 140
 141     fn write(&mut self,  buf: &[u8]) -> Result<(), Error> {
 142         self.writer.write_all(buf)?;
 143         self.writer_pos += buf.len();
 144         Ok(())
 145     }
 146
 147     fn write_item<T: Endian>(&mut self, item: T) ->  Result<(), Error> {
 148
 149         let data = item.to_le();
 150
 151         let buffer = unsafe { std::slice::from_raw_parts(
 152             &data as *const T as *const u8,
 153             std::mem::size_of::<T>()
 154         )};
 155
 156         self.write(buffer)?;
 157
 158         Ok(())
 159     }
 160
 161     fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
 162         self.writer.write_all(&self.file_copy_buffer[..size])?;
 163         self.writer_pos += size;
 164         Ok(())
 165     }
 166
 167     fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
 168
 169         let size = size + (std::mem::size_of::<PxarHeader>() as u64);
 170         self.write_item(PxarHeader { size, htype })?;
 171
 172         Ok(())
 173     }
 174
 175     fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
 176
 177         let buffer = name.to_bytes_with_nul();
 178         self.write_header(PXAR_FILENAME, buffer.len() as u64)?;
 179         self.write(buffer)?;
 180
 181         Ok(())
 182     }
 183
 184     fn create_entry(&self, stat: &FileStat) -> Result<PxarEntry, Error> {
 185
 186         let mode = if is_symlink(&stat) {
 187             (libc::S_IFLNK | 0o777) as u64
 188         } else {
 189             (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
 190         };
 191
 192         let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
 193         if mtime < 0 {
 194             bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
 195         }
 196
 197
 198         let entry = PxarEntry {
 199             mode: mode,
 200             flags: 0,
 201             uid: stat.st_uid,
 202             gid: stat.st_gid,
 203             mtime: mtime as u64,
 204         };
 205
 206         Ok(entry)
 207     }
 208
 209     fn read_chattr(&self, fd: RawFd, entry: &mut PxarEntry) -> Result<(), Error> {
 210
 211         let mut attr: usize = 0;
 212
 213         let res = unsafe { fs::read_attr_fd(fd, &mut attr)};
 214         if let Err(err) = res {
 215             if let nix::Error::Sys(errno) = err {
 216                 if errno_is_unsupported(errno) { return Ok(()) };
 217             }
 218             bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
 219         }
 220
 221         let flags = flags::feature_flags_from_chattr(attr as u32);
 222         entry.flags = entry.flags | flags;
 223
 224         Ok(())
 225     }
 226
 227     fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut PxarEntry) -> Result<(), Error> {
 228         use proxmox::sys::linux::magic::*;
 229
 230         if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC {
 231             return Ok(());
 232         }
 233
 234         let mut attr: u32 = 0;
 235
 236         let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr)};
 237         if let Err(err) = res {
 238             if let nix::Error::Sys(errno) = err {
 239                 if errno_is_unsupported(errno) { return Ok(()) };
 240             }
 241             bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
 242         }
 243
 244         let flags = flags::feature_flags_from_fat_attr(attr);
 245         entry.flags = entry.flags | flags;
 246
 247         Ok(())
 248     }
 249
 250     /// True if all of the given feature flags are set in the Encoder, false otherwise
 251     fn has_features(&self, feature_flags: u64) -> bool {
 252         (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
 253     }
 254
 255     /// True if at least one of the given feature flags is set in the Encoder, false otherwise
 256     fn has_some_features(&self, feature_flags: u64) -> bool {
 257         (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
 258     }
 259
 260     fn read_xattrs(&self, fd: RawFd, stat: &FileStat) -> Result<(Vec<PxarXAttr>, Option<PxarFCaps>), Error> {
 261         let mut xattrs = Vec::new();
 262         let mut fcaps = None;
 263
 264         let flags = flags::WITH_XATTRS | flags::WITH_FCAPS;
 265         if !self.has_some_features(flags) {
 266             return Ok((xattrs, fcaps));
 267         }
 268         // Should never be called on symlinks, just in case check anyway
 269         if is_symlink(&stat) {
 270             return Ok((xattrs, fcaps));
 271         }
 272
 273         let xattr_names = match xattr::flistxattr(fd) {
 274             Ok(names) => names,
 275             // Do not bail if the underlying endpoint does not supports xattrs
 276             Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
 277             // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
 278             Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
 279             Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 280         };
 281
 282         for name in xattr_names.split(|c| *c == b'\0') {
 283             // Only extract the relevant extended attributes
 284             if !xattr::is_valid_xattr_name(&name) {
 285                 continue;
 286             }
 287
 288             let value = match xattr::fgetxattr(fd, name) {
 289                 Ok(value) => value,
 290                 // Vanished between flistattr and getxattr, this is ok, silently ignore
 291                 Err(Errno::ENODATA) => continue,
 292                 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 293             };
 294
 295             if xattr::is_security_capability(&name) {
 296                 if self.has_features(flags::WITH_FCAPS) {
 297                     // fcaps are stored in own format within the archive
 298                     fcaps = Some(PxarFCaps {
 299                         data: value,
 300                     });
 301                 }
 302             } else if self.has_features(flags::WITH_XATTRS) {
 303                 xattrs.push(PxarXAttr {
 304                     name: name.to_vec(),
 305                     value: value,
 306                 });
 307             }
 308         }
 309         xattrs.sort();
 310
 311         Ok((xattrs, fcaps))
 312     }
 313
 314     fn read_acl(&self, fd: RawFd, stat: &FileStat, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 315         let ret = PxarACL {
 316             users: Vec::new(),
 317             groups: Vec::new(),
 318             group_obj: None,
 319             default: None,
 320         };
 321
 322         if !self.has_features(flags::WITH_ACL) {
 323             return Ok(ret);
 324         }
 325         if is_symlink(&stat) {
 326             return Ok(ret);
 327         }
 328         if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
 329             bail!("ACL_TYPE_DEFAULT only defined for directories.");
 330         }
 331
 332         // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
 333         // to create a path for acl_get_file(). acl_get_fd() only allows to get
 334         // ACL_TYPE_ACCESS attributes.
 335         let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
 336         let acl = match acl::ACL::get_file(&proc_path, acl_type) {
 337             Ok(acl) => acl,
 338             // Don't bail if underlying endpoint does not support acls
 339             Err(Errno::EOPNOTSUPP) => return Ok(ret),
 340             // Don't bail if the endpoint cannot carry acls
 341             Err(Errno::EBADF) => return Ok(ret),
 342             // Don't bail if there is no data
 343             Err(Errno::ENODATA) => return Ok(ret),
 344             Err(err) => bail!("error while reading ACL - {}", err),
 345         };
 346
 347         self.process_acl(acl, acl_type)
 348     }
 349
 350     fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 351         let mut acl_user = Vec::new();
 352         let mut acl_group = Vec::new();
 353         let mut acl_group_obj = None;
 354         let mut acl_default = None;
 355         let mut user_obj_permissions = None;
 356         let mut group_obj_permissions = None;
 357         let mut other_permissions = None;
 358         let mut mask_permissions = None;
 359
 360         for entry in &mut acl.entries() {
 361             let tag = entry.get_tag_type()?;
 362             let permissions = entry.get_permissions()?;
 363             match tag {
 364                 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
 365                 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
 366                 acl::ACL_OTHER => other_permissions = Some(permissions),
 367                 acl::ACL_MASK => mask_permissions = Some(permissions),
 368                 acl::ACL_USER => {
 369                     acl_user.push(PxarACLUser {
 370                         uid: entry.get_qualifier()?,
 371                         permissions: permissions,
 372                     });
 373                 },
 374                 acl::ACL_GROUP => {
 375                     acl_group.push(PxarACLGroup {
 376                         gid: entry.get_qualifier()?,
 377                         permissions: permissions,
 378                     });
 379                 },
 380                 _ => bail!("Unexpected ACL tag encountered!"),
 381             }
 382         }
 383
 384         acl_user.sort();
 385         acl_group.sort();
 386
 387         match acl_type {
 388             acl::ACL_TYPE_ACCESS => {
 389                 // The mask permissions are mapped to the stat group permissions
 390                 // in case that the ACL group permissions were set.
 391                 // Only in that case we need to store the group permissions,
 392                 // in the other cases they are identical to the stat group permissions.
 393                 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
 394                     acl_group_obj = Some(PxarACLGroupObj {
 395                         permissions: gop,
 396                     });
 397                 }
 398             },
 399             acl::ACL_TYPE_DEFAULT => {
 400                 if user_obj_permissions != None ||
 401                    group_obj_permissions != None ||
 402                    other_permissions != None ||
 403                    mask_permissions != None
 404                 {
 405                     acl_default = Some(PxarACLDefault {
 406                         // The value is set to UINT64_MAX as placeholder if one
 407                         // of the permissions is not set
 408                         user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
 409                         group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
 410                         other_permissions: other_permissions.unwrap_or(std::u64::MAX),
 411                         mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
 412                     });
 413                 }
 414             },
 415             _ => bail!("Unexpected ACL type encountered"),
 416         }
 417
 418         Ok(PxarACL {
 419             users: acl_user,
 420             groups: acl_group,
 421             group_obj: acl_group_obj,
 422             default: acl_default,
 423         })
 424     }
 425
 426     /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
 427     fn read_quota_project_id(&self, fd: RawFd, magic: i64, stat: &FileStat) -> Result<Option<PxarQuotaProjID>, Error> {
 428         if !(is_directory(&stat) || is_reg_file(&stat)) {
 429             return Ok(None);
 430         }
 431         if !self.has_features(flags::WITH_QUOTA_PROJID) {
 432             return Ok(None);
 433         }
 434
 435         use proxmox::sys::linux::magic::*;
 436
 437         match magic {
 438             EXT4_SUPER_MAGIC |
 439             XFS_SUPER_MAGIC |
 440             FUSE_SUPER_MAGIC |
 441             ZFS_SUPER_MAGIC => {
 442                 let mut fsxattr = fs::FSXAttr::default();
 443                 let res = unsafe {
 444                     fs::fs_ioc_fsgetxattr(fd, &mut fsxattr)
 445                 };
 446
 447                 // On some FUSE filesystems it can happen that ioctl is not supported.
 448                 // For these cases projid is set to 0 while the error is ignored.
 449                 if let Err(err) = res {
 450                     let errno = err.as_errno().ok_or_else(|| {
 451                         format_err!("error while reading quota project id for {:#?}", self.full_path())
 452                     })?;
 453                     if errno_is_unsupported(errno) {
 454                         return Ok(None);
 455                     } else {
 456                         bail!("error while reading quota project id for {:#?} - {}", self.full_path(), errno);
 457                     }
 458                 }
 459
 460                 let projid = fsxattr.fsx_projid as u64;
 461                 if projid == 0 {
 462                     return Ok(None);
 463                 } else {
 464                     return Ok(Some(PxarQuotaProjID { projid }));
 465                 }
 466             },
 467             _ => return Ok(None),
 468         }
 469     }
 470
 471     fn write_entry(&mut self, entry: PxarEntry) -> Result<(), Error> {
 472
 473         self.write_header(PXAR_ENTRY, std::mem::size_of::<PxarEntry>() as u64)?;
 474         self.write_item(entry)?;
 475
 476         Ok(())
 477     }
 478
 479     fn write_xattr(&mut self, xattr: PxarXAttr) -> Result<(), Error> {
 480         let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
 481         self.write_header(PXAR_XATTR, size as u64)?;
 482         self.write(xattr.name.as_slice())?;
 483         self.write(&[0])?;
 484         self.write(xattr.value.as_slice())?;
 485
 486         Ok(())
 487     }
 488
 489     fn write_fcaps(&mut self, fcaps: Option<PxarFCaps>) -> Result<(), Error> {
 490         if let Some(fcaps) = fcaps {
 491             let size = fcaps.data.len();
 492             self.write_header(PXAR_FCAPS, size as u64)?;
 493             self.write(fcaps.data.as_slice())?;
 494         }
 495
 496         Ok(())
 497     }
 498
 499     fn write_acl_user(&mut self, acl_user: PxarACLUser) -> Result<(), Error> {
 500         self.write_header(PXAR_ACL_USER,  std::mem::size_of::<PxarACLUser>() as u64)?;
 501         self.write_item(acl_user)?;
 502
 503         Ok(())
 504     }
 505
 506     fn write_acl_group(&mut self, acl_group: PxarACLGroup) -> Result<(), Error> {
 507         self.write_header(PXAR_ACL_GROUP,  std::mem::size_of::<PxarACLGroup>() as u64)?;
 508         self.write_item(acl_group)?;
 509
 510         Ok(())
 511     }
 512
 513     fn write_acl_group_obj(&mut self, acl_group_obj: PxarACLGroupObj) -> Result<(), Error> {
 514         self.write_header(PXAR_ACL_GROUP_OBJ,  std::mem::size_of::<PxarACLGroupObj>() as u64)?;
 515         self.write_item(acl_group_obj)?;
 516
 517         Ok(())
 518     }
 519
 520     fn write_acl_default(&mut self, acl_default: PxarACLDefault) -> Result<(), Error> {
 521         self.write_header(PXAR_ACL_DEFAULT,  std::mem::size_of::<PxarACLDefault>() as u64)?;
 522         self.write_item(acl_default)?;
 523
 524         Ok(())
 525     }
 526
 527     fn write_acl_default_user(&mut self, acl_default_user: PxarACLUser) -> Result<(), Error> {
 528         self.write_header(PXAR_ACL_DEFAULT_USER,  std::mem::size_of::<PxarACLUser>() as u64)?;
 529         self.write_item(acl_default_user)?;
 530
 531         Ok(())
 532     }
 533
 534     fn write_acl_default_group(&mut self, acl_default_group: PxarACLGroup) -> Result<(), Error> {
 535         self.write_header(PXAR_ACL_DEFAULT_GROUP,  std::mem::size_of::<PxarACLGroup>() as u64)?;
 536         self.write_item(acl_default_group)?;
 537
 538         Ok(())
 539     }
 540
 541     fn write_quota_project_id(&mut self, projid: PxarQuotaProjID) -> Result<(), Error> {
 542         self.write_header(PXAR_QUOTA_PROJID, std::mem::size_of::<PxarQuotaProjID>() as u64)?;
 543         self.write_item(projid)?;
 544
 545         Ok(())
 546     }
 547
 548     fn write_goodbye_table(&mut self, goodbye_offset: usize, goodbye_items: &mut [PxarGoodbyeItem]) -> Result<(), Error> {
 549
 550         goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
 551
 552         let item_count = goodbye_items.len();
 553
 554         let goodbye_table_size = (item_count + 1)*std::mem::size_of::<PxarGoodbyeItem>();
 555
 556         self.write_header(PXAR_GOODBYE, goodbye_table_size as u64)?;
 557
 558         if self.file_copy_buffer.len() < goodbye_table_size {
 559             let need = goodbye_table_size - self.file_copy_buffer.len();
 560             self.file_copy_buffer.reserve(need);
 561             unsafe { self.file_copy_buffer.set_len(self.file_copy_buffer.capacity()); }
 562         }
 563
 564         let buffer = &mut self.file_copy_buffer;
 565
 566         copy_binary_search_tree(item_count, |s, d| {
 567             let item = &goodbye_items[s];
 568             let offset = d*std::mem::size_of::<PxarGoodbyeItem>();
 569             let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
 570             dest.offset = u64::to_le(item.offset);
 571             dest.size = u64::to_le(item.size);
 572             dest.hash = u64::to_le(item.hash);
 573         });
 574
 575         // append PxarGoodbyeTail as last item
 576         let offset = item_count*std::mem::size_of::<PxarGoodbyeItem>();
 577         let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
 578         dest.offset = u64::to_le(goodbye_offset as u64);
 579         dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<PxarHeader>()) as u64);
 580         dest.hash = u64::to_le(PXAR_GOODBYE_TAIL_MARKER);
 581
 582         self.flush_copy_buffer(goodbye_table_size)?;
 583
 584         Ok(())
 585     }
 586
 587     fn encode_dir(
 588         &mut self,
 589         dir: &mut nix::dir::Dir,
 590         dir_stat: &FileStat,
 591         magic: i64,
 592         match_pattern: Vec<MatchPattern>,
 593     ) -> Result<(), Error> {
 594
 595         //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
 596
 597         let mut name_list = vec![];
 598
 599         let rawfd = dir.as_raw_fd();
 600
 601         let dir_start_pos = self.writer_pos;
 602
 603         let mut dir_entry = self.create_entry(&dir_stat)?;
 604
 605         self.read_chattr(rawfd, &mut dir_entry)?;
 606         self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
 607
 608         // for each node in the directory tree, the filesystem features are
 609         // checked based on the fs magic number.
 610         self.fs_feature_flags = flags::feature_flags_from_magic(magic);
 611
 612         let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
 613         let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
 614         let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
 615         let projid = self.read_quota_project_id(rawfd, magic, &dir_stat)?;
 616
 617         self.write_entry(dir_entry)?;
 618         for xattr in xattrs {
 619             self.write_xattr(xattr)?;
 620         }
 621         self.write_fcaps(fcaps)?;
 622
 623         for user in acl_access.users {
 624             self.write_acl_user(user)?;
 625         }
 626         for group in acl_access.groups {
 627             self.write_acl_group(group)?;
 628         }
 629         if let Some(group_obj) = acl_access.group_obj {
 630             self.write_acl_group_obj(group_obj)?;
 631         }
 632
 633         for default_user in acl_default.users {
 634             self.write_acl_default_user(default_user)?;
 635         }
 636         for default_group in acl_default.groups {
 637             self.write_acl_default_group(default_group)?;
 638         }
 639         if let Some(default) = acl_default.default {
 640             self.write_acl_default(default)?;
 641         }
 642         if let Some(projid) = projid {
 643             self.write_quota_project_id(projid)?;
 644         }
 645
 646         let include_children;
 647         if is_virtual_file_system(magic) {
 648             include_children = false;
 649         } else {
 650             if let Some(set) = &self.device_set {
 651                 include_children = set.contains(&dir_stat.st_dev);
 652             } else {
 653                 include_children = true;
 654             }
 655         }
 656
 657         // Expand the exclude match pattern inherited from the parent by local entries, if present
 658         let mut local_match_pattern = match_pattern.clone();
 659         let pxar_exclude = match MatchPattern::from_file(rawfd, ".pxarexclude") {
 660             Ok(Some((mut excludes, buffer, stat))) => {
 661                 local_match_pattern.append(&mut excludes);
 662                 Some((buffer, stat))
 663             },
 664             Ok(None) => None,
 665             Err(err) => bail!("error while reading exclude file - {}", err),
 666         };
 667
 668         if include_children {
 669             for entry in dir.iter() {
 670                 let entry =  entry.map_err(|err| {
 671                     format_err!("readir {:?} failed - {}", self.full_path(), err)
 672                 })?;
 673                 let filename = entry.file_name().to_owned();
 674
 675                 let name = filename.to_bytes_with_nul();
 676                 if name == b".\0" || name == b"..\0" {
 677                     continue;
 678                 }
 679
 680                 let stat = match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
 681                     Ok(stat) => stat,
 682                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 683                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 684                         self.report_vanished_file(&self.full_path().join(filename_osstr))?;
 685                         continue;
 686                     },
 687                     Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
 688                 };
 689
 690                 match match_filename(&filename, &stat, &local_match_pattern) {
 691                     (MatchType::Positive, _) => {
 692                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 693                         eprintln!("matched by .pxarexclude entry - skipping: {:?}", self.full_path().join(filename_osstr));
 694                     },
 695                     (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
 696                 }
 697
 698                 if name_list.len() > MAX_DIRECTORY_ENTRIES {
 699                     bail!("too many directory items in {:?} (> {})", self.full_path(), MAX_DIRECTORY_ENTRIES);
 700                 }
 701             }
 702         } else {
 703             eprintln!("skip mount point: {:?}", self.full_path());
 704         }
 705
 706         name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
 707
 708         let mut goodbye_items = vec![];
 709
 710         for (filename, stat, exclude_list) in name_list {
 711             let start_pos = self.writer_pos;
 712
 713             if filename.as_bytes() == b".pxarexclude" {
 714                 if let Some((ref content, ref stat)) = pxar_exclude {
 715                     let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
 716                         Ok(filefd) => filefd,
 717                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 718                             self.report_vanished_file(&self.full_path())?;
 719                             continue;
 720                         },
 721                         Err(err) => {
 722                             let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 723                             bail!("open file {:?} failed - {}", self.full_path().join(filename_osstr), err);
 724                         },
 725                     };
 726
 727                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 728                         detect_fs_type(filefd)?
 729                     } else {
 730                         magic
 731                     };
 732
 733                     self.write_filename(&filename)?;
 734                     self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
 735                     continue;
 736                 }
 737             }
 738
 739             self.relative_path.push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
 740
 741             if self.verbose { println!("{:?}", self.full_path()); }
 742
 743             if is_directory(&stat) {
 744
 745                 let mut dir = match nix::dir::Dir::openat(rawfd, filename.as_ref(), OFlag::O_DIRECTORY|OFlag::O_NOFOLLOW, Mode::empty()) {
 746                     Ok(dir) => dir,
 747                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 748                         self.report_vanished_file(&self.full_path())?;
 749                         continue; // fixme!!
 750                     },
 751                     Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
 752                 };
 753
 754                 let child_magic = if dir_stat.st_dev != stat.st_dev {
 755                     detect_fs_type(dir.as_raw_fd())?
 756                 } else {
 757                     magic
 758                 };
 759
 760                 self.write_filename(&filename)?;
 761                 self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
 762
 763             } else if is_reg_file(&stat) {
 764
 765                 let mut hardlink_target = None;
 766
 767                 if stat.st_nlink > 1 {
 768                     let link_info = HardLinkInfo { st_dev: stat.st_dev, st_ino: stat.st_ino };
 769                     hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
 770                         let mut target = v.clone().into_os_string();
 771                         target.push("\0"); // add Nul byte
 772                         (target, (start_pos as u64) - offset)
 773                     });
 774                     if hardlink_target == None {
 775                         self.hardlinks.insert(link_info, (self.relative_path.clone(), start_pos as u64));
 776                     }
 777                 }
 778
 779                 if let Some((target, offset)) = hardlink_target {
 780
 781                     self.write_filename(&filename)?;
 782                     self.encode_hardlink(target.as_bytes(), offset)?;
 783
 784                 } else {
 785
 786                     let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
 787                         Ok(filefd) => filefd,
 788                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 789                             self.report_vanished_file(&self.full_path())?;
 790                             continue;
 791                         },
 792                         Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
 793                     };
 794
 795                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 796                         detect_fs_type(filefd)?
 797                     } else {
 798                         magic
 799                     };
 800
 801                     self.write_filename(&filename)?;
 802                     let res = self.encode_file(filefd, &stat, child_magic);
 803                     let _ = nix::unistd::close(filefd); // ignore close errors
 804                     res?;
 805                 }
 806
 807             } else if is_symlink(&stat) {
 808                 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
 809
 810                 let res = filename.with_nix_path(|cstr| {
 811                     unsafe { libc::readlinkat(rawfd, cstr.as_ptr(), buffer.as_mut_ptr() as *mut libc::c_char, buffer.len()-1) }
 812                 })?;
 813
 814                 match Errno::result(res) {
 815                     Ok(len) => {
 816                         buffer[len as usize] = 0u8; // add Nul byte
 817                         self.write_filename(&filename)?;
 818                         self.encode_symlink(&buffer[..((len+1) as usize)], &stat)?
 819                     }
 820                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 821                         self.report_vanished_file(&self.full_path())?;
 822                         continue;
 823                     }
 824                     Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
 825                 }
 826             } else if is_block_dev(&stat) || is_char_dev(&stat) {
 827                 if self.has_features(flags::WITH_DEVICE_NODES) {
 828                     self.write_filename(&filename)?;
 829                     self.encode_device(&stat)?;
 830                 } else {
 831                     eprintln!("skip device node: {:?}", self.full_path());
 832                 }
 833             } else if is_fifo(&stat) {
 834                 if self.has_features(flags::WITH_FIFOS) {
 835                     self.write_filename(&filename)?;
 836                     self.encode_special(&stat)?;
 837                 } else {
 838                     eprintln!("skip fifo: {:?}", self.full_path());
 839                 }
 840             } else if is_socket(&stat) {
 841                 if self.has_features(flags::WITH_SOCKETS) {
 842                     self.write_filename(&filename)?;
 843                     self.encode_special(&stat)?;
 844                 } else {
 845                     eprintln!("skip socket: {:?}", self.full_path());
 846                 }
 847             } else {
 848                 bail!("unsupported file type (mode {:o} {:?})", stat.st_mode, self.full_path());
 849             }
 850
 851             let end_pos = self.writer_pos;
 852
 853             goodbye_items.push(PxarGoodbyeItem {
 854                 offset: start_pos as u64,
 855                 size: (end_pos - start_pos) as u64,
 856                 hash: compute_goodbye_hash(filename.to_bytes()),
 857             });
 858
 859             self.relative_path.pop();
 860         }
 861
 862         //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
 863
 864         // fixup goodby item offsets
 865         let goodbye_start = self.writer_pos as u64;
 866         for item in &mut goodbye_items {
 867             item.offset = goodbye_start - item.offset;
 868         }
 869
 870         let goodbye_offset = self.writer_pos - dir_start_pos;
 871
 872         self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
 873
 874         //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
 875         Ok(())
 876     }
 877
 878     fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64)  -> Result<(), Error> {
 879
 880         //println!("encode_file: {:?}", self.full_path());
 881
 882         let mut entry = self.create_entry(&stat)?;
 883
 884         self.read_chattr(filefd, &mut entry)?;
 885         self.read_fat_attr(filefd, magic, &mut entry)?;
 886         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
 887         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
 888         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
 889
 890         self.write_entry(entry)?;
 891         for xattr in xattrs {
 892             self.write_xattr(xattr)?;
 893         }
 894         self.write_fcaps(fcaps)?;
 895         for user in acl_access.users {
 896             self.write_acl_user(user)?;
 897         }
 898         for group in acl_access.groups {
 899             self.write_acl_group(group)?;
 900         }
 901         if let Some(group_obj) = acl_access.group_obj {
 902             self.write_acl_group_obj(group_obj)?;
 903         }
 904         if let Some(projid) = projid {
 905             self.write_quota_project_id(projid)?;
 906         }
 907
 908         let include_payload;
 909         if is_virtual_file_system(magic) {
 910             include_payload = false;
 911         } else {
 912             if let Some(ref set) = &self.device_set {
 913                 include_payload = set.contains(&stat.st_dev);
 914             } else {
 915                 include_payload = true;
 916             }
 917         }
 918
 919         if !include_payload {
 920             eprintln!("skip content: {:?}", self.full_path());
 921             self.write_header(PXAR_PAYLOAD, 0)?;
 922             return Ok(());
 923         }
 924
 925         let size = stat.st_size as u64;
 926
 927         self.write_header(PXAR_PAYLOAD, size)?;
 928
 929         let mut pos: u64 = 0;
 930         loop {
 931             let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
 932                 Ok(n) => n,
 933                 Err(nix::Error::Sys(Errno::EINTR)) => continue /* try again */,
 934                 Err(err) =>  bail!("read {:?} failed - {}", self.full_path(), err),
 935             };
 936             if n == 0 { // EOF
 937                 if pos != size {
 938                     // Note:: casync format cannot handle that
 939                     bail!("detected shrinked file {:?} ({} < {})", self.full_path(), pos, size);
 940                 }
 941                 break;
 942             }
 943
 944             let mut next = pos + (n as u64);
 945
 946             if next > size { next = size; }
 947
 948             let count = (next - pos) as usize;
 949
 950             self.flush_copy_buffer(count)?;
 951
 952             pos = next;
 953
 954             if pos >= size { break; }
 955         }
 956
 957         Ok(())
 958     }
 959
 960     fn encode_device(&mut self, stat: &FileStat)  -> Result<(), Error> {
 961
 962         let entry = self.create_entry(&stat)?;
 963
 964         self.write_entry(entry)?;
 965
 966         let major = unsafe { libc::major(stat.st_rdev) } as u64;
 967         let minor = unsafe { libc::minor(stat.st_rdev) } as u64;
 968
 969         //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
 970
 971         self.write_header(PXAR_DEVICE, std::mem::size_of::<PxarDevice>() as u64)?;
 972         self.write_item(PxarDevice { major, minor })?;
 973
 974         Ok(())
 975     }
 976
 977     // FIFO or Socket
 978     fn encode_special(&mut self, stat: &FileStat)  -> Result<(), Error> {
 979
 980         let entry = self.create_entry(&stat)?;
 981
 982         self.write_entry(entry)?;
 983
 984         Ok(())
 985     }
 986
 987     fn encode_symlink(&mut self, target: &[u8], stat: &FileStat)  -> Result<(), Error> {
 988
 989         //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
 990
 991         let entry = self.create_entry(&stat)?;
 992         self.write_entry(entry)?;
 993
 994         self.write_header(PXAR_SYMLINK, target.len() as u64)?;
 995         self.write(target)?;
 996
 997         Ok(())
 998     }
 999
1000     fn encode_hardlink(&mut self, target: &[u8], offset: u64)  -> Result<(), Error> {
1001
1002         //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
1003
1004         // Note: HARDLINK replaces an ENTRY.
1005         self.write_header(PXAR_FORMAT_HARDLINK, (target.len() as u64) + 8)?;
1006         self.write_item(offset)?;
1007         self.write(target)?;
1008
1009         Ok(())
1010     }
1011
1012     fn encode_pxar_exclude(&mut self, filefd: RawFd, stat: &FileStat, magic: i64, content: &[u8]) -> Result<(), Error> {
1013         let mut entry = self.create_entry(&stat)?;
1014
1015         self.read_chattr(filefd, &mut entry)?;
1016         self.read_fat_attr(filefd, magic, &mut entry)?;
1017         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1018         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1019         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1020
1021         self.write_entry(entry)?;
1022         for xattr in xattrs {
1023             self.write_xattr(xattr)?;
1024         }
1025         self.write_fcaps(fcaps)?;
1026         for user in acl_access.users {
1027             self.write_acl_user(user)?;
1028         }
1029         for group in acl_access.groups {
1030             self.write_acl_group(group)?;
1031         }
1032         if let Some(group_obj) = acl_access.group_obj {
1033             self.write_acl_group_obj(group_obj)?;
1034         }
1035         if let Some(projid) = projid {
1036             self.write_quota_project_id(projid)?;
1037         }
1038
1039         let include_payload;
1040         if is_virtual_file_system(magic) {
1041             include_payload = false;
1042         } else {
1043             if let Some(set) = &self.device_set {
1044                 include_payload = set.contains(&stat.st_dev);
1045             } else {
1046                 include_payload = true;
1047             }
1048         }
1049
1050         if !include_payload {
1051             eprintln!("skip content: {:?}", self.full_path());
1052             self.write_header(PXAR_PAYLOAD, 0)?;
1053             return Ok(());
1054         }
1055
1056         let size = content.len();
1057         self.write_header(PXAR_PAYLOAD, size as u64)?;
1058         self.writer.write_all(content)?;
1059         self.writer_pos += size;
1060
1061         Ok(())
1062     }
1063
1064     // the report_XXX method may raise and error - depending on encoder configuration
1065
1066     fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
1067
1068         eprintln!("WARNING: detected vanished file {:?}", path);
1069
1070         Ok(())
1071     }
1072 }
1073
1074 // If there is a match, an updated MatchPattern list to pass to the matched child is returned.
1075 fn match_filename(
1076     filename: &CStr,
1077     stat: &FileStat,
1078     match_pattern: &Vec<MatchPattern>
1079 ) ->  (MatchType, Vec<MatchPattern>) {
1080     let mut child_pattern = Vec::new();
1081     let mut match_state = MatchType::None;
1082
1083     for pattern in match_pattern {
1084         match pattern.matches_filename(filename, is_directory(&stat)) {
1085             MatchType::None =>  {},
1086             MatchType::Positive =>  match_state = MatchType::Positive,
1087             MatchType::Negative =>  match_state = MatchType::Negative,
1088             match_type => {
1089                 if match_state != MatchType::Positive && match_state != MatchType::Negative {
1090                     match_state = match_type;
1091                 }
1092                 child_pattern.push(pattern.get_rest_pattern());
1093             },
1094         }
1095     }
1096
1097     (match_state, child_pattern)
1098 }
1099
1100 fn errno_is_unsupported(errno: Errno) -> bool {
1101
1102     match errno {
1103         Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => {
1104             true
1105         }
1106         _ => false,
1107     }
1108 }
1109
1110 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
1111     let mut fs_stat: libc::statfs = unsafe { std::mem::uninitialized() };
1112     let res = unsafe { libc::fstatfs(fd, &mut fs_stat) };
1113     Errno::result(res)?;
1114
1115     Ok(fs_stat.f_type)
1116 }
1117
1118 #[inline(always)]
1119 pub fn is_temporary_file_system(magic: i64) -> bool {
1120     use proxmox::sys::linux::magic::*;
1121     magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
1122 }
1123
1124 pub fn is_virtual_file_system(magic: i64) -> bool {
1125     use proxmox::sys::linux::magic::*;
1126
1127     match magic {
1128         BINFMTFS_MAGIC |
1129         CGROUP2_SUPER_MAGIC |
1130         CGROUP_SUPER_MAGIC |
1131         CONFIGFS_MAGIC |
1132         DEBUGFS_MAGIC |
1133         DEVPTS_SUPER_MAGIC |
1134         EFIVARFS_MAGIC |
1135         FUSE_CTL_SUPER_MAGIC |
1136         HUGETLBFS_MAGIC |
1137         MQUEUE_MAGIC |
1138         NFSD_MAGIC |
1139         PROC_SUPER_MAGIC |
1140         PSTOREFS_MAGIC |
1141         RPCAUTH_GSSMAGIC |
1142         SECURITYFS_MAGIC |
1143         SELINUX_MAGIC |
1144         SMACK_MAGIC |
1145         SYSFS_MAGIC => true,
1146         _ => false
1147     }
1148 }