src/pxar/encoder.rs

   1 //! *pxar* format encoder.
   2 //!
   3 //! This module contain the code to generate *pxar* archive files.
   4
   5 use failure::*;
   6 use endian_trait::Endian;
   7 use std::collections::{HashSet, HashMap};
   8
   9 use super::flags;
  10 use super::format_definition::*;
  11 use super::binary_search_tree::*;
  12 use super::helper::*;
  13 use super::exclude_pattern::*;
  14 use crate::tools::fs;
  15 use crate::tools::acl;
  16 use crate::tools::xattr;
  17
  18 use std::io::Write;
  19 use std::os::unix::io::AsRawFd;
  20 use std::os::unix::ffi::OsStrExt;
  21 use std::os::unix::io::RawFd;
  22 use std::path::{Path, PathBuf};
  23
  24 use std::ffi::CStr;
  25
  26 use nix::NixPath;
  27 use nix::fcntl::OFlag;
  28 use nix::sys::stat::Mode;
  29 use nix::errno::Errno;
  30 use nix::sys::stat::FileStat;
  31
  32 use proxmox::tools::vec;
  33
  34 /// The format requires to build sorted directory lookup tables in
  35 /// memory, so we restrict the number of allowed entries to limit
  36 /// maximum memory usage.
  37 pub const MAX_DIRECTORY_ENTRIES: usize = 256*1024;
  38
  39 #[derive(Eq, PartialEq, Hash)]
  40 struct HardLinkInfo {
  41     st_dev: u64,
  42     st_ino: u64,
  43 }
  44
  45 pub struct Encoder<'a, W: Write> {
  46     base_path: PathBuf,
  47     relative_path: PathBuf,
  48     writer: &'a mut W,
  49     writer_pos: usize,
  50     _size: usize,
  51     file_copy_buffer: Vec<u8>,
  52     device_set: Option<HashSet<u64>>,
  53     verbose: bool,
  54     // Flags set by the user
  55     feature_flags: u64,
  56     // Flags signaling features supported by the filesystem
  57     fs_feature_flags: u64,
  58     hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
  59 }
  60
  61 impl <'a, W: Write> Encoder<'a, W> {
  62
  63     // used for error reporting
  64     fn full_path(&self) ->  PathBuf {
  65         self.base_path.join(&self.relative_path)
  66     }
  67
  68     /// Create archive, write result data to ``writer``.
  69     ///
  70     /// The ``device_set`` can be use used to limit included mount points.
  71     ///
  72     /// - ``None``: include all mount points
  73     /// - ``Some(set)``: only include devices listed in this set (the
  74     ///   root path device is automathically added to this list, so
  75     ///   you can pass an empty set if you want to archive a single
  76     ///   mount point.)
  77     pub fn encode(
  78         path: PathBuf,
  79         dir: &mut nix::dir::Dir,
  80         writer: &'a mut W,
  81         device_set: Option<HashSet<u64>>,
  82         verbose: bool,
  83         skip_lost_and_found: bool, // fixme: should be a feature flag ??
  84         feature_flags: u64,
  85     ) -> Result<(), Error> {
  86
  87         const FILE_COPY_BUFFER_SIZE: usize = 1024*1024;
  88
  89         let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
  90         unsafe { file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE); }
  91
  92
  93         // todo: use scandirat??
  94
  95         let dir_fd = dir.as_raw_fd();
  96         let stat = nix::sys::stat::fstat(dir_fd)
  97             .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
  98
  99         if !is_directory(&stat) {
 100             bail!("got unexpected file type {:?} (not a directory)", path);
 101         }
 102
 103         let mut device_set = device_set.clone();
 104         if let Some(ref mut set) = device_set {
 105             set.insert(stat.st_dev);
 106         }
 107
 108         let magic = detect_fs_type(dir_fd)?;
 109
 110         if is_virtual_file_system(magic) {
 111             bail!("backup virtual file systems is disabled!");
 112         }
 113
 114         let fs_feature_flags = flags::feature_flags_from_magic(magic);
 115
 116         let mut me = Self {
 117             base_path: path,
 118             relative_path: PathBuf::new(),
 119             writer: writer,
 120             writer_pos: 0,
 121             _size: 0,
 122             file_copy_buffer,
 123             device_set,
 124             verbose,
 125             feature_flags,
 126             fs_feature_flags,
 127             hardlinks: HashMap::new(),
 128         };
 129
 130         if verbose { println!("{:?}", me.full_path()); }
 131
 132         let mut excludes = Vec::new();
 133         if skip_lost_and_found {
 134             excludes.push(PxarExcludePattern::from_line(b"**/lost+found").unwrap().unwrap());
 135         }
 136         me.encode_dir(dir, &stat, magic, excludes)?;
 137
 138         Ok(())
 139     }
 140
 141     fn write(&mut self,  buf: &[u8]) -> Result<(), Error> {
 142         self.writer.write_all(buf)?;
 143         self.writer_pos += buf.len();
 144         Ok(())
 145     }
 146
 147     fn write_item<T: Endian>(&mut self, item: T) ->  Result<(), Error> {
 148
 149         let data = item.to_le();
 150
 151         let buffer = unsafe { std::slice::from_raw_parts(
 152             &data as *const T as *const u8,
 153             std::mem::size_of::<T>()
 154         )};
 155
 156         self.write(buffer)?;
 157
 158         Ok(())
 159     }
 160
 161     fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
 162         self.writer.write_all(&self.file_copy_buffer[..size])?;
 163         self.writer_pos += size;
 164         Ok(())
 165     }
 166
 167     fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
 168
 169         let size = size + (std::mem::size_of::<CaFormatHeader>() as u64);
 170         self.write_item(CaFormatHeader { size, htype })?;
 171
 172         Ok(())
 173     }
 174
 175     fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
 176
 177         let buffer = name.to_bytes_with_nul();
 178         self.write_header(CA_FORMAT_FILENAME, buffer.len() as u64)?;
 179         self.write(buffer)?;
 180
 181         Ok(())
 182     }
 183
 184     fn create_entry(&self, stat: &FileStat) -> Result<CaFormatEntry, Error> {
 185
 186         let mode = if is_symlink(&stat) {
 187             (libc::S_IFLNK | 0o777) as u64
 188         } else {
 189             (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
 190         };
 191
 192         let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
 193         if mtime < 0 {
 194             bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
 195         }
 196
 197
 198         let entry = CaFormatEntry {
 199             mode: mode,
 200             flags: 0,
 201             uid: stat.st_uid as u64,
 202             gid: stat.st_gid as u64,
 203             mtime: mtime as u64,
 204         };
 205
 206         Ok(entry)
 207     }
 208
 209     fn read_chattr(&self, fd: RawFd, entry: &mut CaFormatEntry) -> Result<(), Error> {
 210
 211         let mut attr: usize = 0;
 212
 213         let res = unsafe { fs::read_attr_fd(fd, &mut attr)};
 214         if let Err(err) = res {
 215             if let nix::Error::Sys(errno) = err {
 216                 if errno_is_unsupported(errno) { return Ok(()) };
 217             }
 218             bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
 219         }
 220
 221         let flags = flags::feature_flags_from_chattr(attr as u32);
 222         entry.flags = entry.flags | flags;
 223
 224         Ok(())
 225     }
 226
 227     fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut CaFormatEntry) -> Result<(), Error> {
 228         use fs::magic::*;
 229
 230         if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC {
 231             return Ok(());
 232         }
 233
 234         let mut attr: u32 = 0;
 235
 236         let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr)};
 237         if let Err(err) = res {
 238             if let nix::Error::Sys(errno) = err {
 239                 if errno_is_unsupported(errno) { return Ok(()) };
 240             }
 241             bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
 242         }
 243
 244         let flags = flags::feature_flags_from_fat_attr(attr);
 245         entry.flags = entry.flags | flags;
 246
 247         Ok(())
 248     }
 249
 250     /// True if all of the given feature flags are set in the Encoder, false otherwise
 251     fn has_features(&self, feature_flags: u64) -> bool {
 252         (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
 253     }
 254
 255     /// True if at least one of the given feature flags is set in the Encoder, false otherwise
 256     fn has_some_features(&self, feature_flags: u64) -> bool {
 257         (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
 258     }
 259
 260     fn read_xattrs(&self, fd: RawFd, stat: &FileStat) -> Result<(Vec<CaFormatXAttr>, Option<CaFormatFCaps>), Error> {
 261         let mut xattrs = Vec::new();
 262         let mut fcaps = None;
 263
 264         let flags = flags::WITH_XATTRS | flags::WITH_FCAPS;
 265         if !self.has_some_features(flags) {
 266             return Ok((xattrs, fcaps));
 267         }
 268         // Should never be called on symlinks, just in case check anyway
 269         if is_symlink(&stat) {
 270             return Ok((xattrs, fcaps));
 271         }
 272
 273         let xattr_names = match xattr::flistxattr(fd) {
 274             Ok(names) => names,
 275             // Do not bail if the underlying endpoint does not supports xattrs
 276             Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
 277             // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
 278             Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
 279             Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 280         };
 281
 282         for name in xattr_names.split(|c| *c == b'\0') {
 283             // Only extract the relevant extended attributes
 284             if !xattr::is_valid_xattr_name(&name) {
 285                 continue;
 286             }
 287
 288             let value = match xattr::fgetxattr(fd, name) {
 289                 Ok(value) => value,
 290                 // Vanished between flistattr and getxattr, this is ok, silently ignore
 291                 Err(Errno::ENODATA) => continue,
 292                 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 293             };
 294
 295             if xattr::is_security_capability(&name) {
 296                 if self.has_features(flags::WITH_FCAPS) {
 297                     // fcaps are stored in own format within the archive
 298                     fcaps = Some(CaFormatFCaps {
 299                         data: value,
 300                     });
 301                 }
 302             } else if self.has_features(flags::WITH_XATTRS) {
 303                 xattrs.push(CaFormatXAttr {
 304                     name: name.to_vec(),
 305                     value: value,
 306                 });
 307             }
 308         }
 309         xattrs.sort();
 310
 311         Ok((xattrs, fcaps))
 312     }
 313
 314     fn read_acl(&self, fd: RawFd, stat: &FileStat, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 315         let ret = PxarACL {
 316             users: Vec::new(),
 317             groups: Vec::new(),
 318             group_obj: None,
 319             default: None,
 320         };
 321
 322         if !self.has_features(flags::WITH_ACL) {
 323             return Ok(ret);
 324         }
 325         if is_symlink(&stat) {
 326             return Ok(ret);
 327         }
 328         if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
 329             bail!("ACL_TYPE_DEFAULT only defined for directories.");
 330         }
 331
 332         // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
 333         // to create a path for acl_get_file(). acl_get_fd() only allows to get
 334         // ACL_TYPE_ACCESS attributes.
 335         let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
 336         let acl = match acl::ACL::get_file(&proc_path, acl_type) {
 337             Ok(acl) => acl,
 338             // Don't bail if underlying endpoint does not support acls
 339             Err(Errno::EOPNOTSUPP) => return Ok(ret),
 340             // Don't bail if the endpoint cannot carry acls
 341             Err(Errno::EBADF) => return Ok(ret),
 342             // Don't bail if there is no data
 343             Err(Errno::ENODATA) => return Ok(ret),
 344             Err(err) => bail!("error while reading ACL - {}", err),
 345         };
 346
 347         self.process_acl(acl, acl_type)
 348     }
 349
 350     fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 351         let mut acl_user = Vec::new();
 352         let mut acl_group = Vec::new();
 353         let mut acl_group_obj = None;
 354         let mut acl_default = None;
 355         let mut user_obj_permissions = None;
 356         let mut group_obj_permissions = None;
 357         let mut other_permissions = None;
 358         let mut mask_permissions = None;
 359
 360         for entry in &mut acl.entries() {
 361             let tag = entry.get_tag_type()?;
 362             let permissions = entry.get_permissions()?;
 363             match tag {
 364                 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
 365                 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
 366                 acl::ACL_OTHER => other_permissions = Some(permissions),
 367                 acl::ACL_MASK => mask_permissions = Some(permissions),
 368                 acl::ACL_USER => {
 369                     acl_user.push(CaFormatACLUser {
 370                         uid: entry.get_qualifier()?,
 371                         permissions: permissions,
 372                     });
 373                 },
 374                 acl::ACL_GROUP => {
 375                     acl_group.push(CaFormatACLGroup {
 376                         gid: entry.get_qualifier()?,
 377                         permissions: permissions,
 378                     });
 379                 },
 380                 _ => bail!("Unexpected ACL tag encountered!"),
 381             }
 382         }
 383
 384         acl_user.sort();
 385         acl_group.sort();
 386
 387         match acl_type {
 388             acl::ACL_TYPE_ACCESS => {
 389                 // The mask permissions are mapped to the stat group permissions
 390                 // in case that the ACL group permissions were set.
 391                 // Only in that case we need to store the group permissions,
 392                 // in the other cases they are identical to the stat group permissions.
 393                 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
 394                     acl_group_obj = Some(CaFormatACLGroupObj {
 395                         permissions: gop,
 396                     });
 397                 }
 398             },
 399             acl::ACL_TYPE_DEFAULT => {
 400                 if user_obj_permissions != None ||
 401                    group_obj_permissions != None ||
 402                    other_permissions != None ||
 403                    mask_permissions != None
 404                 {
 405                     acl_default = Some(CaFormatACLDefault {
 406                         // The value is set to UINT64_MAX as placeholder if one
 407                         // of the permissions is not set
 408                         user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
 409                         group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
 410                         other_permissions: other_permissions.unwrap_or(std::u64::MAX),
 411                         mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
 412                     });
 413                 }
 414             },
 415             _ => bail!("Unexpected ACL type encountered"),
 416         }
 417
 418         Ok(PxarACL {
 419             users: acl_user,
 420             groups: acl_group,
 421             group_obj: acl_group_obj,
 422             default: acl_default,
 423         })
 424     }
 425
 426     /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
 427     fn read_quota_project_id(&self, fd: RawFd, magic: i64, stat: &FileStat) -> Result<Option<CaFormatQuotaProjID>, Error> {
 428         if !(is_directory(&stat) || is_reg_file(&stat)) {
 429             return Ok(None);
 430         }
 431         if !self.has_features(flags::WITH_QUOTA_PROJID) {
 432             return Ok(None);
 433         }
 434
 435         match magic {
 436             fs::magic::EXT4_SUPER_MAGIC |
 437             fs::magic::XFS_SUPER_MAGIC |
 438             fs::magic::FUSE_SUPER_MAGIC |
 439             fs::magic::ZFS_SUPER_MAGIC => {
 440                 let mut fsxattr = fs::FSXAttr::default();
 441                 let res = unsafe {
 442                     fs::fs_ioc_fsgetxattr(fd, &mut fsxattr)
 443                 };
 444
 445                 // On some FUSE filesystems it can happen that ioctl is not supported.
 446                 // For these cases projid is set to 0 while the error is ignored.
 447                 if let Err(err) = res {
 448                     let errno = err.as_errno().ok_or_else(|| {
 449                         format_err!("error while reading quota project id for {:#?}", self.full_path())
 450                     })?;
 451                     if errno_is_unsupported(errno) {
 452                         return Ok(None);
 453                     } else {
 454                         bail!("error while reading quota project id for {:#?} - {}", self.full_path(), errno);
 455                     }
 456                 }
 457
 458                 let projid = fsxattr.fsx_projid as u64;
 459                 if projid == 0 {
 460                     return Ok(None);
 461                 } else {
 462                     return Ok(Some(CaFormatQuotaProjID { projid }));
 463                 }
 464             },
 465             _ => return Ok(None),
 466         }
 467     }
 468
 469     fn write_entry(&mut self, entry: CaFormatEntry) -> Result<(), Error> {
 470
 471         self.write_header(CA_FORMAT_ENTRY, std::mem::size_of::<CaFormatEntry>() as u64)?;
 472         self.write_item(entry)?;
 473
 474         Ok(())
 475     }
 476
 477     fn write_xattr(&mut self, xattr: CaFormatXAttr) -> Result<(), Error> {
 478         let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
 479         self.write_header(CA_FORMAT_XATTR, size as u64)?;
 480         self.write(xattr.name.as_slice())?;
 481         self.write(&[0])?;
 482         self.write(xattr.value.as_slice())?;
 483
 484         Ok(())
 485     }
 486
 487     fn write_fcaps(&mut self, fcaps: Option<CaFormatFCaps>) -> Result<(), Error> {
 488         if let Some(fcaps) = fcaps {
 489             let size = fcaps.data.len();
 490             self.write_header(CA_FORMAT_FCAPS, size as u64)?;
 491             self.write(fcaps.data.as_slice())?;
 492         }
 493
 494         Ok(())
 495     }
 496
 497     fn write_acl_user(&mut self, acl_user: CaFormatACLUser) -> Result<(), Error> {
 498         self.write_header(CA_FORMAT_ACL_USER,  std::mem::size_of::<CaFormatACLUser>() as u64)?;
 499         self.write_item(acl_user)?;
 500
 501         Ok(())
 502     }
 503
 504     fn write_acl_group(&mut self, acl_group: CaFormatACLGroup) -> Result<(), Error> {
 505         self.write_header(CA_FORMAT_ACL_GROUP,  std::mem::size_of::<CaFormatACLGroup>() as u64)?;
 506         self.write_item(acl_group)?;
 507
 508         Ok(())
 509     }
 510
 511     fn write_acl_group_obj(&mut self, acl_group_obj: CaFormatACLGroupObj) -> Result<(), Error> {
 512         self.write_header(CA_FORMAT_ACL_GROUP_OBJ,  std::mem::size_of::<CaFormatACLGroupObj>() as u64)?;
 513         self.write_item(acl_group_obj)?;
 514
 515         Ok(())
 516     }
 517
 518     fn write_acl_default(&mut self, acl_default: CaFormatACLDefault) -> Result<(), Error> {
 519         self.write_header(CA_FORMAT_ACL_DEFAULT,  std::mem::size_of::<CaFormatACLDefault>() as u64)?;
 520         self.write_item(acl_default)?;
 521
 522         Ok(())
 523     }
 524
 525     fn write_acl_default_user(&mut self, acl_default_user: CaFormatACLUser) -> Result<(), Error> {
 526         self.write_header(CA_FORMAT_ACL_DEFAULT_USER,  std::mem::size_of::<CaFormatACLUser>() as u64)?;
 527         self.write_item(acl_default_user)?;
 528
 529         Ok(())
 530     }
 531
 532     fn write_acl_default_group(&mut self, acl_default_group: CaFormatACLGroup) -> Result<(), Error> {
 533         self.write_header(CA_FORMAT_ACL_DEFAULT_GROUP,  std::mem::size_of::<CaFormatACLGroup>() as u64)?;
 534         self.write_item(acl_default_group)?;
 535
 536         Ok(())
 537     }
 538
 539     fn write_quota_project_id(&mut self, projid: CaFormatQuotaProjID) -> Result<(), Error> {
 540         self.write_header(CA_FORMAT_QUOTA_PROJID, std::mem::size_of::<CaFormatQuotaProjID>() as u64)?;
 541         self.write_item(projid)?;
 542
 543         Ok(())
 544     }
 545
 546     fn write_goodbye_table(&mut self, goodbye_offset: usize, goodbye_items: &mut [CaFormatGoodbyeItem]) -> Result<(), Error> {
 547
 548         goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
 549
 550         let item_count = goodbye_items.len();
 551
 552         let goodbye_table_size = (item_count + 1)*std::mem::size_of::<CaFormatGoodbyeItem>();
 553
 554         self.write_header(CA_FORMAT_GOODBYE, goodbye_table_size as u64)?;
 555
 556         if self.file_copy_buffer.len() < goodbye_table_size {
 557             let need = goodbye_table_size - self.file_copy_buffer.len();
 558             self.file_copy_buffer.reserve(need);
 559             unsafe { self.file_copy_buffer.set_len(self.file_copy_buffer.capacity()); }
 560         }
 561
 562         let buffer = &mut self.file_copy_buffer;
 563
 564         copy_binary_search_tree(item_count, |s, d| {
 565             let item = &goodbye_items[s];
 566             let offset = d*std::mem::size_of::<CaFormatGoodbyeItem>();
 567             let dest = crate::tools::map_struct_mut::<CaFormatGoodbyeItem>(&mut buffer[offset..]).unwrap();
 568             dest.offset = u64::to_le(item.offset);
 569             dest.size = u64::to_le(item.size);
 570             dest.hash = u64::to_le(item.hash);
 571         });
 572
 573         // append CaFormatGoodbyeTail as last item
 574         let offset = item_count*std::mem::size_of::<CaFormatGoodbyeItem>();
 575         let dest = crate::tools::map_struct_mut::<CaFormatGoodbyeItem>(&mut buffer[offset..]).unwrap();
 576         dest.offset = u64::to_le(goodbye_offset as u64);
 577         dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<CaFormatHeader>()) as u64);
 578         dest.hash = u64::to_le(CA_FORMAT_GOODBYE_TAIL_MARKER);
 579
 580         self.flush_copy_buffer(goodbye_table_size)?;
 581
 582         Ok(())
 583     }
 584
 585     fn encode_dir(&mut self, dir: &mut nix::dir::Dir, dir_stat: &FileStat, magic: i64, match_pattern: Vec<PxarExcludePattern>)  -> Result<(), Error> {
 586
 587         //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
 588
 589         let mut name_list = vec![];
 590
 591         let rawfd = dir.as_raw_fd();
 592
 593         let dir_start_pos = self.writer_pos;
 594
 595         let mut dir_entry = self.create_entry(&dir_stat)?;
 596
 597         self.read_chattr(rawfd, &mut dir_entry)?;
 598         self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
 599
 600         // for each node in the directory tree, the filesystem features are
 601         // checked based on the fs magic number.
 602         self.fs_feature_flags = flags::feature_flags_from_magic(magic);
 603
 604         let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
 605         let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
 606         let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
 607         let projid = self.read_quota_project_id(rawfd, magic, &dir_stat)?;
 608
 609         self.write_entry(dir_entry)?;
 610         for xattr in xattrs {
 611             self.write_xattr(xattr)?;
 612         }
 613         self.write_fcaps(fcaps)?;
 614
 615         for user in acl_access.users {
 616             self.write_acl_user(user)?;
 617         }
 618         for group in acl_access.groups {
 619             self.write_acl_group(group)?;
 620         }
 621         if let Some(group_obj) = acl_access.group_obj {
 622             self.write_acl_group_obj(group_obj)?;
 623         }
 624
 625         for default_user in acl_default.users {
 626             self.write_acl_default_user(default_user)?;
 627         }
 628         for default_group in acl_default.groups {
 629             self.write_acl_default_group(default_group)?;
 630         }
 631         if let Some(default) = acl_default.default {
 632             self.write_acl_default(default)?;
 633         }
 634         if let Some(projid) = projid {
 635             self.write_quota_project_id(projid)?;
 636         }
 637
 638         let include_children;
 639         if is_virtual_file_system(magic) {
 640             include_children = false;
 641         } else {
 642             if let Some(set) = &self.device_set {
 643                 include_children = set.contains(&dir_stat.st_dev);
 644             } else {
 645                 include_children = true;
 646             }
 647         }
 648
 649         // Expand the exclude match pattern inherited from the parent by local entries, if present
 650         let mut local_match_pattern = match_pattern.clone();
 651         let pxar_exclude = match PxarExcludePattern::from_file(rawfd, ".pxarexclude") {
 652             Ok(Some((mut excludes, buffer, stat))) => {
 653                 local_match_pattern.append(&mut excludes);
 654                 Some((buffer, stat))
 655             },
 656             Ok(None) => None,
 657             Err(err) => bail!("error while reading exclude file - {}", err),
 658         };
 659
 660         if include_children {
 661             for entry in dir.iter() {
 662                 let entry =  entry.map_err(|err| {
 663                     format_err!("readir {:?} failed - {}", self.full_path(), err)
 664                 })?;
 665                 let filename = entry.file_name().to_owned();
 666
 667                 let name = filename.to_bytes_with_nul();
 668                 if name == b".\0" || name == b"..\0" {
 669                     continue;
 670                 }
 671
 672                 let stat = match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
 673                     Ok(stat) => stat,
 674                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 675                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 676                         self.report_vanished_file(&self.full_path().join(filename_osstr))?;
 677                         continue;
 678                     },
 679                     Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
 680                 };
 681
 682                 match match_exclude_pattern(&filename, &stat, &local_match_pattern) {
 683                     (MatchType::Exclude, _) => {
 684                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 685                         eprintln!("matched by .pxarexclude entry - skipping: {:?}", self.full_path().join(filename_osstr));
 686                     },
 687                     (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
 688                 }
 689
 690                 if name_list.len() > MAX_DIRECTORY_ENTRIES {
 691                     bail!("too many directory items in {:?} (> {})", self.full_path(), MAX_DIRECTORY_ENTRIES);
 692                 }
 693             }
 694         } else {
 695             eprintln!("skip mount point: {:?}", self.full_path());
 696         }
 697
 698         name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
 699
 700         let mut goodbye_items = vec![];
 701
 702         for (filename, stat, exclude_list) in name_list {
 703             let start_pos = self.writer_pos;
 704
 705             if filename.as_bytes() == b".pxarexclude" {
 706                 if let Some((ref content, ref stat)) = pxar_exclude {
 707                     let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
 708                         Ok(filefd) => filefd,
 709                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 710                             self.report_vanished_file(&self.full_path())?;
 711                             continue;
 712                         },
 713                         Err(err) => {
 714                             let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 715                             bail!("open file {:?} failed - {}", self.full_path().join(filename_osstr), err);
 716                         },
 717                     };
 718
 719                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 720                         detect_fs_type(filefd)?
 721                     } else {
 722                         magic
 723                     };
 724
 725                     self.write_filename(&filename)?;
 726                     self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
 727                     continue;
 728                 }
 729             }
 730
 731             self.relative_path.push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
 732
 733             if self.verbose { println!("{:?}", self.full_path()); }
 734
 735             if is_directory(&stat) {
 736
 737                 let mut dir = match nix::dir::Dir::openat(rawfd, filename.as_ref(), OFlag::O_DIRECTORY|OFlag::O_NOFOLLOW, Mode::empty()) {
 738                     Ok(dir) => dir,
 739                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 740                         self.report_vanished_file(&self.full_path())?;
 741                         continue; // fixme!!
 742                     },
 743                     Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
 744                 };
 745
 746                 let child_magic = if dir_stat.st_dev != stat.st_dev {
 747                     detect_fs_type(dir.as_raw_fd())?
 748                 } else {
 749                     magic
 750                 };
 751
 752                 self.write_filename(&filename)?;
 753                 self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
 754
 755             } else if is_reg_file(&stat) {
 756
 757                 let mut hardlink_target = None;
 758
 759                 if stat.st_nlink > 1 {
 760                     let link_info = HardLinkInfo { st_dev: stat.st_dev, st_ino: stat.st_ino };
 761                     hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
 762                         let mut target = v.clone().into_os_string();
 763                         target.push("\0"); // add Nul byte
 764                         (target, (start_pos as u64) - offset)
 765                     });
 766                     if hardlink_target == None {
 767                         self.hardlinks.insert(link_info, (self.relative_path.clone(), start_pos as u64));
 768                     }
 769                 }
 770
 771                 if let Some((target, offset)) = hardlink_target {
 772
 773                     self.write_filename(&filename)?;
 774                     self.encode_hardlink(target.as_bytes(), offset)?;
 775
 776                 } else {
 777
 778                     let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
 779                         Ok(filefd) => filefd,
 780                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 781                             self.report_vanished_file(&self.full_path())?;
 782                             continue;
 783                         },
 784                         Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
 785                     };
 786
 787                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 788                         detect_fs_type(filefd)?
 789                     } else {
 790                         magic
 791                     };
 792
 793                     self.write_filename(&filename)?;
 794                     let res = self.encode_file(filefd, &stat, child_magic);
 795                     let _ = nix::unistd::close(filefd); // ignore close errors
 796                     res?;
 797                 }
 798
 799             } else if is_symlink(&stat) {
 800                 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
 801
 802                 let res = filename.with_nix_path(|cstr| {
 803                     unsafe { libc::readlinkat(rawfd, cstr.as_ptr(), buffer.as_mut_ptr() as *mut libc::c_char, buffer.len()-1) }
 804                 })?;
 805
 806                 match Errno::result(res) {
 807                     Ok(len) => {
 808                         buffer[len as usize] = 0u8; // add Nul byte
 809                         self.write_filename(&filename)?;
 810                         self.encode_symlink(&buffer[..((len+1) as usize)], &stat)?
 811                     }
 812                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 813                         self.report_vanished_file(&self.full_path())?;
 814                         continue;
 815                     }
 816                     Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
 817                 }
 818             } else if is_block_dev(&stat) || is_char_dev(&stat) {
 819                 if self.has_features(flags::WITH_DEVICE_NODES) {
 820                     self.write_filename(&filename)?;
 821                     self.encode_device(&stat)?;
 822                 } else {
 823                     eprintln!("skip device node: {:?}", self.full_path());
 824                 }
 825             } else if is_fifo(&stat) {
 826                 if self.has_features(flags::WITH_FIFOS) {
 827                     self.write_filename(&filename)?;
 828                     self.encode_special(&stat)?;
 829                 } else {
 830                     eprintln!("skip fifo: {:?}", self.full_path());
 831                 }
 832             } else if is_socket(&stat) {
 833                 if self.has_features(flags::WITH_SOCKETS) {
 834                     self.write_filename(&filename)?;
 835                     self.encode_special(&stat)?;
 836                 } else {
 837                     eprintln!("skip socket: {:?}", self.full_path());
 838                 }
 839             } else {
 840                 bail!("unsupported file type (mode {:o} {:?})", stat.st_mode, self.full_path());
 841             }
 842
 843             let end_pos = self.writer_pos;
 844
 845             goodbye_items.push(CaFormatGoodbyeItem {
 846                 offset: start_pos as u64,
 847                 size: (end_pos - start_pos) as u64,
 848                 hash: compute_goodbye_hash(filename.to_bytes()),
 849             });
 850
 851             self.relative_path.pop();
 852         }
 853
 854         //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
 855
 856         // fixup goodby item offsets
 857         let goodbye_start = self.writer_pos as u64;
 858         for item in &mut goodbye_items {
 859             item.offset = goodbye_start - item.offset;
 860         }
 861
 862         let goodbye_offset = self.writer_pos - dir_start_pos;
 863
 864         self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
 865
 866         //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
 867         Ok(())
 868     }
 869
 870     fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64)  -> Result<(), Error> {
 871
 872         //println!("encode_file: {:?}", self.full_path());
 873
 874         let mut entry = self.create_entry(&stat)?;
 875
 876         self.read_chattr(filefd, &mut entry)?;
 877         self.read_fat_attr(filefd, magic, &mut entry)?;
 878         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
 879         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
 880         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
 881
 882         self.write_entry(entry)?;
 883         for xattr in xattrs {
 884             self.write_xattr(xattr)?;
 885         }
 886         self.write_fcaps(fcaps)?;
 887         for user in acl_access.users {
 888             self.write_acl_user(user)?;
 889         }
 890         for group in acl_access.groups {
 891             self.write_acl_group(group)?;
 892         }
 893         if let Some(group_obj) = acl_access.group_obj {
 894             self.write_acl_group_obj(group_obj)?;
 895         }
 896         if let Some(projid) = projid {
 897             self.write_quota_project_id(projid)?;
 898         }
 899
 900         let include_payload;
 901         if is_virtual_file_system(magic) {
 902             include_payload = false;
 903         } else {
 904             if let Some(ref set) = &self.device_set {
 905                 include_payload = set.contains(&stat.st_dev);
 906             } else {
 907                 include_payload = true;
 908             }
 909         }
 910
 911         if !include_payload {
 912             eprintln!("skip content: {:?}", self.full_path());
 913             self.write_header(CA_FORMAT_PAYLOAD, 0)?;
 914             return Ok(());
 915         }
 916
 917         let size = stat.st_size as u64;
 918
 919         self.write_header(CA_FORMAT_PAYLOAD, size)?;
 920
 921         let mut pos: u64 = 0;
 922         loop {
 923             let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
 924                 Ok(n) => n,
 925                 Err(nix::Error::Sys(Errno::EINTR)) => continue /* try again */,
 926                 Err(err) =>  bail!("read {:?} failed - {}", self.full_path(), err),
 927             };
 928             if n == 0 { // EOF
 929                 if pos != size {
 930                     // Note:: casync format cannot handle that
 931                     bail!("detected shrinked file {:?} ({} < {})", self.full_path(), pos, size);
 932                 }
 933                 break;
 934             }
 935
 936             let mut next = pos + (n as u64);
 937
 938             if next > size { next = size; }
 939
 940             let count = (next - pos) as usize;
 941
 942             self.flush_copy_buffer(count)?;
 943
 944             pos = next;
 945
 946             if pos >= size { break; }
 947         }
 948
 949         Ok(())
 950     }
 951
 952     fn encode_device(&mut self, stat: &FileStat)  -> Result<(), Error> {
 953
 954         let entry = self.create_entry(&stat)?;
 955
 956         self.write_entry(entry)?;
 957
 958         let major = unsafe { libc::major(stat.st_rdev) } as u64;
 959         let minor = unsafe { libc::minor(stat.st_rdev) } as u64;
 960
 961         //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
 962
 963         self.write_header(CA_FORMAT_DEVICE, std::mem::size_of::<CaFormatDevice>() as u64)?;
 964         self.write_item(CaFormatDevice { major, minor })?;
 965
 966         Ok(())
 967     }
 968
 969     // FIFO or Socket
 970     fn encode_special(&mut self, stat: &FileStat)  -> Result<(), Error> {
 971
 972         let entry = self.create_entry(&stat)?;
 973
 974         self.write_entry(entry)?;
 975
 976         Ok(())
 977     }
 978
 979     fn encode_symlink(&mut self, target: &[u8], stat: &FileStat)  -> Result<(), Error> {
 980
 981         //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
 982
 983         let entry = self.create_entry(&stat)?;
 984         self.write_entry(entry)?;
 985
 986         self.write_header(CA_FORMAT_SYMLINK, target.len() as u64)?;
 987         self.write(target)?;
 988
 989         Ok(())
 990     }
 991
 992     fn encode_hardlink(&mut self, target: &[u8], offset: u64)  -> Result<(), Error> {
 993
 994         //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
 995
 996         // Note: HARDLINK replaces an ENTRY.
 997         self.write_header(PXAR_FORMAT_HARDLINK, (target.len() as u64) + 8)?;
 998         self.write_item(offset)?;
 999         self.write(target)?;
1000
1001         Ok(())
1002     }
1003
1004     fn encode_pxar_exclude(&mut self, filefd: RawFd, stat: &FileStat, magic: i64, content: &[u8]) -> Result<(), Error> {
1005         let mut entry = self.create_entry(&stat)?;
1006
1007         self.read_chattr(filefd, &mut entry)?;
1008         self.read_fat_attr(filefd, magic, &mut entry)?;
1009         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1010         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1011         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1012
1013         self.write_entry(entry)?;
1014         for xattr in xattrs {
1015             self.write_xattr(xattr)?;
1016         }
1017         self.write_fcaps(fcaps)?;
1018         for user in acl_access.users {
1019             self.write_acl_user(user)?;
1020         }
1021         for group in acl_access.groups {
1022             self.write_acl_group(group)?;
1023         }
1024         if let Some(group_obj) = acl_access.group_obj {
1025             self.write_acl_group_obj(group_obj)?;
1026         }
1027         if let Some(projid) = projid {
1028             self.write_quota_project_id(projid)?;
1029         }
1030
1031         let include_payload;
1032         if is_virtual_file_system(magic) {
1033             include_payload = false;
1034         } else {
1035             if let Some(set) = &self.device_set {
1036                 include_payload = set.contains(&stat.st_dev);
1037             } else {
1038                 include_payload = true;
1039             }
1040         }
1041
1042         if !include_payload {
1043             eprintln!("skip content: {:?}", self.full_path());
1044             self.write_header(CA_FORMAT_PAYLOAD, 0)?;
1045             return Ok(());
1046         }
1047
1048         let size = content.len();
1049         self.write_header(CA_FORMAT_PAYLOAD, size as u64)?;
1050         self.writer.write_all(content)?;
1051         self.writer_pos += size;
1052
1053         Ok(())
1054     }
1055
1056     // the report_XXX method may raise and error - depending on encoder configuration
1057
1058     fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
1059
1060         eprintln!("WARNING: detected vanished file {:?}", path);
1061
1062         Ok(())
1063     }
1064 }
1065
1066 // If there is a match, an updated PxarExcludePattern list to pass to the matched child is returned.
1067 fn match_exclude_pattern(
1068     filename: &CStr,
1069     stat: &FileStat,
1070     match_pattern: &Vec<PxarExcludePattern>
1071 ) ->  (MatchType, Vec<PxarExcludePattern>) {
1072     let mut child_pattern = Vec::new();
1073     let mut match_state = MatchType::None;
1074
1075     for pattern in match_pattern {
1076         match pattern.matches_filename(filename, is_directory(&stat)) {
1077             MatchType::None =>  {},
1078             MatchType::Exclude =>  match_state = MatchType::Exclude,
1079             MatchType::Include =>  match_state = MatchType::Include,
1080             MatchType::PartialExclude =>  {
1081                 if match_state != MatchType::Exclude && match_state != MatchType::Include {
1082                     match_state = MatchType::PartialExclude;
1083                 }
1084                 child_pattern.push(pattern.get_rest_pattern());
1085             },
1086             MatchType::PartialInclude =>  {
1087                 if match_state != MatchType::Exclude && match_state != MatchType::Include {
1088                     match_state = MatchType::PartialInclude;
1089                 }
1090                 child_pattern.push(pattern.get_rest_pattern());
1091             },
1092         }
1093     }
1094
1095     (match_state, child_pattern)
1096 }
1097
1098 fn errno_is_unsupported(errno: Errno) -> bool {
1099
1100     match errno {
1101         Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => {
1102             true
1103         }
1104         _ => false,
1105     }
1106 }
1107
1108 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
1109     let mut fs_stat: libc::statfs = unsafe { std::mem::uninitialized() };
1110     let res = unsafe { libc::fstatfs(fd, &mut fs_stat) };
1111     Errno::result(res)?;
1112
1113     Ok(fs_stat.f_type)
1114 }
1115
1116 #[inline(always)]
1117 pub fn is_temporary_file_system(magic: i64) -> bool {
1118     use fs::magic::*;
1119     magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
1120 }
1121
1122 pub fn is_virtual_file_system(magic: i64) -> bool {
1123     use fs::magic::*;
1124
1125     match magic {
1126         BINFMTFS_MAGIC |
1127         CGROUP2_SUPER_MAGIC |
1128         CGROUP_SUPER_MAGIC |
1129         CONFIGFS_MAGIC |
1130         DEBUGFS_MAGIC |
1131         DEVPTS_SUPER_MAGIC |
1132         EFIVARFS_MAGIC |
1133         FUSE_CTL_SUPER_MAGIC |
1134         HUGETLBFS_MAGIC |
1135         MQUEUE_MAGIC |
1136         NFSD_MAGIC |
1137         PROC_SUPER_MAGIC |
1138         PSTOREFS_MAGIC |
1139         RPCAUTH_GSSMAGIC |
1140         SECURITYFS_MAGIC |
1141         SELINUX_MAGIC |
1142         SMACK_MAGIC |
1143         SYSFS_MAGIC => true,
1144         _ => false
1145     }
1146 }