src/pxar/encoder.rs

   1 //! *pxar* format encoder.
   2 //!
   3 //! This module contain the code to generate *pxar* archive files.
   4
   5 use failure::*;
   6 use endian_trait::Endian;
   7 use std::collections::{HashSet, HashMap};
   8
   9 use super::format_definition::*;
  10 use super::binary_search_tree::*;
  11 use super::helper::*;
  12 use super::exclude_pattern::*;
  13 use crate::tools::fs;
  14 use crate::tools::acl;
  15 use crate::tools::xattr;
  16
  17 use std::io::Write;
  18 use std::os::unix::io::AsRawFd;
  19 use std::os::unix::ffi::OsStrExt;
  20 use std::os::unix::io::RawFd;
  21 use std::path::{Path, PathBuf};
  22
  23 use std::ffi::CStr;
  24
  25 use nix::NixPath;
  26 use nix::fcntl::OFlag;
  27 use nix::sys::stat::Mode;
  28 use nix::errno::Errno;
  29 use nix::sys::stat::FileStat;
  30
  31 use proxmox::tools::vec;
  32
  33 /// The format requires to build sorted directory lookup tables in
  34 /// memory, so we restrict the number of allowed entries to limit
  35 /// maximum memory usage.
  36 pub const MAX_DIRECTORY_ENTRIES: usize = 256*1024;
  37
  38 #[derive(Eq, PartialEq, Hash)]
  39 struct HardLinkInfo {
  40     st_dev: u64,
  41     st_ino: u64,
  42 }
  43
  44 pub struct Encoder<'a, W: Write> {
  45     base_path: PathBuf,
  46     relative_path: PathBuf,
  47     writer: &'a mut W,
  48     writer_pos: usize,
  49     _size: usize,
  50     file_copy_buffer: Vec<u8>,
  51     device_set: Option<HashSet<u64>>,
  52     verbose: bool,
  53     // Flags set by the user
  54     feature_flags: u64,
  55     // Flags signaling features supported by the filesystem
  56     fs_feature_flags: u64,
  57     hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
  58 }
  59
  60 impl <'a, W: Write> Encoder<'a, W> {
  61
  62     // used for error reporting
  63     fn full_path(&self) ->  PathBuf {
  64         self.base_path.join(&self.relative_path)
  65     }
  66
  67     /// Create archive, write result data to ``writer``.
  68     ///
  69     /// The ``device_set`` can be use used to limit included mount points.
  70     ///
  71     /// - ``None``: include all mount points
  72     /// - ``Some(set)``: only include devices listed in this set (the
  73     ///   root path device is automathically added to this list, so
  74     ///   you can pass an empty set if you want to archive a single
  75     ///   mount point.)
  76     pub fn encode(
  77         path: PathBuf,
  78         dir: &mut nix::dir::Dir,
  79         writer: &'a mut W,
  80         device_set: Option<HashSet<u64>>,
  81         verbose: bool,
  82         skip_lost_and_found: bool, // fixme: should be a feature flag ??
  83         feature_flags: u64,
  84     ) -> Result<(), Error> {
  85
  86         const FILE_COPY_BUFFER_SIZE: usize = 1024*1024;
  87
  88         let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
  89         unsafe { file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE); }
  90
  91
  92         // todo: use scandirat??
  93
  94         let dir_fd = dir.as_raw_fd();
  95         let stat = nix::sys::stat::fstat(dir_fd)
  96             .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
  97
  98         if !is_directory(&stat) {
  99             bail!("got unexpected file type {:?} (not a directory)", path);
 100         }
 101
 102         let mut device_set = device_set.clone();
 103         if let Some(ref mut set) = device_set {
 104             set.insert(stat.st_dev);
 105         }
 106
 107         let magic = detect_fs_type(dir_fd)?;
 108
 109         if is_virtual_file_system(magic) {
 110             bail!("backup virtual file systems is disabled!");
 111         }
 112
 113         let fs_feature_flags = feature_flags_from_magic(magic);
 114
 115         let mut me = Self {
 116             base_path: path,
 117             relative_path: PathBuf::new(),
 118             writer: writer,
 119             writer_pos: 0,
 120             _size: 0,
 121             file_copy_buffer,
 122             device_set,
 123             verbose,
 124             feature_flags,
 125             fs_feature_flags,
 126             hardlinks: HashMap::new(),
 127         };
 128
 129         if verbose { println!("{:?}", me.full_path()); }
 130
 131         let mut excludes = Vec::new();
 132         if skip_lost_and_found {
 133             excludes.push(PxarExcludePattern::from_line(b"**/lost+found").unwrap().unwrap());
 134         }
 135         me.encode_dir(dir, &stat, magic, excludes)?;
 136
 137         Ok(())
 138     }
 139
 140     fn write(&mut self,  buf: &[u8]) -> Result<(), Error> {
 141         self.writer.write_all(buf)?;
 142         self.writer_pos += buf.len();
 143         Ok(())
 144     }
 145
 146     fn write_item<T: Endian>(&mut self, item: T) ->  Result<(), Error> {
 147
 148         let data = item.to_le();
 149
 150         let buffer = unsafe { std::slice::from_raw_parts(
 151             &data as *const T as *const u8,
 152             std::mem::size_of::<T>()
 153         )};
 154
 155         self.write(buffer)?;
 156
 157         Ok(())
 158     }
 159
 160     fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
 161         self.writer.write_all(&self.file_copy_buffer[..size])?;
 162         self.writer_pos += size;
 163         Ok(())
 164     }
 165
 166     fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
 167
 168         let size = size + (std::mem::size_of::<CaFormatHeader>() as u64);
 169         self.write_item(CaFormatHeader { size, htype })?;
 170
 171         Ok(())
 172     }
 173
 174     fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
 175
 176         let buffer = name.to_bytes_with_nul();
 177         self.write_header(CA_FORMAT_FILENAME, buffer.len() as u64)?;
 178         self.write(buffer)?;
 179
 180         Ok(())
 181     }
 182
 183     fn create_entry(&self, stat: &FileStat) -> Result<CaFormatEntry, Error> {
 184
 185         let mode = if is_symlink(&stat) {
 186             (libc::S_IFLNK | 0o777) as u64
 187         } else {
 188             (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
 189         };
 190
 191         let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
 192         if mtime < 0 {
 193             bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
 194         }
 195
 196
 197         let entry = CaFormatEntry {
 198             mode: mode,
 199             flags: 0,
 200             uid: stat.st_uid as u64,
 201             gid: stat.st_gid as u64,
 202             mtime: mtime as u64,
 203         };
 204
 205         Ok(entry)
 206     }
 207
 208     fn read_chattr(&self, fd: RawFd, entry: &mut CaFormatEntry) -> Result<(), Error> {
 209
 210         let mut attr: usize = 0;
 211
 212         let res = unsafe { fs::read_attr_fd(fd, &mut attr)};
 213         if let Err(err) = res {
 214             if let nix::Error::Sys(errno) = err {
 215                 if errno_is_unsupported(errno) { return Ok(()) };
 216             }
 217             bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
 218         }
 219
 220         let flags = ca_feature_flags_from_chattr(attr as u32);
 221         entry.flags = entry.flags | flags;
 222
 223         Ok(())
 224     }
 225
 226     fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut CaFormatEntry) -> Result<(), Error> {
 227
 228         if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC { return Ok(()); }
 229
 230         let mut attr: u32 = 0;
 231
 232         let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr)};
 233         if let Err(err) = res {
 234             if let nix::Error::Sys(errno) = err {
 235                 if errno_is_unsupported(errno) { return Ok(()) };
 236             }
 237             bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
 238         }
 239
 240         let flags = ca_feature_flags_from_fat_attr(attr);
 241         entry.flags = entry.flags | flags;
 242
 243         Ok(())
 244     }
 245
 246     /// True if all of the given feature flags are set in the Encoder, false otherwise
 247     fn has_features(&self, feature_flags: u64) -> bool {
 248         (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
 249     }
 250
 251     /// True if at least one of the given feature flags is set in the Encoder, false otherwise
 252     fn has_some_features(&self, feature_flags: u64) -> bool {
 253         (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
 254     }
 255
 256     fn read_xattrs(&self, fd: RawFd, stat: &FileStat) -> Result<(Vec<CaFormatXAttr>, Option<CaFormatFCaps>), Error> {
 257         let mut xattrs = Vec::new();
 258         let mut fcaps = None;
 259
 260         let flags = CA_FORMAT_WITH_XATTRS | CA_FORMAT_WITH_FCAPS;
 261         if !self.has_some_features(flags) {
 262             return Ok((xattrs, fcaps));
 263         }
 264         // Should never be called on symlinks, just in case check anyway
 265         if is_symlink(&stat) {
 266             return Ok((xattrs, fcaps));
 267         }
 268
 269         let xattr_names = match xattr::flistxattr(fd) {
 270             Ok(names) => names,
 271             // Do not bail if the underlying endpoint does not supports xattrs
 272             Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
 273             // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
 274             Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
 275             Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 276         };
 277
 278         for name in xattr_names.split(|c| *c == b'\0') {
 279             // Only extract the relevant extended attributes
 280             if !xattr::is_valid_xattr_name(&name) {
 281                 continue;
 282             }
 283
 284             let value = match xattr::fgetxattr(fd, name) {
 285                 Ok(value) => value,
 286                 // Vanished between flistattr and getxattr, this is ok, silently ignore
 287                 Err(Errno::ENODATA) => continue,
 288                 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 289             };
 290
 291             if xattr::is_security_capability(&name) {
 292                 if self.has_features(CA_FORMAT_WITH_FCAPS) {
 293                     // fcaps are stored in own format within the archive
 294                     fcaps = Some(CaFormatFCaps {
 295                         data: value,
 296                     });
 297                 }
 298             } else if self.has_features(CA_FORMAT_WITH_XATTRS) {
 299                 xattrs.push(CaFormatXAttr {
 300                     name: name.to_vec(),
 301                     value: value,
 302                 });
 303             }
 304         }
 305         xattrs.sort();
 306
 307         Ok((xattrs, fcaps))
 308     }
 309
 310     fn read_acl(&self, fd: RawFd, stat: &FileStat, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 311         let ret = PxarACL {
 312             users: Vec::new(),
 313             groups: Vec::new(),
 314             group_obj: None,
 315             default: None,
 316         };
 317
 318         if !self.has_features(CA_FORMAT_WITH_ACL) {
 319             return Ok(ret);
 320         }
 321         if is_symlink(&stat) {
 322             return Ok(ret);
 323         }
 324         if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
 325             bail!("ACL_TYPE_DEFAULT only defined for directories.");
 326         }
 327
 328         // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
 329         // to create a path for acl_get_file(). acl_get_fd() only allows to get
 330         // ACL_TYPE_ACCESS attributes.
 331         let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
 332         let acl = match acl::ACL::get_file(&proc_path, acl_type) {
 333             Ok(acl) => acl,
 334             // Don't bail if underlying endpoint does not support acls
 335             Err(Errno::EOPNOTSUPP) => return Ok(ret),
 336             // Don't bail if the endpoint cannot carry acls
 337             Err(Errno::EBADF) => return Ok(ret),
 338             // Don't bail if there is no data
 339             Err(Errno::ENODATA) => return Ok(ret),
 340             Err(err) => bail!("error while reading ACL - {}", err),
 341         };
 342
 343         self.process_acl(acl, acl_type)
 344     }
 345
 346     fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 347         let mut acl_user = Vec::new();
 348         let mut acl_group = Vec::new();
 349         let mut acl_group_obj = None;
 350         let mut acl_default = None;
 351         let mut user_obj_permissions = None;
 352         let mut group_obj_permissions = None;
 353         let mut other_permissions = None;
 354         let mut mask_permissions = None;
 355
 356         for entry in &mut acl.entries() {
 357             let tag = entry.get_tag_type()?;
 358             let permissions = entry.get_permissions()?;
 359             match tag {
 360                 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
 361                 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
 362                 acl::ACL_OTHER => other_permissions = Some(permissions),
 363                 acl::ACL_MASK => mask_permissions = Some(permissions),
 364                 acl::ACL_USER => {
 365                     acl_user.push(CaFormatACLUser {
 366                         uid: entry.get_qualifier()?,
 367                         permissions: permissions,
 368                     });
 369                 },
 370                 acl::ACL_GROUP => {
 371                     acl_group.push(CaFormatACLGroup {
 372                         gid: entry.get_qualifier()?,
 373                         permissions: permissions,
 374                     });
 375                 },
 376                 _ => bail!("Unexpected ACL tag encountered!"),
 377             }
 378         }
 379
 380         acl_user.sort();
 381         acl_group.sort();
 382
 383         match acl_type {
 384             acl::ACL_TYPE_ACCESS => {
 385                 // The mask permissions are mapped to the stat group permissions
 386                 // in case that the ACL group permissions were set.
 387                 // Only in that case we need to store the group permissions,
 388                 // in the other cases they are identical to the stat group permissions.
 389                 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
 390                     acl_group_obj = Some(CaFormatACLGroupObj {
 391                         permissions: gop,
 392                     });
 393                 }
 394             },
 395             acl::ACL_TYPE_DEFAULT => {
 396                 if user_obj_permissions != None ||
 397                    group_obj_permissions != None ||
 398                    other_permissions != None ||
 399                    mask_permissions != None
 400                 {
 401                     acl_default = Some(CaFormatACLDefault {
 402                         // The value is set to UINT64_MAX as placeholder if one
 403                         // of the permissions is not set
 404                         user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
 405                         group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
 406                         other_permissions: other_permissions.unwrap_or(std::u64::MAX),
 407                         mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
 408                     });
 409                 }
 410             },
 411             _ => bail!("Unexpected ACL type encountered"),
 412         }
 413
 414         Ok(PxarACL {
 415             users: acl_user,
 416             groups: acl_group,
 417             group_obj: acl_group_obj,
 418             default: acl_default,
 419         })
 420     }
 421
 422     /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
 423     fn read_quota_project_id(&self, fd: RawFd, magic: i64, stat: &FileStat) -> Result<Option<CaFormatQuotaProjID>, Error> {
 424         if !(is_directory(&stat) || is_reg_file(&stat)) {
 425             return Ok(None);
 426         }
 427         if !self.has_features(CA_FORMAT_WITH_QUOTA_PROJID) {
 428             return Ok(None);
 429         }
 430
 431         match magic {
 432             EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC | ZFS_SUPER_MAGIC => {
 433                 let mut fsxattr = fs::FSXAttr::default();
 434                 let res = unsafe {
 435                     fs::fs_ioc_fsgetxattr(fd, &mut fsxattr)
 436                 };
 437
 438                 // On some FUSE filesystems it can happen that ioctl is not supported.
 439                 // For these cases projid is set to 0 while the error is ignored.
 440                 if let Err(err) = res {
 441                     let errno = err.as_errno().ok_or_else(|| {
 442                         format_err!("error while reading quota project id for {:#?}", self.full_path())
 443                     })?;
 444                     if errno_is_unsupported(errno) {
 445                         return Ok(None);
 446                     } else {
 447                         bail!("error while reading quota project id for {:#?} - {}", self.full_path(), errno);
 448                     }
 449                 }
 450
 451                 let projid = fsxattr.fsx_projid as u64;
 452                 if projid == 0 {
 453                     return Ok(None);
 454                 } else {
 455                     return Ok(Some(CaFormatQuotaProjID { projid }));
 456                 }
 457             },
 458             _ => return Ok(None),
 459         }
 460     }
 461
 462     fn write_entry(&mut self, entry: CaFormatEntry) -> Result<(), Error> {
 463
 464         self.write_header(CA_FORMAT_ENTRY, std::mem::size_of::<CaFormatEntry>() as u64)?;
 465         self.write_item(entry)?;
 466
 467         Ok(())
 468     }
 469
 470     fn write_xattr(&mut self, xattr: CaFormatXAttr) -> Result<(), Error> {
 471         let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
 472         self.write_header(CA_FORMAT_XATTR, size as u64)?;
 473         self.write(xattr.name.as_slice())?;
 474         self.write(&[0])?;
 475         self.write(xattr.value.as_slice())?;
 476
 477         Ok(())
 478     }
 479
 480     fn write_fcaps(&mut self, fcaps: Option<CaFormatFCaps>) -> Result<(), Error> {
 481         if let Some(fcaps) = fcaps {
 482             let size = fcaps.data.len();
 483             self.write_header(CA_FORMAT_FCAPS, size as u64)?;
 484             self.write(fcaps.data.as_slice())?;
 485         }
 486
 487         Ok(())
 488     }
 489
 490     fn write_acl_user(&mut self, acl_user: CaFormatACLUser) -> Result<(), Error> {
 491         self.write_header(CA_FORMAT_ACL_USER,  std::mem::size_of::<CaFormatACLUser>() as u64)?;
 492         self.write_item(acl_user)?;
 493
 494         Ok(())
 495     }
 496
 497     fn write_acl_group(&mut self, acl_group: CaFormatACLGroup) -> Result<(), Error> {
 498         self.write_header(CA_FORMAT_ACL_GROUP,  std::mem::size_of::<CaFormatACLGroup>() as u64)?;
 499         self.write_item(acl_group)?;
 500
 501         Ok(())
 502     }
 503
 504     fn write_acl_group_obj(&mut self, acl_group_obj: CaFormatACLGroupObj) -> Result<(), Error> {
 505         self.write_header(CA_FORMAT_ACL_GROUP_OBJ,  std::mem::size_of::<CaFormatACLGroupObj>() as u64)?;
 506         self.write_item(acl_group_obj)?;
 507
 508         Ok(())
 509     }
 510
 511     fn write_acl_default(&mut self, acl_default: CaFormatACLDefault) -> Result<(), Error> {
 512         self.write_header(CA_FORMAT_ACL_DEFAULT,  std::mem::size_of::<CaFormatACLDefault>() as u64)?;
 513         self.write_item(acl_default)?;
 514
 515         Ok(())
 516     }
 517
 518     fn write_acl_default_user(&mut self, acl_default_user: CaFormatACLUser) -> Result<(), Error> {
 519         self.write_header(CA_FORMAT_ACL_DEFAULT_USER,  std::mem::size_of::<CaFormatACLUser>() as u64)?;
 520         self.write_item(acl_default_user)?;
 521
 522         Ok(())
 523     }
 524
 525     fn write_acl_default_group(&mut self, acl_default_group: CaFormatACLGroup) -> Result<(), Error> {
 526         self.write_header(CA_FORMAT_ACL_DEFAULT_GROUP,  std::mem::size_of::<CaFormatACLGroup>() as u64)?;
 527         self.write_item(acl_default_group)?;
 528
 529         Ok(())
 530     }
 531
 532     fn write_quota_project_id(&mut self, projid: CaFormatQuotaProjID) -> Result<(), Error> {
 533         self.write_header(CA_FORMAT_QUOTA_PROJID, std::mem::size_of::<CaFormatQuotaProjID>() as u64)?;
 534         self.write_item(projid)?;
 535
 536         Ok(())
 537     }
 538
 539     fn write_goodbye_table(&mut self, goodbye_offset: usize, goodbye_items: &mut [CaFormatGoodbyeItem]) -> Result<(), Error> {
 540
 541         goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
 542
 543         let item_count = goodbye_items.len();
 544
 545         let goodbye_table_size = (item_count + 1)*std::mem::size_of::<CaFormatGoodbyeItem>();
 546
 547         self.write_header(CA_FORMAT_GOODBYE, goodbye_table_size as u64)?;
 548
 549         if self.file_copy_buffer.len() < goodbye_table_size {
 550             let need = goodbye_table_size - self.file_copy_buffer.len();
 551             self.file_copy_buffer.reserve(need);
 552             unsafe { self.file_copy_buffer.set_len(self.file_copy_buffer.capacity()); }
 553         }
 554
 555         let buffer = &mut self.file_copy_buffer;
 556
 557         copy_binary_search_tree(item_count, |s, d| {
 558             let item = &goodbye_items[s];
 559             let offset = d*std::mem::size_of::<CaFormatGoodbyeItem>();
 560             let dest = crate::tools::map_struct_mut::<CaFormatGoodbyeItem>(&mut buffer[offset..]).unwrap();
 561             dest.offset = u64::to_le(item.offset);
 562             dest.size = u64::to_le(item.size);
 563             dest.hash = u64::to_le(item.hash);
 564         });
 565
 566         // append CaFormatGoodbyeTail as last item
 567         let offset = item_count*std::mem::size_of::<CaFormatGoodbyeItem>();
 568         let dest = crate::tools::map_struct_mut::<CaFormatGoodbyeItem>(&mut buffer[offset..]).unwrap();
 569         dest.offset = u64::to_le(goodbye_offset as u64);
 570         dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<CaFormatHeader>()) as u64);
 571         dest.hash = u64::to_le(CA_FORMAT_GOODBYE_TAIL_MARKER);
 572
 573         self.flush_copy_buffer(goodbye_table_size)?;
 574
 575         Ok(())
 576     }
 577
 578     fn encode_dir(&mut self, dir: &mut nix::dir::Dir, dir_stat: &FileStat, magic: i64, match_pattern: Vec<PxarExcludePattern>)  -> Result<(), Error> {
 579
 580         //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
 581
 582         let mut name_list = vec![];
 583
 584         let rawfd = dir.as_raw_fd();
 585
 586         let dir_start_pos = self.writer_pos;
 587
 588         let mut dir_entry = self.create_entry(&dir_stat)?;
 589
 590         self.read_chattr(rawfd, &mut dir_entry)?;
 591         self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
 592
 593         // for each node in the directory tree, the filesystem features are
 594         // checked based on the fs magic number.
 595         self.fs_feature_flags = feature_flags_from_magic(magic);
 596
 597         let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
 598         let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
 599         let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
 600         let projid = self.read_quota_project_id(rawfd, magic, &dir_stat)?;
 601
 602         self.write_entry(dir_entry)?;
 603         for xattr in xattrs {
 604             self.write_xattr(xattr)?;
 605         }
 606         self.write_fcaps(fcaps)?;
 607
 608         for user in acl_access.users {
 609             self.write_acl_user(user)?;
 610         }
 611         for group in acl_access.groups {
 612             self.write_acl_group(group)?;
 613         }
 614         if let Some(group_obj) = acl_access.group_obj {
 615             self.write_acl_group_obj(group_obj)?;
 616         }
 617
 618         for default_user in acl_default.users {
 619             self.write_acl_default_user(default_user)?;
 620         }
 621         for default_group in acl_default.groups {
 622             self.write_acl_default_group(default_group)?;
 623         }
 624         if let Some(default) = acl_default.default {
 625             self.write_acl_default(default)?;
 626         }
 627         if let Some(projid) = projid {
 628             self.write_quota_project_id(projid)?;
 629         }
 630
 631         let include_children;
 632         if is_virtual_file_system(magic) {
 633             include_children = false;
 634         } else {
 635             if let Some(set) = &self.device_set {
 636                 include_children = set.contains(&dir_stat.st_dev);
 637             } else {
 638                 include_children = true;
 639             }
 640         }
 641
 642         // Expand the exclude match pattern inherited from the parent by local entries, if present
 643         let mut local_match_pattern = match_pattern.clone();
 644         let pxar_exclude = match PxarExcludePattern::from_file(rawfd, ".pxarexclude") {
 645             Ok(Some((mut excludes, buffer, stat))) => {
 646                 local_match_pattern.append(&mut excludes);
 647                 Some((buffer, stat))
 648             },
 649             Ok(None) => None,
 650             Err(err) => bail!("error while reading exclude file - {}", err),
 651         };
 652
 653         if include_children {
 654             for entry in dir.iter() {
 655                 let entry =  entry.map_err(|err| {
 656                     format_err!("readir {:?} failed - {}", self.full_path(), err)
 657                 })?;
 658                 let filename = entry.file_name().to_owned();
 659
 660                 let name = filename.to_bytes_with_nul();
 661                 if name == b".\0" || name == b"..\0" {
 662                     continue;
 663                 }
 664
 665                 let stat = match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
 666                     Ok(stat) => stat,
 667                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 668                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 669                         self.report_vanished_file(&self.full_path().join(filename_osstr))?;
 670                         continue;
 671                     },
 672                     Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
 673                 };
 674
 675                 match match_exclude_pattern(&filename, &stat, &local_match_pattern) {
 676                     (MatchType::Exclude, _) => {
 677                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 678                         eprintln!("matched by .pxarexclude entry - skipping: {:?}", self.full_path().join(filename_osstr));
 679                     },
 680                     (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
 681                 }
 682
 683                 if name_list.len() > MAX_DIRECTORY_ENTRIES {
 684                     bail!("too many directory items in {:?} (> {})", self.full_path(), MAX_DIRECTORY_ENTRIES);
 685                 }
 686             }
 687         } else {
 688             eprintln!("skip mount point: {:?}", self.full_path());
 689         }
 690
 691         name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
 692
 693         let mut goodbye_items = vec![];
 694
 695         for (filename, stat, exclude_list) in name_list {
 696             let start_pos = self.writer_pos;
 697
 698             if filename.as_bytes() == b".pxarexclude" {
 699                 if let Some((ref content, ref stat)) = pxar_exclude {
 700                     let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
 701                         Ok(filefd) => filefd,
 702                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 703                             self.report_vanished_file(&self.full_path())?;
 704                             continue;
 705                         },
 706                         Err(err) => {
 707                             let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 708                             bail!("open file {:?} failed - {}", self.full_path().join(filename_osstr), err);
 709                         },
 710                     };
 711
 712                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 713                         detect_fs_type(filefd)?
 714                     } else {
 715                         magic
 716                     };
 717
 718                     self.write_filename(&filename)?;
 719                     self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
 720                     continue;
 721                 }
 722             }
 723
 724             self.relative_path.push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
 725
 726             if self.verbose { println!("{:?}", self.full_path()); }
 727
 728             if is_directory(&stat) {
 729
 730                 let mut dir = match nix::dir::Dir::openat(rawfd, filename.as_ref(), OFlag::O_DIRECTORY|OFlag::O_NOFOLLOW, Mode::empty()) {
 731                     Ok(dir) => dir,
 732                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 733                         self.report_vanished_file(&self.full_path())?;
 734                         continue; // fixme!!
 735                     },
 736                     Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
 737                 };
 738
 739                 let child_magic = if dir_stat.st_dev != stat.st_dev {
 740                     detect_fs_type(dir.as_raw_fd())?
 741                 } else {
 742                     magic
 743                 };
 744
 745                 self.write_filename(&filename)?;
 746                 self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
 747
 748             } else if is_reg_file(&stat) {
 749
 750                 let mut hardlink_target = None;
 751
 752                 if stat.st_nlink > 1 {
 753                     let link_info = HardLinkInfo { st_dev: stat.st_dev, st_ino: stat.st_ino };
 754                     hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
 755                         let mut target = v.clone().into_os_string();
 756                         target.push("\0"); // add Nul byte
 757                         (target, (start_pos as u64) - offset)
 758                     });
 759                     if hardlink_target == None {
 760                         self.hardlinks.insert(link_info, (self.relative_path.clone(), start_pos as u64));
 761                     }
 762                 }
 763
 764                 if let Some((target, offset)) = hardlink_target {
 765
 766                     self.write_filename(&filename)?;
 767                     self.encode_hardlink(target.as_bytes(), offset)?;
 768
 769                 } else {
 770
 771                     let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
 772                         Ok(filefd) => filefd,
 773                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 774                             self.report_vanished_file(&self.full_path())?;
 775                             continue;
 776                         },
 777                         Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
 778                     };
 779
 780                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 781                         detect_fs_type(filefd)?
 782                     } else {
 783                         magic
 784                     };
 785
 786                     self.write_filename(&filename)?;
 787                     let res = self.encode_file(filefd, &stat, child_magic);
 788                     let _ = nix::unistd::close(filefd); // ignore close errors
 789                     res?;
 790                 }
 791
 792             } else if is_symlink(&stat) {
 793                 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
 794
 795                 let res = filename.with_nix_path(|cstr| {
 796                     unsafe { libc::readlinkat(rawfd, cstr.as_ptr(), buffer.as_mut_ptr() as *mut libc::c_char, buffer.len()-1) }
 797                 })?;
 798
 799                 match Errno::result(res) {
 800                     Ok(len) => {
 801                         buffer[len as usize] = 0u8; // add Nul byte
 802                         self.write_filename(&filename)?;
 803                         self.encode_symlink(&buffer[..((len+1) as usize)], &stat)?
 804                     }
 805                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 806                         self.report_vanished_file(&self.full_path())?;
 807                         continue;
 808                     }
 809                     Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
 810                 }
 811             } else if is_block_dev(&stat) || is_char_dev(&stat) {
 812                 self.write_filename(&filename)?;
 813                 self.encode_device(&stat)?;
 814             } else if is_fifo(&stat) || is_socket(&stat) {
 815                 self.write_filename(&filename)?;
 816                 self.encode_special(&stat)?;
 817             } else {
 818                 bail!("unsupported file type (mode {:o} {:?})", stat.st_mode, self.full_path());
 819             }
 820
 821             let end_pos = self.writer_pos;
 822
 823             goodbye_items.push(CaFormatGoodbyeItem {
 824                 offset: start_pos as u64,
 825                 size: (end_pos - start_pos) as u64,
 826                 hash: compute_goodbye_hash(filename.to_bytes()),
 827             });
 828
 829             self.relative_path.pop();
 830         }
 831
 832         //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
 833
 834         // fixup goodby item offsets
 835         let goodbye_start = self.writer_pos as u64;
 836         for item in &mut goodbye_items {
 837             item.offset = goodbye_start - item.offset;
 838         }
 839
 840         let goodbye_offset = self.writer_pos - dir_start_pos;
 841
 842         self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
 843
 844         //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
 845         Ok(())
 846     }
 847
 848     fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64)  -> Result<(), Error> {
 849
 850         //println!("encode_file: {:?}", self.full_path());
 851
 852         let mut entry = self.create_entry(&stat)?;
 853
 854         self.read_chattr(filefd, &mut entry)?;
 855         self.read_fat_attr(filefd, magic, &mut entry)?;
 856         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
 857         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
 858         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
 859
 860         self.write_entry(entry)?;
 861         for xattr in xattrs {
 862             self.write_xattr(xattr)?;
 863         }
 864         self.write_fcaps(fcaps)?;
 865         for user in acl_access.users {
 866             self.write_acl_user(user)?;
 867         }
 868         for group in acl_access.groups {
 869             self.write_acl_group(group)?;
 870         }
 871         if let Some(group_obj) = acl_access.group_obj {
 872             self.write_acl_group_obj(group_obj)?;
 873         }
 874         if let Some(projid) = projid {
 875             self.write_quota_project_id(projid)?;
 876         }
 877
 878         let include_payload;
 879         if is_virtual_file_system(magic) {
 880             include_payload = false;
 881         } else {
 882             if let Some(ref set) = &self.device_set {
 883                 include_payload = set.contains(&stat.st_dev);
 884             } else {
 885                 include_payload = true;
 886             }
 887         }
 888
 889         if !include_payload {
 890             eprintln!("skip content: {:?}", self.full_path());
 891             self.write_header(CA_FORMAT_PAYLOAD, 0)?;
 892             return Ok(());
 893         }
 894
 895         let size = stat.st_size as u64;
 896
 897         self.write_header(CA_FORMAT_PAYLOAD, size)?;
 898
 899         let mut pos: u64 = 0;
 900         loop {
 901             let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
 902                 Ok(n) => n,
 903                 Err(nix::Error::Sys(Errno::EINTR)) => continue /* try again */,
 904                 Err(err) =>  bail!("read {:?} failed - {}", self.full_path(), err),
 905             };
 906             if n == 0 { // EOF
 907                 if pos != size {
 908                     // Note:: casync format cannot handle that
 909                     bail!("detected shrinked file {:?} ({} < {})", self.full_path(), pos, size);
 910                 }
 911                 break;
 912             }
 913
 914             let mut next = pos + (n as u64);
 915
 916             if next > size { next = size; }
 917
 918             let count = (next - pos) as usize;
 919
 920             self.flush_copy_buffer(count)?;
 921
 922             pos = next;
 923
 924             if pos >= size { break; }
 925         }
 926
 927         Ok(())
 928     }
 929
 930     fn encode_device(&mut self, stat: &FileStat)  -> Result<(), Error> {
 931
 932         let entry = self.create_entry(&stat)?;
 933
 934         self.write_entry(entry)?;
 935
 936         let major = unsafe { libc::major(stat.st_rdev) } as u64;
 937         let minor = unsafe { libc::minor(stat.st_rdev) } as u64;
 938
 939         //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
 940
 941         self.write_header(CA_FORMAT_DEVICE, std::mem::size_of::<CaFormatDevice>() as u64)?;
 942         self.write_item(CaFormatDevice { major, minor })?;
 943
 944         Ok(())
 945     }
 946
 947     // FIFO or Socket
 948     fn encode_special(&mut self, stat: &FileStat)  -> Result<(), Error> {
 949
 950         let entry = self.create_entry(&stat)?;
 951
 952         self.write_entry(entry)?;
 953
 954         Ok(())
 955     }
 956
 957     fn encode_symlink(&mut self, target: &[u8], stat: &FileStat)  -> Result<(), Error> {
 958
 959         //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
 960
 961         let entry = self.create_entry(&stat)?;
 962         self.write_entry(entry)?;
 963
 964         self.write_header(CA_FORMAT_SYMLINK, target.len() as u64)?;
 965         self.write(target)?;
 966
 967         Ok(())
 968     }
 969
 970     fn encode_hardlink(&mut self, target: &[u8], offset: u64)  -> Result<(), Error> {
 971
 972         //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
 973
 974         // Note: HARDLINK replaces an ENTRY.
 975         self.write_header(PXAR_FORMAT_HARDLINK, (target.len() as u64) + 8)?;
 976         self.write_item(offset)?;
 977         self.write(target)?;
 978
 979         Ok(())
 980     }
 981
 982     fn encode_pxar_exclude(&mut self, filefd: RawFd, stat: &FileStat, magic: i64, content: &[u8]) -> Result<(), Error> {
 983         let mut entry = self.create_entry(&stat)?;
 984
 985         self.read_chattr(filefd, &mut entry)?;
 986         self.read_fat_attr(filefd, magic, &mut entry)?;
 987         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
 988         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
 989         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
 990
 991         self.write_entry(entry)?;
 992         for xattr in xattrs {
 993             self.write_xattr(xattr)?;
 994         }
 995         self.write_fcaps(fcaps)?;
 996         for user in acl_access.users {
 997             self.write_acl_user(user)?;
 998         }
 999         for group in acl_access.groups {
1000             self.write_acl_group(group)?;
1001         }
1002         if let Some(group_obj) = acl_access.group_obj {
1003             self.write_acl_group_obj(group_obj)?;
1004         }
1005         if let Some(projid) = projid {
1006             self.write_quota_project_id(projid)?;
1007         }
1008
1009         let include_payload;
1010         if is_virtual_file_system(magic) {
1011             include_payload = false;
1012         } else {
1013             if let Some(set) = &self.device_set {
1014                 include_payload = set.contains(&stat.st_dev);
1015             } else {
1016                 include_payload = true;
1017             }
1018         }
1019
1020         if !include_payload {
1021             eprintln!("skip content: {:?}", self.full_path());
1022             self.write_header(CA_FORMAT_PAYLOAD, 0)?;
1023             return Ok(());
1024         }
1025
1026         let size = content.len();
1027         self.write_header(CA_FORMAT_PAYLOAD, size as u64)?;
1028         self.writer.write_all(content)?;
1029         self.writer_pos += size;
1030
1031         Ok(())
1032     }
1033
1034     // the report_XXX method may raise and error - depending on encoder configuration
1035
1036     fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
1037
1038         eprintln!("WARNING: detected vanished file {:?}", path);
1039
1040         Ok(())
1041     }
1042 }
1043
1044 // If there is a match, an updated PxarExcludePattern list to pass to the matched child is returned.
1045 fn match_exclude_pattern(
1046     filename: &CStr,
1047     stat: &FileStat,
1048     match_pattern: &Vec<PxarExcludePattern>
1049 ) ->  (MatchType, Vec<PxarExcludePattern>) {
1050     let mut child_pattern = Vec::new();
1051     let mut match_state = MatchType::None;
1052
1053     for pattern in match_pattern {
1054         match pattern.matches_filename(filename, is_directory(&stat)) {
1055             MatchType::None =>  {},
1056             MatchType::Exclude =>  match_state = MatchType::Exclude,
1057             MatchType::Include =>  match_state = MatchType::Include,
1058             MatchType::PartialExclude =>  {
1059                 if match_state != MatchType::Exclude && match_state != MatchType::Include {
1060                     match_state = MatchType::PartialExclude;
1061                 }
1062                 child_pattern.push(pattern.get_rest_pattern());
1063             },
1064             MatchType::PartialInclude =>  {
1065                 if match_state != MatchType::Exclude && match_state != MatchType::Include {
1066                     match_state = MatchType::PartialInclude;
1067                 }
1068                 child_pattern.push(pattern.get_rest_pattern());
1069             },
1070         }
1071     }
1072
1073     (match_state, child_pattern)
1074 }
1075
1076 fn errno_is_unsupported(errno: Errno) -> bool {
1077
1078     match errno {
1079         Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => {
1080             true
1081         }
1082         _ => false,
1083     }
1084 }
1085
1086 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
1087     let mut fs_stat: libc::statfs = unsafe { std::mem::uninitialized() };
1088     let res = unsafe { libc::fstatfs(fd, &mut fs_stat) };
1089     Errno::result(res)?;
1090
1091     Ok(fs_stat.f_type)
1092 }
1093
1094 #[inline(always)]
1095 pub fn is_temporary_file_system(magic: i64) -> bool {
1096     magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
1097 }
1098
1099 pub fn is_virtual_file_system(magic: i64) -> bool {
1100
1101     match magic {
1102         BINFMTFS_MAGIC |
1103         CGROUP2_SUPER_MAGIC |
1104         CGROUP_SUPER_MAGIC |
1105         CONFIGFS_MAGIC |
1106         DEBUGFS_MAGIC |
1107         DEVPTS_SUPER_MAGIC |
1108         EFIVARFS_MAGIC |
1109         FUSE_CTL_SUPER_MAGIC |
1110         HUGETLBFS_MAGIC |
1111         MQUEUE_MAGIC |
1112         NFSD_MAGIC |
1113         PROC_SUPER_MAGIC |
1114         PSTOREFS_MAGIC |
1115         RPCAUTH_GSSMAGIC |
1116         SECURITYFS_MAGIC |
1117         SELINUX_MAGIC |
1118         SMACK_MAGIC |
1119         SYSFS_MAGIC => true,
1120         _ => false
1121     }
1122 }