src/pxar/encoder.rs

   1 //! *pxar* format encoder.
   2 //!
   3 //! This module contain the code to generate *pxar* archive files.
   4
   5 use failure::*;
   6 use endian_trait::Endian;
   7 use std::collections::{HashSet, HashMap};
   8
   9 use super::format_definition::*;
  10 use super::binary_search_tree::*;
  11 use super::helper::*;
  12 use super::exclude_pattern::*;
  13 use crate::tools::fs;
  14 use crate::tools::acl;
  15 use crate::tools::xattr;
  16
  17 use std::io::Write;
  18 use std::os::unix::io::AsRawFd;
  19 use std::os::unix::ffi::OsStrExt;
  20 use std::os::unix::io::RawFd;
  21 use std::path::{Path, PathBuf};
  22
  23 use std::ffi::CStr;
  24
  25 use nix::NixPath;
  26 use nix::fcntl::OFlag;
  27 use nix::sys::stat::Mode;
  28 use nix::errno::Errno;
  29 use nix::sys::stat::FileStat;
  30
  31 use proxmox::tools::vec;
  32
  33 /// The format requires to build sorted directory lookup tables in
  34 /// memory, so we restrict the number of allowed entries to limit
  35 /// maximum memory usage.
  36 pub const MAX_DIRECTORY_ENTRIES: usize = 256*1024;
  37
  38 #[derive(Eq, PartialEq, Hash)]
  39 struct HardLinkInfo {
  40     st_dev: u64,
  41     st_ino: u64,
  42 }
  43
  44 pub struct Encoder<'a, W: Write> {
  45     base_path: PathBuf,
  46     relative_path: PathBuf,
  47     writer: &'a mut W,
  48     writer_pos: usize,
  49     _size: usize,
  50     file_copy_buffer: Vec<u8>,
  51     device_set: Option<HashSet<u64>>,
  52     verbose: bool,
  53     // Flags set by the user
  54     feature_flags: u64,
  55     // Flags signaling features supported by the filesystem
  56     fs_feature_flags: u64,
  57     hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
  58 }
  59
  60 impl <'a, W: Write> Encoder<'a, W> {
  61
  62     // used for error reporting
  63     fn full_path(&self) ->  PathBuf {
  64         self.base_path.join(&self.relative_path)
  65     }
  66
  67     /// Create archive, write result data to ``writer``.
  68     ///
  69     /// The ``device_set`` can be use used to limit included mount points.
  70     ///
  71     /// - ``None``: include all mount points
  72     /// - ``Some(set)``: only include devices listed in this set (the
  73     ///   root path device is automathically added to this list, so
  74     ///   you can pass an empty set if you want to archive a single
  75     ///   mount point.)
  76     pub fn encode(
  77         path: PathBuf,
  78         dir: &mut nix::dir::Dir,
  79         writer: &'a mut W,
  80         device_set: Option<HashSet<u64>>,
  81         verbose: bool,
  82         skip_lost_and_found: bool, // fixme: should be a feature flag ??
  83         feature_flags: u64,
  84     ) -> Result<(), Error> {
  85
  86         const FILE_COPY_BUFFER_SIZE: usize = 1024*1024;
  87
  88         let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
  89         unsafe { file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE); }
  90
  91
  92         // todo: use scandirat??
  93
  94         let dir_fd = dir.as_raw_fd();
  95         let stat = nix::sys::stat::fstat(dir_fd)
  96             .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
  97
  98         if !is_directory(&stat) {
  99             bail!("got unexpected file type {:?} (not a directory)", path);
 100         }
 101
 102         let mut device_set = device_set.clone();
 103         if let Some(ref mut set) = device_set {
 104             set.insert(stat.st_dev);
 105         }
 106
 107         let magic = detect_fs_type(dir_fd)?;
 108
 109         if is_virtual_file_system(magic) {
 110             bail!("backup virtual file systems is disabled!");
 111         }
 112
 113         let fs_feature_flags = feature_flags_from_magic(magic);
 114
 115         let mut me = Self {
 116             base_path: path,
 117             relative_path: PathBuf::new(),
 118             writer: writer,
 119             writer_pos: 0,
 120             _size: 0,
 121             file_copy_buffer,
 122             device_set,
 123             verbose,
 124             feature_flags,
 125             fs_feature_flags,
 126             hardlinks: HashMap::new(),
 127         };
 128
 129         if verbose { println!("{:?}", me.full_path()); }
 130
 131         let mut excludes = Vec::new();
 132         if skip_lost_and_found {
 133             excludes.push(PxarExcludePattern::from_line(b"**/lost+found").unwrap().unwrap());
 134         }
 135         me.encode_dir(dir, &stat, magic, excludes)?;
 136
 137         Ok(())
 138     }
 139
 140     fn write(&mut self,  buf: &[u8]) -> Result<(), Error> {
 141         self.writer.write_all(buf)?;
 142         self.writer_pos += buf.len();
 143         Ok(())
 144     }
 145
 146     fn write_item<T: Endian>(&mut self, item: T) ->  Result<(), Error> {
 147
 148         let data = item.to_le();
 149
 150         let buffer = unsafe { std::slice::from_raw_parts(
 151             &data as *const T as *const u8,
 152             std::mem::size_of::<T>()
 153         )};
 154
 155         self.write(buffer)?;
 156
 157         Ok(())
 158     }
 159
 160     fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
 161         self.writer.write_all(&self.file_copy_buffer[..size])?;
 162         self.writer_pos += size;
 163         Ok(())
 164     }
 165
 166     fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
 167
 168         let size = size + (std::mem::size_of::<CaFormatHeader>() as u64);
 169         self.write_item(CaFormatHeader { size, htype })?;
 170
 171         Ok(())
 172     }
 173
 174     fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
 175
 176         let buffer = name.to_bytes_with_nul();
 177         self.write_header(CA_FORMAT_FILENAME, buffer.len() as u64)?;
 178         self.write(buffer)?;
 179
 180         Ok(())
 181     }
 182
 183     fn create_entry(&self, stat: &FileStat) -> Result<CaFormatEntry, Error> {
 184
 185         let mode = if is_symlink(&stat) {
 186             (libc::S_IFLNK | 0o777) as u64
 187         } else {
 188             (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
 189         };
 190
 191         let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
 192         if mtime < 0 {
 193             bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
 194         }
 195
 196
 197         let entry = CaFormatEntry {
 198             mode: mode,
 199             flags: 0,
 200             uid: stat.st_uid as u64,
 201             gid: stat.st_gid as u64,
 202             mtime: mtime as u64,
 203         };
 204
 205         Ok(entry)
 206     }
 207
 208     fn read_chattr(&self, fd: RawFd, entry: &mut CaFormatEntry) -> Result<(), Error> {
 209
 210         let mut attr: usize = 0;
 211
 212         let res = unsafe { fs::read_attr_fd(fd, &mut attr)};
 213         if let Err(err) = res {
 214             if let nix::Error::Sys(errno) = err {
 215                 if errno_is_unsupported(errno) { return Ok(()) };
 216             }
 217             bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
 218         }
 219
 220         let flags = ca_feature_flags_from_chattr(attr as u32);
 221         entry.flags = entry.flags | flags;
 222
 223         Ok(())
 224     }
 225
 226     fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut CaFormatEntry) -> Result<(), Error> {
 227
 228         if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC { return Ok(()); }
 229
 230         let mut attr: u32 = 0;
 231
 232         let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr)};
 233         if let Err(err) = res {
 234             if let nix::Error::Sys(errno) = err {
 235                 if errno_is_unsupported(errno) { return Ok(()) };
 236             }
 237             bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
 238         }
 239
 240         let flags = ca_feature_flags_from_fat_attr(attr);
 241         entry.flags = entry.flags | flags;
 242
 243         Ok(())
 244     }
 245
 246     /// True if all of the given feature flags are set in the Encoder, false otherwise
 247     fn has_features(&self, feature_flags: u64) -> bool {
 248         (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
 249     }
 250
 251     /// True if at least one of the given feature flags is set in the Encoder, false otherwise
 252     fn has_some_features(&self, feature_flags: u64) -> bool {
 253         (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
 254     }
 255
 256     fn read_xattrs(&self, fd: RawFd, stat: &FileStat) -> Result<(Vec<CaFormatXAttr>, Option<CaFormatFCaps>), Error> {
 257         let mut xattrs = Vec::new();
 258         let mut fcaps = None;
 259
 260         let flags = CA_FORMAT_WITH_XATTRS | CA_FORMAT_WITH_FCAPS;
 261         if !self.has_some_features(flags) {
 262             return Ok((xattrs, fcaps));
 263         }
 264         // Should never be called on symlinks, just in case check anyway
 265         if is_symlink(&stat) {
 266             return Ok((xattrs, fcaps));
 267         }
 268
 269         let xattr_names = match xattr::flistxattr(fd) {
 270             Ok(names) => names,
 271             // Do not bail if the underlying endpoint does not supports xattrs
 272             Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
 273             // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
 274             Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
 275             Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 276         };
 277
 278         for name in xattr_names.split(|c| *c == b'\0') {
 279             // Only extract the relevant extended attributes
 280             if !xattr::is_valid_xattr_name(&name) {
 281                 continue;
 282             }
 283
 284             let value = match xattr::fgetxattr(fd, name) {
 285                 Ok(value) => value,
 286                 // Vanished between flistattr and getxattr, this is ok, silently ignore
 287                 Err(Errno::ENODATA) => continue,
 288                 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 289             };
 290
 291             if xattr::is_security_capability(&name) {
 292                 if self.has_features(CA_FORMAT_WITH_FCAPS) {
 293                     // fcaps are stored in own format within the archive
 294                     fcaps = Some(CaFormatFCaps {
 295                         data: value,
 296                     });
 297                 }
 298             } else if self.has_features(CA_FORMAT_WITH_XATTRS) {
 299                 xattrs.push(CaFormatXAttr {
 300                     name: name.to_vec(),
 301                     value: value,
 302                 });
 303             }
 304         }
 305         xattrs.sort();
 306
 307         Ok((xattrs, fcaps))
 308     }
 309
 310     fn read_acl(&self, fd: RawFd, stat: &FileStat, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 311         let ret = PxarACL {
 312             users: Vec::new(),
 313             groups: Vec::new(),
 314             group_obj: None,
 315             default: None,
 316         };
 317
 318         if !self.has_features(CA_FORMAT_WITH_ACL) {
 319             return Ok(ret);
 320         }
 321         if is_symlink(&stat) {
 322             return Ok(ret);
 323         }
 324         if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
 325             bail!("ACL_TYPE_DEFAULT only defined for directories.");
 326         }
 327
 328         // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
 329         // to create a path for acl_get_file(). acl_get_fd() only allows to get
 330         // ACL_TYPE_ACCESS attributes.
 331         let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
 332         let acl = match acl::ACL::get_file(&proc_path, acl_type) {
 333             Ok(acl) => acl,
 334             // Don't bail if underlying endpoint does not support acls
 335             Err(Errno::EOPNOTSUPP) => return Ok(ret),
 336             // Don't bail if the endpoint cannot carry acls
 337             Err(Errno::EBADF) => return Ok(ret),
 338             // Don't bail if there is no data
 339             Err(Errno::ENODATA) => return Ok(ret),
 340             Err(err) => bail!("error while reading ACL - {}", err),
 341         };
 342
 343         self.process_acl(acl, acl_type)
 344     }
 345
 346     fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 347         let mut acl_user = Vec::new();
 348         let mut acl_group = Vec::new();
 349         let mut acl_group_obj = None;
 350         let mut acl_default = None;
 351         let mut user_obj_permissions = None;
 352         let mut group_obj_permissions = None;
 353         let mut other_permissions = None;
 354         let mut mask_permissions = None;
 355
 356         for entry in &mut acl.entries() {
 357             let tag = entry.get_tag_type()?;
 358             let permissions = entry.get_permissions()?;
 359             match tag {
 360                 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
 361                 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
 362                 acl::ACL_OTHER => other_permissions = Some(permissions),
 363                 acl::ACL_MASK => mask_permissions = Some(permissions),
 364                 acl::ACL_USER => {
 365                     acl_user.push(CaFormatACLUser {
 366                         uid: entry.get_qualifier()?,
 367                         permissions: permissions,
 368                     });
 369                 },
 370                 acl::ACL_GROUP => {
 371                     acl_group.push(CaFormatACLGroup {
 372                         gid: entry.get_qualifier()?,
 373                         permissions: permissions,
 374                     });
 375                 },
 376                 _ => bail!("Unexpected ACL tag encountered!"),
 377             }
 378         }
 379
 380         acl_user.sort();
 381         acl_group.sort();
 382
 383         match acl_type {
 384             acl::ACL_TYPE_ACCESS => {
 385                 // The mask permissions are mapped to the stat group permissions
 386                 // in case that the ACL group permissions were set.
 387                 // Only in that case we need to store the group permissions,
 388                 // in the other cases they are identical to the stat group permissions.
 389                 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
 390                     acl_group_obj = Some(CaFormatACLGroupObj {
 391                         permissions: gop,
 392                     });
 393                 }
 394             },
 395             acl::ACL_TYPE_DEFAULT => {
 396                 if user_obj_permissions != None ||
 397                    group_obj_permissions != None ||
 398                    other_permissions != None ||
 399                    mask_permissions != None
 400                 {
 401                     acl_default = Some(CaFormatACLDefault {
 402                         // The value is set to UINT64_MAX as placeholder if one
 403                         // of the permissions is not set
 404                         user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
 405                         group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
 406                         other_permissions: other_permissions.unwrap_or(std::u64::MAX),
 407                         mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
 408                     });
 409                 }
 410             },
 411             _ => bail!("Unexpected ACL type encountered"),
 412         }
 413
 414         Ok(PxarACL {
 415             users: acl_user,
 416             groups: acl_group,
 417             group_obj: acl_group_obj,
 418             default: acl_default,
 419         })
 420     }
 421
 422     /// Read the project quota id for an inode, supported on ext4/XFS/FUSE/(ZFS TODO impl) filesystems
 423     fn read_quota_project_id(&self, fd: RawFd, magic: i64, stat: &FileStat) -> Result<Option<CaFormatQuotaProjID>, Error> {
 424         if !(is_directory(&stat) || is_reg_file(&stat)) {
 425             return Ok(None);
 426         }
 427         if !self.has_features(CA_FORMAT_WITH_QUOTA_PROJID) {
 428             return Ok(None);
 429         }
 430
 431         match magic {
 432             //TODO ZFS quota
 433             EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC => {
 434                 let mut fsxattr = fs::FSXAttr::default();
 435                 let res = unsafe {
 436                     fs::fs_ioc_fsgetxattr(fd, &mut fsxattr)
 437                 };
 438
 439                 // On some FUSE filesystems it can happen that ioctl is not supported.
 440                 // For these cases projid is set to 0 while the error is ignored.
 441                 if let Err(err) = res {
 442                     let errno = err.as_errno().ok_or_else(|| {
 443                         format_err!("error while reading quota project id for {:#?}", self.full_path())
 444                     })?;
 445                     if errno_is_unsupported(errno) {
 446                         return Ok(None);
 447                     } else {
 448                         bail!("error while reading quota project id for {:#?} - {}", self.full_path(), errno);
 449                     }
 450                 }
 451
 452                 let projid = fsxattr.fsx_projid as u64;
 453                 if projid == 0 {
 454                     return Ok(None);
 455                 } else {
 456                     return Ok(Some(CaFormatQuotaProjID { projid }));
 457                 }
 458             },
 459             _ => return Ok(None),
 460         }
 461     }
 462
 463     fn write_entry(&mut self, entry: CaFormatEntry) -> Result<(), Error> {
 464
 465         self.write_header(CA_FORMAT_ENTRY, std::mem::size_of::<CaFormatEntry>() as u64)?;
 466         self.write_item(entry)?;
 467
 468         Ok(())
 469     }
 470
 471     fn write_xattr(&mut self, xattr: CaFormatXAttr) -> Result<(), Error> {
 472         let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
 473         self.write_header(CA_FORMAT_XATTR, size as u64)?;
 474         self.write(xattr.name.as_slice())?;
 475         self.write(&[0])?;
 476         self.write(xattr.value.as_slice())?;
 477
 478         Ok(())
 479     }
 480
 481     fn write_fcaps(&mut self, fcaps: Option<CaFormatFCaps>) -> Result<(), Error> {
 482         if let Some(fcaps) = fcaps {
 483             let size = fcaps.data.len();
 484             self.write_header(CA_FORMAT_FCAPS, size as u64)?;
 485             self.write(fcaps.data.as_slice())?;
 486         }
 487
 488         Ok(())
 489     }
 490
 491     fn write_acl_user(&mut self, acl_user: CaFormatACLUser) -> Result<(), Error> {
 492         self.write_header(CA_FORMAT_ACL_USER,  std::mem::size_of::<CaFormatACLUser>() as u64)?;
 493         self.write_item(acl_user)?;
 494
 495         Ok(())
 496     }
 497
 498     fn write_acl_group(&mut self, acl_group: CaFormatACLGroup) -> Result<(), Error> {
 499         self.write_header(CA_FORMAT_ACL_GROUP,  std::mem::size_of::<CaFormatACLGroup>() as u64)?;
 500         self.write_item(acl_group)?;
 501
 502         Ok(())
 503     }
 504
 505     fn write_acl_group_obj(&mut self, acl_group_obj: CaFormatACLGroupObj) -> Result<(), Error> {
 506         self.write_header(CA_FORMAT_ACL_GROUP_OBJ,  std::mem::size_of::<CaFormatACLGroupObj>() as u64)?;
 507         self.write_item(acl_group_obj)?;
 508
 509         Ok(())
 510     }
 511
 512     fn write_acl_default(&mut self, acl_default: CaFormatACLDefault) -> Result<(), Error> {
 513         self.write_header(CA_FORMAT_ACL_DEFAULT,  std::mem::size_of::<CaFormatACLDefault>() as u64)?;
 514         self.write_item(acl_default)?;
 515
 516         Ok(())
 517     }
 518
 519     fn write_acl_default_user(&mut self, acl_default_user: CaFormatACLUser) -> Result<(), Error> {
 520         self.write_header(CA_FORMAT_ACL_DEFAULT_USER,  std::mem::size_of::<CaFormatACLUser>() as u64)?;
 521         self.write_item(acl_default_user)?;
 522
 523         Ok(())
 524     }
 525
 526     fn write_acl_default_group(&mut self, acl_default_group: CaFormatACLGroup) -> Result<(), Error> {
 527         self.write_header(CA_FORMAT_ACL_DEFAULT_GROUP,  std::mem::size_of::<CaFormatACLGroup>() as u64)?;
 528         self.write_item(acl_default_group)?;
 529
 530         Ok(())
 531     }
 532
 533     fn write_quota_project_id(&mut self, projid: CaFormatQuotaProjID) -> Result<(), Error> {
 534         self.write_header(CA_FORMAT_QUOTA_PROJID, std::mem::size_of::<CaFormatQuotaProjID>() as u64)?;
 535         self.write_item(projid)?;
 536
 537         Ok(())
 538     }
 539
 540     fn write_goodbye_table(&mut self, goodbye_offset: usize, goodbye_items: &mut [CaFormatGoodbyeItem]) -> Result<(), Error> {
 541
 542         goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
 543
 544         let item_count = goodbye_items.len();
 545
 546         let goodbye_table_size = (item_count + 1)*std::mem::size_of::<CaFormatGoodbyeItem>();
 547
 548         self.write_header(CA_FORMAT_GOODBYE, goodbye_table_size as u64)?;
 549
 550         if self.file_copy_buffer.len() < goodbye_table_size {
 551             let need = goodbye_table_size - self.file_copy_buffer.len();
 552             self.file_copy_buffer.reserve(need);
 553             unsafe { self.file_copy_buffer.set_len(self.file_copy_buffer.capacity()); }
 554         }
 555
 556         let buffer = &mut self.file_copy_buffer;
 557
 558         copy_binary_search_tree(item_count, |s, d| {
 559             let item = &goodbye_items[s];
 560             let offset = d*std::mem::size_of::<CaFormatGoodbyeItem>();
 561             let dest = crate::tools::map_struct_mut::<CaFormatGoodbyeItem>(&mut buffer[offset..]).unwrap();
 562             dest.offset = u64::to_le(item.offset);
 563             dest.size = u64::to_le(item.size);
 564             dest.hash = u64::to_le(item.hash);
 565         });
 566
 567         // append CaFormatGoodbyeTail as last item
 568         let offset = item_count*std::mem::size_of::<CaFormatGoodbyeItem>();
 569         let dest = crate::tools::map_struct_mut::<CaFormatGoodbyeItem>(&mut buffer[offset..]).unwrap();
 570         dest.offset = u64::to_le(goodbye_offset as u64);
 571         dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<CaFormatHeader>()) as u64);
 572         dest.hash = u64::to_le(CA_FORMAT_GOODBYE_TAIL_MARKER);
 573
 574         self.flush_copy_buffer(goodbye_table_size)?;
 575
 576         Ok(())
 577     }
 578
 579     fn encode_dir(&mut self, dir: &mut nix::dir::Dir, dir_stat: &FileStat, magic: i64, match_pattern: Vec<PxarExcludePattern>)  -> Result<(), Error> {
 580
 581         //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
 582
 583         let mut name_list = vec![];
 584
 585         let rawfd = dir.as_raw_fd();
 586
 587         let dir_start_pos = self.writer_pos;
 588
 589         let mut dir_entry = self.create_entry(&dir_stat)?;
 590
 591         self.read_chattr(rawfd, &mut dir_entry)?;
 592         self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
 593
 594         // for each node in the directory tree, the filesystem features are
 595         // checked based on the fs magic number.
 596         self.fs_feature_flags = feature_flags_from_magic(magic);
 597
 598         let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
 599         let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
 600         let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
 601         let projid = self.read_quota_project_id(rawfd, magic, &dir_stat)?;
 602
 603         self.write_entry(dir_entry)?;
 604         for xattr in xattrs {
 605             self.write_xattr(xattr)?;
 606         }
 607         self.write_fcaps(fcaps)?;
 608
 609         for user in acl_access.users {
 610             self.write_acl_user(user)?;
 611         }
 612         for group in acl_access.groups {
 613             self.write_acl_group(group)?;
 614         }
 615         if let Some(group_obj) = acl_access.group_obj {
 616             self.write_acl_group_obj(group_obj)?;
 617         }
 618
 619         for default_user in acl_default.users {
 620             self.write_acl_default_user(default_user)?;
 621         }
 622         for default_group in acl_default.groups {
 623             self.write_acl_default_group(default_group)?;
 624         }
 625         if let Some(default) = acl_default.default {
 626             self.write_acl_default(default)?;
 627         }
 628         if let Some(projid) = projid {
 629             self.write_quota_project_id(projid)?;
 630         }
 631
 632         let include_children;
 633         if is_virtual_file_system(magic) {
 634             include_children = false;
 635         } else {
 636             if let Some(set) = &self.device_set {
 637                 include_children = set.contains(&dir_stat.st_dev);
 638             } else {
 639                 include_children = true;
 640             }
 641         }
 642
 643         // Expand the exclude match pattern inherited from the parent by local entries, if present
 644         let mut local_match_pattern = match_pattern.clone();
 645         let pxar_exclude = match PxarExcludePattern::from_file(rawfd, ".pxarexclude") {
 646             Ok(Some((mut excludes, buffer, stat))) => {
 647                 local_match_pattern.append(&mut excludes);
 648                 Some((buffer, stat))
 649             },
 650             Ok(None) => None,
 651             Err(err) => bail!("error while reading exclude file - {}", err),
 652         };
 653
 654         if include_children {
 655             for entry in dir.iter() {
 656                 let entry =  entry.map_err(|err| {
 657                     format_err!("readir {:?} failed - {}", self.full_path(), err)
 658                 })?;
 659                 let filename = entry.file_name().to_owned();
 660
 661                 let name = filename.to_bytes_with_nul();
 662                 if name == b".\0" || name == b"..\0" {
 663                     continue;
 664                 }
 665
 666                 let stat = match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
 667                     Ok(stat) => stat,
 668                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 669                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 670                         self.report_vanished_file(&self.full_path().join(filename_osstr))?;
 671                         continue;
 672                     },
 673                     Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
 674                 };
 675
 676                 match match_exclude_pattern(&filename, &stat, &local_match_pattern) {
 677                     (MatchType::Exclude, _) => {
 678                         let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 679                         eprintln!("matched by .pxarexclude entry - skipping: {:?}", self.full_path().join(filename_osstr));
 680                     },
 681                     (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
 682                 }
 683
 684                 if name_list.len() > MAX_DIRECTORY_ENTRIES {
 685                     bail!("too many directory items in {:?} (> {})", self.full_path(), MAX_DIRECTORY_ENTRIES);
 686                 }
 687             }
 688         } else {
 689             eprintln!("skip mount point: {:?}", self.full_path());
 690         }
 691
 692         name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
 693
 694         let mut goodbye_items = vec![];
 695
 696         for (filename, stat, exclude_list) in name_list {
 697             let start_pos = self.writer_pos;
 698
 699             if filename.as_bytes() == b".pxarexclude" {
 700                 if let Some((ref content, ref stat)) = pxar_exclude {
 701                     let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
 702                         Ok(filefd) => filefd,
 703                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 704                             self.report_vanished_file(&self.full_path())?;
 705                             continue;
 706                         },
 707                         Err(err) => {
 708                             let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
 709                             bail!("open file {:?} failed - {}", self.full_path().join(filename_osstr), err);
 710                         },
 711                     };
 712
 713                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 714                         detect_fs_type(filefd)?
 715                     } else {
 716                         magic
 717                     };
 718
 719                     self.write_filename(&filename)?;
 720                     self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
 721                     continue;
 722                 }
 723             }
 724
 725             self.relative_path.push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
 726
 727             if self.verbose { println!("{:?}", self.full_path()); }
 728
 729             if is_directory(&stat) {
 730
 731                 let mut dir = match nix::dir::Dir::openat(rawfd, filename.as_ref(), OFlag::O_DIRECTORY|OFlag::O_NOFOLLOW, Mode::empty()) {
 732                     Ok(dir) => dir,
 733                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 734                         self.report_vanished_file(&self.full_path())?;
 735                         continue; // fixme!!
 736                     },
 737                     Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
 738                 };
 739
 740                 let child_magic = if dir_stat.st_dev != stat.st_dev {
 741                     detect_fs_type(dir.as_raw_fd())?
 742                 } else {
 743                     magic
 744                 };
 745
 746                 self.write_filename(&filename)?;
 747                 self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
 748
 749             } else if is_reg_file(&stat) {
 750
 751                 let mut hardlink_target = None;
 752
 753                 if stat.st_nlink > 1 {
 754                     let link_info = HardLinkInfo { st_dev: stat.st_dev, st_ino: stat.st_ino };
 755                     hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
 756                         let mut target = v.clone().into_os_string();
 757                         target.push("\0"); // add Nul byte
 758                         (target, (start_pos as u64) - offset)
 759                     });
 760                     if hardlink_target == None {
 761                         self.hardlinks.insert(link_info, (self.relative_path.clone(), start_pos as u64));
 762                     }
 763                 }
 764
 765                 if let Some((target, offset)) = hardlink_target {
 766
 767                     self.write_filename(&filename)?;
 768                     self.encode_hardlink(target.as_bytes(), offset)?;
 769
 770                 } else {
 771
 772                     let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
 773                         Ok(filefd) => filefd,
 774                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 775                             self.report_vanished_file(&self.full_path())?;
 776                             continue;
 777                         },
 778                         Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
 779                     };
 780
 781                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 782                         detect_fs_type(filefd)?
 783                     } else {
 784                         magic
 785                     };
 786
 787                     self.write_filename(&filename)?;
 788                     let res = self.encode_file(filefd, &stat, child_magic);
 789                     let _ = nix::unistd::close(filefd); // ignore close errors
 790                     res?;
 791                 }
 792
 793             } else if is_symlink(&stat) {
 794                 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
 795
 796                 let res = filename.with_nix_path(|cstr| {
 797                     unsafe { libc::readlinkat(rawfd, cstr.as_ptr(), buffer.as_mut_ptr() as *mut libc::c_char, buffer.len()-1) }
 798                 })?;
 799
 800                 match Errno::result(res) {
 801                     Ok(len) => {
 802                         buffer[len as usize] = 0u8; // add Nul byte
 803                         self.write_filename(&filename)?;
 804                         self.encode_symlink(&buffer[..((len+1) as usize)], &stat)?
 805                     }
 806                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 807                         self.report_vanished_file(&self.full_path())?;
 808                         continue;
 809                     }
 810                     Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
 811                 }
 812             } else if is_block_dev(&stat) || is_char_dev(&stat) {
 813                 self.write_filename(&filename)?;
 814                 self.encode_device(&stat)?;
 815             } else if is_fifo(&stat) || is_socket(&stat) {
 816                 self.write_filename(&filename)?;
 817                 self.encode_special(&stat)?;
 818             } else {
 819                 bail!("unsupported file type (mode {:o} {:?})", stat.st_mode, self.full_path());
 820             }
 821
 822             let end_pos = self.writer_pos;
 823
 824             goodbye_items.push(CaFormatGoodbyeItem {
 825                 offset: start_pos as u64,
 826                 size: (end_pos - start_pos) as u64,
 827                 hash: compute_goodbye_hash(filename.to_bytes()),
 828             });
 829
 830             self.relative_path.pop();
 831         }
 832
 833         //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
 834
 835         // fixup goodby item offsets
 836         let goodbye_start = self.writer_pos as u64;
 837         for item in &mut goodbye_items {
 838             item.offset = goodbye_start - item.offset;
 839         }
 840
 841         let goodbye_offset = self.writer_pos - dir_start_pos;
 842
 843         self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
 844
 845         //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
 846         Ok(())
 847     }
 848
 849     fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64)  -> Result<(), Error> {
 850
 851         //println!("encode_file: {:?}", self.full_path());
 852
 853         let mut entry = self.create_entry(&stat)?;
 854
 855         self.read_chattr(filefd, &mut entry)?;
 856         self.read_fat_attr(filefd, magic, &mut entry)?;
 857         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
 858         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
 859         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
 860
 861         self.write_entry(entry)?;
 862         for xattr in xattrs {
 863             self.write_xattr(xattr)?;
 864         }
 865         self.write_fcaps(fcaps)?;
 866         for user in acl_access.users {
 867             self.write_acl_user(user)?;
 868         }
 869         for group in acl_access.groups {
 870             self.write_acl_group(group)?;
 871         }
 872         if let Some(group_obj) = acl_access.group_obj {
 873             self.write_acl_group_obj(group_obj)?;
 874         }
 875         if let Some(projid) = projid {
 876             self.write_quota_project_id(projid)?;
 877         }
 878
 879         let include_payload;
 880         if is_virtual_file_system(magic) {
 881             include_payload = false;
 882         } else {
 883             if let Some(ref set) = &self.device_set {
 884                 include_payload = set.contains(&stat.st_dev);
 885             } else {
 886                 include_payload = true;
 887             }
 888         }
 889
 890         if !include_payload {
 891             eprintln!("skip content: {:?}", self.full_path());
 892             self.write_header(CA_FORMAT_PAYLOAD, 0)?;
 893             return Ok(());
 894         }
 895
 896         let size = stat.st_size as u64;
 897
 898         self.write_header(CA_FORMAT_PAYLOAD, size)?;
 899
 900         let mut pos: u64 = 0;
 901         loop {
 902             let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
 903                 Ok(n) => n,
 904                 Err(nix::Error::Sys(Errno::EINTR)) => continue /* try again */,
 905                 Err(err) =>  bail!("read {:?} failed - {}", self.full_path(), err),
 906             };
 907             if n == 0 { // EOF
 908                 if pos != size {
 909                     // Note:: casync format cannot handle that
 910                     bail!("detected shrinked file {:?} ({} < {})", self.full_path(), pos, size);
 911                 }
 912                 break;
 913             }
 914
 915             let mut next = pos + (n as u64);
 916
 917             if next > size { next = size; }
 918
 919             let count = (next - pos) as usize;
 920
 921             self.flush_copy_buffer(count)?;
 922
 923             pos = next;
 924
 925             if pos >= size { break; }
 926         }
 927
 928         Ok(())
 929     }
 930
 931     fn encode_device(&mut self, stat: &FileStat)  -> Result<(), Error> {
 932
 933         let entry = self.create_entry(&stat)?;
 934
 935         self.write_entry(entry)?;
 936
 937         let major = unsafe { libc::major(stat.st_rdev) } as u64;
 938         let minor = unsafe { libc::minor(stat.st_rdev) } as u64;
 939
 940         //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
 941
 942         self.write_header(CA_FORMAT_DEVICE, std::mem::size_of::<CaFormatDevice>() as u64)?;
 943         self.write_item(CaFormatDevice { major, minor })?;
 944
 945         Ok(())
 946     }
 947
 948     // FIFO or Socket
 949     fn encode_special(&mut self, stat: &FileStat)  -> Result<(), Error> {
 950
 951         let entry = self.create_entry(&stat)?;
 952
 953         self.write_entry(entry)?;
 954
 955         Ok(())
 956     }
 957
 958     fn encode_symlink(&mut self, target: &[u8], stat: &FileStat)  -> Result<(), Error> {
 959
 960         //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
 961
 962         let entry = self.create_entry(&stat)?;
 963         self.write_entry(entry)?;
 964
 965         self.write_header(CA_FORMAT_SYMLINK, target.len() as u64)?;
 966         self.write(target)?;
 967
 968         Ok(())
 969     }
 970
 971     fn encode_hardlink(&mut self, target: &[u8], offset: u64)  -> Result<(), Error> {
 972
 973         //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
 974
 975         // Note: HARDLINK replaces an ENTRY.
 976         self.write_header(PXAR_FORMAT_HARDLINK, (target.len() as u64) + 8)?;
 977         self.write_item(offset)?;
 978         self.write(target)?;
 979
 980         Ok(())
 981     }
 982
 983     fn encode_pxar_exclude(&mut self, filefd: RawFd, stat: &FileStat, magic: i64, content: &[u8]) -> Result<(), Error> {
 984         let mut entry = self.create_entry(&stat)?;
 985
 986         self.read_chattr(filefd, &mut entry)?;
 987         self.read_fat_attr(filefd, magic, &mut entry)?;
 988         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
 989         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
 990         let projid = self.read_quota_project_id(filefd, magic, &stat)?;
 991
 992         self.write_entry(entry)?;
 993         for xattr in xattrs {
 994             self.write_xattr(xattr)?;
 995         }
 996         self.write_fcaps(fcaps)?;
 997         for user in acl_access.users {
 998             self.write_acl_user(user)?;
 999         }
1000         for group in acl_access.groups {
1001             self.write_acl_group(group)?;
1002         }
1003         if let Some(group_obj) = acl_access.group_obj {
1004             self.write_acl_group_obj(group_obj)?;
1005         }
1006         if let Some(projid) = projid {
1007             self.write_quota_project_id(projid)?;
1008         }
1009
1010         let include_payload;
1011         if is_virtual_file_system(magic) {
1012             include_payload = false;
1013         } else {
1014             if let Some(set) = &self.device_set {
1015                 include_payload = set.contains(&stat.st_dev);
1016             } else {
1017                 include_payload = true;
1018             }
1019         }
1020
1021         if !include_payload {
1022             eprintln!("skip content: {:?}", self.full_path());
1023             self.write_header(CA_FORMAT_PAYLOAD, 0)?;
1024             return Ok(());
1025         }
1026
1027         let size = content.len();
1028         self.write_header(CA_FORMAT_PAYLOAD, size as u64)?;
1029         self.writer.write_all(content)?;
1030         self.writer_pos += size;
1031
1032         Ok(())
1033     }
1034
1035     // the report_XXX method may raise and error - depending on encoder configuration
1036
1037     fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
1038
1039         eprintln!("WARNING: detected vanished file {:?}", path);
1040
1041         Ok(())
1042     }
1043 }
1044
1045 // If there is a match, an updated PxarExcludePattern list to pass to the matched child is returned.
1046 fn match_exclude_pattern(
1047     filename: &CStr,
1048     stat: &FileStat,
1049     match_pattern: &Vec<PxarExcludePattern>
1050 ) ->  (MatchType, Vec<PxarExcludePattern>) {
1051     let mut child_pattern = Vec::new();
1052     let mut match_state = MatchType::None;
1053
1054     for pattern in match_pattern {
1055         match pattern.matches_filename(filename, is_directory(&stat)) {
1056             MatchType::None =>  {},
1057             MatchType::Exclude =>  match_state = MatchType::Exclude,
1058             MatchType::Include =>  match_state = MatchType::Include,
1059             MatchType::PartialExclude =>  {
1060                 if match_state != MatchType::Exclude && match_state != MatchType::Include {
1061                     match_state = MatchType::PartialExclude;
1062                 }
1063                 child_pattern.push(pattern.get_rest_pattern());
1064             },
1065             MatchType::PartialInclude =>  {
1066                 if match_state != MatchType::Exclude && match_state != MatchType::Include {
1067                     match_state = MatchType::PartialInclude;
1068                 }
1069                 child_pattern.push(pattern.get_rest_pattern());
1070             },
1071         }
1072     }
1073
1074     (match_state, child_pattern)
1075 }
1076
1077 fn errno_is_unsupported(errno: Errno) -> bool {
1078
1079     match errno {
1080         Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => {
1081             true
1082         }
1083         _ => false,
1084     }
1085 }
1086
1087 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
1088     let mut fs_stat: libc::statfs = unsafe { std::mem::uninitialized() };
1089     let res = unsafe { libc::fstatfs(fd, &mut fs_stat) };
1090     Errno::result(res)?;
1091
1092     Ok(fs_stat.f_type)
1093 }
1094
1095 #[inline(always)]
1096 pub fn is_temporary_file_system(magic: i64) -> bool {
1097     magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
1098 }
1099
1100 pub fn is_virtual_file_system(magic: i64) -> bool {
1101
1102     match magic {
1103         BINFMTFS_MAGIC |
1104         CGROUP2_SUPER_MAGIC |
1105         CGROUP_SUPER_MAGIC |
1106         CONFIGFS_MAGIC |
1107         DEBUGFS_MAGIC |
1108         DEVPTS_SUPER_MAGIC |
1109         EFIVARFS_MAGIC |
1110         FUSE_CTL_SUPER_MAGIC |
1111         HUGETLBFS_MAGIC |
1112         MQUEUE_MAGIC |
1113         NFSD_MAGIC |
1114         PROC_SUPER_MAGIC |
1115         PSTOREFS_MAGIC |
1116         RPCAUTH_GSSMAGIC |
1117         SECURITYFS_MAGIC |
1118         SELINUX_MAGIC |
1119         SMACK_MAGIC |
1120         SYSFS_MAGIC => true,
1121         _ => false
1122     }
1123 }