src/pxar/encoder.rs

   1 //! *pxar* format encoder.
   2 //!
   3 //! This module contain the code to generate *pxar* archive files.
   4
   5 use failure::*;
   6 use endian_trait::Endian;
   7 use std::collections::HashMap;
   8
   9 use super::format_definition::*;
  10 use super::binary_search_tree::*;
  11 use super::helper::*;
  12 use crate::tools::acl;
  13 use crate::tools::xattr;
  14
  15 use std::io::Write;
  16 use std::os::unix::io::AsRawFd;
  17 use std::os::unix::ffi::OsStrExt;
  18 use std::os::unix::io::RawFd;
  19 use std::path::{Path, PathBuf};
  20
  21 use std::ffi::CStr;
  22
  23 use nix::NixPath;
  24 use nix::fcntl::OFlag;
  25 use nix::sys::stat::Mode;
  26 use nix::errno::Errno;
  27 use nix::sys::stat::FileStat;
  28
  29 use crate::tools::vec;
  30
  31 /// The format requires to build sorted directory lookup tables in
  32 /// memory, so we restrict the number of allowed entries to limit
  33 /// maximum memory usage.
  34 pub const MAX_DIRECTORY_ENTRIES: usize = 256*1024;
  35
  36 #[derive(Eq, PartialEq, Hash)]
  37 struct HardLinkInfo {
  38     st_dev: u64,
  39     st_ino: u64,
  40 }
  41
  42 pub struct Encoder<'a, W: Write> {
  43     base_path: PathBuf,
  44     relative_path: PathBuf,
  45     writer: &'a mut W,
  46     writer_pos: usize,
  47     _size: usize,
  48     file_copy_buffer: Vec<u8>,
  49     all_file_systems: bool,
  50     root_st_dev: u64,
  51     verbose: bool,
  52     feature_flags: u64,
  53     hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
  54 }
  55
  56 impl <'a, W: Write> Encoder<'a, W> {
  57
  58     // used for error reporting
  59     fn full_path(&self) ->  PathBuf {
  60         self.base_path.join(&self.relative_path)
  61     }
  62
  63     pub fn encode(
  64         path: PathBuf,
  65         dir: &mut nix::dir::Dir,
  66         writer: &'a mut W,
  67         all_file_systems: bool,
  68         verbose: bool,
  69         feature_flags: u64,
  70     ) -> Result<(), Error> {
  71
  72         const FILE_COPY_BUFFER_SIZE: usize = 1024*1024;
  73
  74         let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
  75         unsafe { file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE); }
  76
  77
  78         // todo: use scandirat??
  79
  80         let dir_fd = dir.as_raw_fd();
  81         let stat = match nix::sys::stat::fstat(dir_fd) {
  82             Ok(stat) => stat,
  83             Err(err) => bail!("fstat {:?} failed - {}", path, err),
  84         };
  85
  86         if !is_directory(&stat) {
  87             bail!("got unexpected file type {:?} (not a directory)", path);
  88         }
  89
  90         let magic = detect_fs_type(dir_fd)?;
  91
  92         if is_virtual_file_system(magic) {
  93             bail!("backup virtual file systems is disabled!");
  94         }
  95
  96         let mut me = Self {
  97             base_path: path,
  98             relative_path: PathBuf::new(),
  99             writer: writer,
 100             writer_pos: 0,
 101             _size: 0,
 102             file_copy_buffer,
 103             all_file_systems,
 104             root_st_dev: stat.st_dev,
 105             verbose,
 106             feature_flags,
 107             hardlinks: HashMap::new(),
 108         };
 109
 110         if verbose { println!("{:?}", me.full_path()); }
 111
 112         me.encode_dir(dir, &stat, magic)?;
 113
 114         Ok(())
 115     }
 116
 117     fn write(&mut self,  buf: &[u8]) -> Result<(), Error> {
 118         self.writer.write_all(buf)?;
 119         self.writer_pos += buf.len();
 120         Ok(())
 121     }
 122
 123     fn write_item<T: Endian>(&mut self, item: T) ->  Result<(), Error> {
 124
 125         let data = item.to_le();
 126
 127         let buffer = unsafe { std::slice::from_raw_parts(
 128             &data as *const T as *const u8,
 129             std::mem::size_of::<T>()
 130         )};
 131
 132         self.write(buffer)?;
 133
 134         Ok(())
 135     }
 136
 137     fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
 138         self.writer.write_all(&self.file_copy_buffer[..size])?;
 139         self.writer_pos += size;
 140         Ok(())
 141     }
 142
 143     fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
 144
 145         let size = size + (std::mem::size_of::<CaFormatHeader>() as u64);
 146         self.write_item(CaFormatHeader { size, htype })?;
 147
 148         Ok(())
 149     }
 150
 151     fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
 152
 153         let buffer = name.to_bytes_with_nul();
 154         self.write_header(CA_FORMAT_FILENAME, buffer.len() as u64)?;
 155         self.write(buffer)?;
 156
 157         Ok(())
 158     }
 159
 160     fn create_entry(&self, stat: &FileStat) -> Result<CaFormatEntry, Error> {
 161
 162         let mode = if is_symlink(&stat) {
 163             (libc::S_IFLNK | 0o777) as u64
 164         } else {
 165             (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
 166         };
 167
 168         let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
 169         if mtime < 0 {
 170             bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
 171         }
 172
 173
 174         let entry = CaFormatEntry {
 175             mode: mode,
 176             flags: 0,
 177             uid: stat.st_uid as u64,
 178             gid: stat.st_gid as u64,
 179             mtime: mtime as u64,
 180         };
 181
 182         Ok(entry)
 183     }
 184
 185     fn read_chattr(&self, fd: RawFd, entry: &mut CaFormatEntry) -> Result<(), Error> {
 186
 187         let mut attr: usize = 0;
 188
 189         let res = unsafe { read_attr_fd(fd, &mut attr)};
 190         if let Err(err) = res {
 191             if let nix::Error::Sys(errno) = err {
 192                 if errno_is_unsupported(errno) { return Ok(()) };
 193             }
 194             bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
 195         }
 196
 197         let flags = ca_feature_flags_from_chattr(attr as u32);
 198         entry.flags = entry.flags | flags;
 199
 200         Ok(())
 201     }
 202
 203     fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut CaFormatEntry) -> Result<(), Error> {
 204
 205         if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC { return Ok(()); }
 206
 207         let mut attr: u32 = 0;
 208
 209         let res = unsafe { read_fat_attr_fd(fd, &mut attr)};
 210         if let Err(err) = res {
 211             if let nix::Error::Sys(errno) = err {
 212                 if errno_is_unsupported(errno) { return Ok(()) };
 213             }
 214             bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
 215         }
 216
 217         let flags = ca_feature_flags_from_fat_attr(attr);
 218         entry.flags = entry.flags | flags;
 219
 220         Ok(())
 221     }
 222
 223     fn has_features(&self, feature_flags: u64) -> bool {
 224         (self.feature_flags & feature_flags) == feature_flags
 225     }
 226
 227     fn read_xattrs(&self, fd: RawFd, stat: &FileStat) -> Result<(Vec<CaFormatXAttr>, Option<CaFormatFCaps>), Error> {
 228         let mut xattrs = Vec::new();
 229         let mut fcaps = None;
 230
 231         let flags = CA_FORMAT_WITH_XATTRS | CA_FORMAT_WITH_FCAPS;
 232         if !self.has_features(flags) {
 233             return Ok((xattrs, fcaps));
 234         }
 235         // Should never be called on symlinks, just in case check anyway
 236         if is_symlink(&stat) {
 237             return Ok((xattrs, fcaps));
 238         }
 239
 240         let xattr_names = match xattr::flistxattr(fd) {
 241             Ok(names) => names,
 242             // Do not bail if the underlying endpoint does not supports xattrs
 243             Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
 244             // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
 245             Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
 246             Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 247         };
 248
 249         for name in xattr_names.split(|c| *c == b'\0') {
 250             // Only extract the relevant extended attributes
 251             if !xattr::is_valid_xattr_name(&name) {
 252                 continue;
 253             }
 254
 255             let value = match xattr::fgetxattr(fd, name) {
 256                 Ok(value) => value,
 257                 // Vanished between flistattr and getxattr, this is ok, silently ignore
 258                 Err(Errno::ENODATA) => continue,
 259                 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
 260             };
 261
 262             if xattr::is_security_capability(&name) {
 263                 // fcaps are stored in own format within the archive
 264                 fcaps = Some(CaFormatFCaps {
 265                     data: value,
 266                 });
 267             } else {
 268                 xattrs.push(CaFormatXAttr {
 269                     name: name.to_vec(),
 270                     value: value,
 271                 });
 272             }
 273         }
 274         xattrs.sort();
 275
 276         Ok((xattrs, fcaps))
 277     }
 278
 279     fn read_acl(&self, fd: RawFd, stat: &FileStat, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 280         let ret = PxarACL {
 281             users: Vec::new(),
 282             groups: Vec::new(),
 283             group_obj: None,
 284             default: None,
 285         };
 286
 287         if !self.has_features(CA_FORMAT_WITH_ACL) {
 288             return Ok(ret);
 289         }
 290         if is_symlink(&stat) {
 291             return Ok(ret);
 292         }
 293         if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
 294             bail!("ACL_TYPE_DEFAULT only defined for directories.");
 295         }
 296
 297         // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
 298         // to create a path for acl_get_file(). acl_get_fd() only allows to get
 299         // ACL_TYPE_ACCESS attributes.
 300         let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
 301         let acl = match acl::ACL::get_file(&proc_path, acl_type) {
 302             Ok(acl) => acl,
 303             // Don't bail if underlying endpoint does not support acls
 304             Err(Errno::EOPNOTSUPP) => return Ok(ret),
 305             // Don't bail if the endpoint cannot carry acls
 306             Err(Errno::EBADF) => return Ok(ret),
 307             // Don't bail if there is no data
 308             Err(Errno::ENODATA) => return Ok(ret),
 309             Err(err) => bail!("error while reading ACL - {}", err),
 310         };
 311
 312         self.process_acl(acl, acl_type)
 313     }
 314
 315     fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
 316         let mut acl_user = Vec::new();
 317         let mut acl_group = Vec::new();
 318         let mut acl_group_obj = None;
 319         let mut acl_default = None;
 320         let mut user_obj_permissions = None;
 321         let mut group_obj_permissions = None;
 322         let mut other_permissions = None;
 323         let mut mask_permissions = None;
 324
 325         for entry in &mut acl.entries() {
 326             let tag = entry.get_tag_type()?;
 327             let permissions = entry.get_permissions()?;
 328             match tag {
 329                 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
 330                 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
 331                 acl::ACL_OTHER => other_permissions = Some(permissions),
 332                 acl::ACL_MASK => mask_permissions = Some(permissions),
 333                 acl::ACL_USER => {
 334                     acl_user.push(CaFormatACLUser {
 335                         uid: entry.get_qualifier()?,
 336                         permissions: permissions,
 337                     });
 338                 },
 339                 acl::ACL_GROUP => {
 340                     acl_group.push(CaFormatACLGroup {
 341                         gid: entry.get_qualifier()?,
 342                         permissions: permissions,
 343                     });
 344                 },
 345                 _ => bail!("Unexpected ACL tag encountered!"),
 346             }
 347         }
 348
 349         acl_user.sort();
 350         acl_group.sort();
 351
 352         match acl_type {
 353             acl::ACL_TYPE_ACCESS => {
 354                 // The mask permissions are mapped to the stat group permissions
 355                 // in case that the ACL group permissions were set.
 356                 // Only in that case we need to store the group permissions,
 357                 // in the other cases they are identical to the stat group permissions.
 358                 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
 359                     acl_group_obj = Some(CaFormatACLGroupObj {
 360                         permissions: gop,
 361                     });
 362                 }
 363             },
 364             acl::ACL_TYPE_DEFAULT => {
 365                 if user_obj_permissions != None ||
 366                    group_obj_permissions != None ||
 367                    other_permissions != None ||
 368                    mask_permissions != None
 369                 {
 370                     acl_default = Some(CaFormatACLDefault {
 371                         // The value is set to UINT64_MAX as placeholder if one
 372                         // of the permissions is not set
 373                         user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
 374                         group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
 375                         other_permissions: other_permissions.unwrap_or(std::u64::MAX),
 376                         mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
 377                     });
 378                 }
 379             },
 380             _ => bail!("Unexpected ACL type encountered"),
 381         }
 382
 383         Ok(PxarACL {
 384             users: acl_user,
 385             groups: acl_group,
 386             group_obj: acl_group_obj,
 387             default: acl_default,
 388         })
 389     }
 390
 391     fn write_entry(&mut self, entry: CaFormatEntry) -> Result<(), Error> {
 392
 393         self.write_header(CA_FORMAT_ENTRY, std::mem::size_of::<CaFormatEntry>() as u64)?;
 394         self.write_item(entry)?;
 395
 396         Ok(())
 397     }
 398
 399     fn write_xattr(&mut self, xattr: CaFormatXAttr) -> Result<(), Error> {
 400         let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
 401         self.write_header(CA_FORMAT_XATTR, size as u64)?;
 402         self.write(xattr.name.as_slice())?;
 403         self.write(&[0])?;
 404         self.write(xattr.value.as_slice())?;
 405
 406         Ok(())
 407     }
 408
 409     fn write_fcaps(&mut self, fcaps: Option<CaFormatFCaps>) -> Result<(), Error> {
 410         if let Some(fcaps) = fcaps {
 411             let size = fcaps.data.len();
 412             self.write_header(CA_FORMAT_FCAPS, size as u64)?;
 413             self.write(fcaps.data.as_slice())?;
 414         }
 415
 416         Ok(())
 417     }
 418
 419     fn write_acl_user(&mut self, acl_user: CaFormatACLUser) -> Result<(), Error> {
 420         self.write_header(CA_FORMAT_ACL_USER,  std::mem::size_of::<CaFormatACLUser>() as u64)?;
 421         self.write_item(acl_user)?;
 422
 423         Ok(())
 424     }
 425
 426     fn write_acl_group(&mut self, acl_group: CaFormatACLGroup) -> Result<(), Error> {
 427         self.write_header(CA_FORMAT_ACL_GROUP,  std::mem::size_of::<CaFormatACLGroup>() as u64)?;
 428         self.write_item(acl_group)?;
 429
 430         Ok(())
 431     }
 432
 433     fn write_acl_group_obj(&mut self, acl_group_obj: CaFormatACLGroupObj) -> Result<(), Error> {
 434         self.write_header(CA_FORMAT_ACL_GROUP_OBJ,  std::mem::size_of::<CaFormatACLGroupObj>() as u64)?;
 435         self.write_item(acl_group_obj)?;
 436
 437         Ok(())
 438     }
 439
 440     fn write_acl_default(&mut self, acl_default: CaFormatACLDefault) -> Result<(), Error> {
 441         self.write_header(CA_FORMAT_ACL_DEFAULT,  std::mem::size_of::<CaFormatACLDefault>() as u64)?;
 442         self.write_item(acl_default)?;
 443
 444         Ok(())
 445     }
 446
 447     fn write_acl_default_user(&mut self, acl_default_user: CaFormatACLUser) -> Result<(), Error> {
 448         self.write_header(CA_FORMAT_ACL_DEFAULT_USER,  std::mem::size_of::<CaFormatACLUser>() as u64)?;
 449         self.write_item(acl_default_user)?;
 450
 451         Ok(())
 452     }
 453
 454     fn write_acl_default_group(&mut self, acl_default_group: CaFormatACLGroup) -> Result<(), Error> {
 455         self.write_header(CA_FORMAT_ACL_DEFAULT_GROUP,  std::mem::size_of::<CaFormatACLGroup>() as u64)?;
 456         self.write_item(acl_default_group)?;
 457
 458         Ok(())
 459     }
 460
 461     fn write_goodbye_table(&mut self, goodbye_offset: usize, goodbye_items: &mut [CaFormatGoodbyeItem]) -> Result<(), Error> {
 462
 463         goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
 464
 465         let item_count = goodbye_items.len();
 466
 467         let goodbye_table_size = (item_count + 1)*std::mem::size_of::<CaFormatGoodbyeItem>();
 468
 469         self.write_header(CA_FORMAT_GOODBYE, goodbye_table_size as u64)?;
 470
 471         if self.file_copy_buffer.len() < goodbye_table_size {
 472             let need = goodbye_table_size - self.file_copy_buffer.len();
 473             self.file_copy_buffer.reserve(need);
 474             unsafe { self.file_copy_buffer.set_len(self.file_copy_buffer.capacity()); }
 475         }
 476
 477         let buffer = &mut self.file_copy_buffer;
 478
 479         copy_binary_search_tree(item_count, |s, d| {
 480             let item = &goodbye_items[s];
 481             let offset = d*std::mem::size_of::<CaFormatGoodbyeItem>();
 482             let dest = crate::tools::map_struct_mut::<CaFormatGoodbyeItem>(&mut buffer[offset..]).unwrap();
 483             dest.offset = u64::to_le(item.offset);
 484             dest.size = u64::to_le(item.size);
 485             dest.hash = u64::to_le(item.hash);
 486         });
 487
 488         // append CaFormatGoodbyeTail as last item
 489         let offset = item_count*std::mem::size_of::<CaFormatGoodbyeItem>();
 490         let dest = crate::tools::map_struct_mut::<CaFormatGoodbyeItem>(&mut buffer[offset..]).unwrap();
 491         dest.offset = u64::to_le(goodbye_offset as u64);
 492         dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<CaFormatHeader>()) as u64);
 493         dest.hash = u64::to_le(CA_FORMAT_GOODBYE_TAIL_MARKER);
 494
 495         self.flush_copy_buffer(goodbye_table_size)?;
 496
 497         Ok(())
 498     }
 499
 500     fn encode_dir(&mut self, dir: &mut nix::dir::Dir, dir_stat: &FileStat, magic: i64)  -> Result<(), Error> {
 501
 502         //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
 503
 504         let mut name_list = vec![];
 505
 506         let rawfd = dir.as_raw_fd();
 507
 508         let dir_start_pos = self.writer_pos;
 509
 510         let mut dir_entry = self.create_entry(&dir_stat)?;
 511
 512         self.read_chattr(rawfd, &mut dir_entry)?;
 513         self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
 514         let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
 515         let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
 516         let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
 517
 518         self.write_entry(dir_entry)?;
 519         for xattr in xattrs {
 520             self.write_xattr(xattr)?;
 521         }
 522         self.write_fcaps(fcaps)?;
 523
 524         for user in acl_access.users {
 525             self.write_acl_user(user)?;
 526         }
 527         for group in acl_access.groups {
 528             self.write_acl_group(group)?;
 529         }
 530         if let Some(group_obj) = acl_access.group_obj {
 531             self.write_acl_group_obj(group_obj)?;
 532         }
 533
 534         for default_user in acl_default.users {
 535             self.write_acl_default_user(default_user)?;
 536         }
 537         for default_group in acl_default.groups {
 538             self.write_acl_default_group(default_group)?;
 539         }
 540         if let Some(default) = acl_default.default {
 541             self.write_acl_default(default)?;
 542         }
 543
 544         let mut dir_count = 0;
 545
 546         let include_children;
 547         if is_virtual_file_system(magic) {
 548             include_children = false;
 549         } else {
 550             include_children = (self.root_st_dev == dir_stat.st_dev) || self.all_file_systems;
 551         }
 552
 553         if include_children {
 554             for entry in dir.iter() {
 555                 dir_count += 1;
 556                 if dir_count > MAX_DIRECTORY_ENTRIES {
 557                     bail!("too many directory items in {:?} (> {})",
 558                           self.full_path(), MAX_DIRECTORY_ENTRIES);
 559                 }
 560
 561                 let entry = match entry {
 562                     Ok(entry) => entry,
 563                     Err(err) => bail!("readir {:?} failed - {}", self.full_path(), err),
 564                 };
 565                 let filename = entry.file_name().to_owned();
 566
 567                 let name = filename.to_bytes_with_nul();
 568                 let name_len = name.len();
 569                 if name_len == 2 && name[0] == b'.' && name[1] == 0u8 { continue; }
 570                 if name_len == 3 && name[0] == b'.' && name[1] == b'.' && name[2] == 0u8 { continue; }
 571
 572                 name_list.push(filename);
 573             }
 574         } else {
 575             eprintln!("skip mount point: {:?}", self.full_path());
 576         }
 577
 578         name_list.sort_unstable_by(|a, b| a.cmp(&b));
 579
 580         let mut goodbye_items = vec![];
 581
 582         for filename in &name_list {
 583             self.relative_path.push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
 584
 585             if self.verbose { println!("{:?}", self.full_path()); }
 586
 587             let stat = match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
 588                 Ok(stat) => stat,
 589                 Err(nix::Error::Sys(Errno::ENOENT)) => {
 590                     self.report_vanished_file(&self.full_path())?;
 591                     continue;
 592                 }
 593                 Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
 594             };
 595
 596             let start_pos = self.writer_pos;
 597
 598             if is_directory(&stat) {
 599
 600                 let mut dir = match nix::dir::Dir::openat(rawfd, filename.as_ref(), OFlag::O_DIRECTORY|OFlag::O_NOFOLLOW, Mode::empty()) {
 601                     Ok(dir) => dir,
 602                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 603                         self.report_vanished_file(&self.full_path())?;
 604                         continue; // fixme!!
 605                     },
 606                     Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
 607                 };
 608
 609                 let child_magic = if dir_stat.st_dev != stat.st_dev {
 610                     detect_fs_type(dir.as_raw_fd())?
 611                 } else {
 612                     magic
 613                 };
 614
 615                 self.write_filename(&filename)?;
 616                 self.encode_dir(&mut dir, &stat, child_magic)?;
 617
 618             } else if is_reg_file(&stat) {
 619
 620                 let mut hardlink_target = None;
 621
 622                 if stat.st_nlink > 1 {
 623                     let link_info = HardLinkInfo { st_dev: stat.st_dev, st_ino: stat.st_ino };
 624                     hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
 625                         let mut target = v.clone().into_os_string();
 626                         target.push("\0"); // add Nul byte
 627                         (target, (start_pos as u64) - offset)
 628                     });
 629                     if hardlink_target == None {
 630                         self.hardlinks.insert(link_info, (self.relative_path.clone(), start_pos as u64));
 631                     }
 632                 }
 633
 634                 if let Some((target, offset)) = hardlink_target {
 635
 636                     self.write_filename(&filename)?;
 637                     self.encode_hardlink(target.as_bytes(), offset)?;
 638
 639                 } else {
 640
 641                     let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
 642                         Ok(filefd) => filefd,
 643                         Err(nix::Error::Sys(Errno::ENOENT)) => {
 644                             self.report_vanished_file(&self.full_path())?;
 645                             continue;
 646                         },
 647                         Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
 648                     };
 649
 650                     let child_magic = if dir_stat.st_dev != stat.st_dev {
 651                         detect_fs_type(filefd)?
 652                     } else {
 653                         magic
 654                     };
 655
 656                     self.write_filename(&filename)?;
 657                     let res = self.encode_file(filefd, &stat, child_magic);
 658                     let _ = nix::unistd::close(filefd); // ignore close errors
 659                     res?;
 660                 }
 661
 662             } else if is_symlink(&stat) {
 663                 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
 664
 665                 let res = filename.with_nix_path(|cstr| {
 666                     unsafe { libc::readlinkat(rawfd, cstr.as_ptr(), buffer.as_mut_ptr() as *mut libc::c_char, buffer.len()-1) }
 667                 })?;
 668
 669                 match Errno::result(res) {
 670                     Ok(len) => {
 671                         buffer[len as usize] = 0u8; // add Nul byte
 672                         self.write_filename(&filename)?;
 673                         self.encode_symlink(&buffer[..((len+1) as usize)], &stat)?
 674                     }
 675                     Err(nix::Error::Sys(Errno::ENOENT)) => {
 676                         self.report_vanished_file(&self.full_path())?;
 677                         continue;
 678                     }
 679                     Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
 680                 }
 681             } else if is_block_dev(&stat) || is_char_dev(&stat) {
 682                 self.write_filename(&filename)?;
 683                 self.encode_device(&stat)?;
 684             } else if is_fifo(&stat) || is_socket(&stat) {
 685                 self.write_filename(&filename)?;
 686                 self.encode_special(&stat)?;
 687             } else {
 688                 bail!("unsupported file type (mode {:o} {:?})", stat.st_mode, self.full_path());
 689             }
 690
 691             let end_pos = self.writer_pos;
 692
 693             goodbye_items.push(CaFormatGoodbyeItem {
 694                 offset: start_pos as u64,
 695                 size: (end_pos - start_pos) as u64,
 696                 hash: compute_goodbye_hash(filename.to_bytes()),
 697             });
 698
 699             self.relative_path.pop();
 700         }
 701
 702         //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
 703
 704         // fixup goodby item offsets
 705         let goodbye_start = self.writer_pos as u64;
 706         for item in &mut goodbye_items {
 707             item.offset = goodbye_start - item.offset;
 708         }
 709
 710         let goodbye_offset = self.writer_pos - dir_start_pos;
 711
 712         self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
 713
 714         //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
 715         Ok(())
 716     }
 717
 718     fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64)  -> Result<(), Error> {
 719
 720         //println!("encode_file: {:?}", self.full_path());
 721
 722         let mut entry = self.create_entry(&stat)?;
 723
 724         self.read_chattr(filefd, &mut entry)?;
 725         self.read_fat_attr(filefd, magic, &mut entry)?;
 726         let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
 727         let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
 728
 729         self.write_entry(entry)?;
 730         for xattr in xattrs {
 731             self.write_xattr(xattr)?;
 732         }
 733         self.write_fcaps(fcaps)?;
 734         for user in acl_access.users {
 735             self.write_acl_user(user)?;
 736         }
 737         for group in acl_access.groups {
 738             self.write_acl_group(group)?;
 739         }
 740         if let Some(group_obj) = acl_access.group_obj {
 741             self.write_acl_group_obj(group_obj)?;
 742         }
 743
 744         let include_payload;
 745         if is_virtual_file_system(magic) {
 746             include_payload = false;
 747         } else {
 748             include_payload = (stat.st_dev == self.root_st_dev) || self.all_file_systems;
 749         }
 750
 751         if !include_payload {
 752             eprintln!("skip content: {:?}", self.full_path());
 753             self.write_header(CA_FORMAT_PAYLOAD, 0)?;
 754             return Ok(());
 755         }
 756
 757         let size = stat.st_size as u64;
 758
 759         self.write_header(CA_FORMAT_PAYLOAD, size)?;
 760
 761         let mut pos: u64 = 0;
 762         loop {
 763             let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
 764                 Ok(n) => n,
 765                 Err(nix::Error::Sys(Errno::EINTR)) => continue /* try again */,
 766                 Err(err) =>  bail!("read {:?} failed - {}", self.full_path(), err),
 767             };
 768             if n == 0 { // EOF
 769                 if pos != size {
 770                     // Note:: casync format cannot handle that
 771                     bail!("detected shrinked file {:?} ({} < {})", self.full_path(), pos, size);
 772                 }
 773                 break;
 774             }
 775
 776             let mut next = pos + (n as u64);
 777
 778             if next > size { next = size; }
 779
 780             let count = (next - pos) as usize;
 781
 782             self.flush_copy_buffer(count)?;
 783
 784             pos = next;
 785
 786             if pos >= size { break; }
 787         }
 788
 789         Ok(())
 790     }
 791
 792     fn encode_device(&mut self, stat: &FileStat)  -> Result<(), Error> {
 793
 794         let entry = self.create_entry(&stat)?;
 795
 796         self.write_entry(entry)?;
 797
 798         let major = unsafe { libc::major(stat.st_rdev) } as u64;
 799         let minor = unsafe { libc::minor(stat.st_rdev) } as u64;
 800
 801         //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
 802
 803         self.write_header(CA_FORMAT_DEVICE, std::mem::size_of::<CaFormatDevice>() as u64)?;
 804         self.write_item(CaFormatDevice { major, minor })?;
 805
 806         Ok(())
 807     }
 808
 809     // FIFO or Socket
 810     fn encode_special(&mut self, stat: &FileStat)  -> Result<(), Error> {
 811
 812         let entry = self.create_entry(&stat)?;
 813
 814         self.write_entry(entry)?;
 815
 816         Ok(())
 817     }
 818
 819     fn encode_symlink(&mut self, target: &[u8], stat: &FileStat)  -> Result<(), Error> {
 820
 821         //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
 822
 823         let entry = self.create_entry(&stat)?;
 824         self.write_entry(entry)?;
 825
 826         self.write_header(CA_FORMAT_SYMLINK, target.len() as u64)?;
 827         self.write(target)?;
 828
 829         Ok(())
 830     }
 831
 832     fn encode_hardlink(&mut self, target: &[u8], offset: u64)  -> Result<(), Error> {
 833
 834         //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
 835
 836         // Note: HARDLINK replaces an ENTRY.
 837         self.write_header(PXAR_FORMAT_HARDLINK, (target.len() as u64) + 8)?;
 838         self.write_item(offset)?;
 839         self.write(target)?;
 840
 841         Ok(())
 842     }
 843
 844     // the report_XXX method may raise and error - depending on encoder configuration
 845
 846     fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
 847
 848         eprintln!("WARNING: detected vanished file {:?}", path);
 849
 850         Ok(())
 851     }
 852 }
 853
 854 fn errno_is_unsupported(errno: Errno) -> bool {
 855
 856     match errno {
 857         Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => {
 858             true
 859         }
 860         _ => false,
 861     }
 862 }
 863
 864 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
 865     let mut fs_stat: libc::statfs = unsafe { std::mem::uninitialized() };
 866     let res = unsafe { libc::fstatfs(fd, &mut fs_stat) };
 867     Errno::result(res)?;
 868
 869     Ok(fs_stat.f_type)
 870 }
 871
 872 use nix::{convert_ioctl_res, request_code_read, ioc};
 873
 874 // /usr/include/linux/fs.h: #define FS_IOC_GETFLAGS _IOR('f', 1, long)
 875 /// read Linux file system attributes (see man chattr)
 876 nix::ioctl_read!(read_attr_fd, b'f', 1, usize);
 877
 878 // /usr/include/linux/msdos_fs.h: #define FAT_IOCTL_GET_ATTRIBUTES _IOR('r', 0x10, __u32)
 879 // read FAT file system attributes
 880 nix::ioctl_read!(read_fat_attr_fd, b'r', 0x10, u32);
 881
 882
 883 // from /usr/include/linux/magic.h
 884 // and from casync util.h
 885 pub const BINFMTFS_MAGIC: i64 =        0x42494e4d;
 886 pub const CGROUP2_SUPER_MAGIC: i64 =   0x63677270;
 887 pub const CGROUP_SUPER_MAGIC: i64 =    0x0027e0eb;
 888 pub const CONFIGFS_MAGIC: i64 =        0x62656570;
 889 pub const DEBUGFS_MAGIC: i64 =         0x64626720;
 890 pub const DEVPTS_SUPER_MAGIC: i64 =    0x00001cd1;
 891 pub const EFIVARFS_MAGIC: i64 =        0xde5e81e4;
 892 pub const FUSE_CTL_SUPER_MAGIC: i64 =  0x65735543;
 893 pub const HUGETLBFS_MAGIC: i64 =       0x958458f6;
 894 pub const MQUEUE_MAGIC: i64 =          0x19800202;
 895 pub const NFSD_MAGIC: i64 =            0x6e667364;
 896 pub const PROC_SUPER_MAGIC: i64 =      0x00009fa0;
 897 pub const PSTOREFS_MAGIC: i64 =        0x6165676C;
 898 pub const RPCAUTH_GSSMAGIC: i64 =      0x67596969;
 899 pub const SECURITYFS_MAGIC: i64 =      0x73636673;
 900 pub const SELINUX_MAGIC: i64 =         0xf97cff8c;
 901 pub const SMACK_MAGIC: i64 =           0x43415d53;
 902 pub const RAMFS_MAGIC: i64 =           0x858458f6;
 903 pub const TMPFS_MAGIC: i64 =           0x01021994;
 904 pub const SYSFS_MAGIC: i64 =           0x62656572;
 905 pub const MSDOS_SUPER_MAGIC: i64 =     0x00004d44;
 906 pub const FUSE_SUPER_MAGIC: i64 =      0x65735546;
 907
 908
 909 #[inline(always)]
 910 pub fn is_temporary_file_system(magic: i64) -> bool {
 911     magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
 912 }
 913
 914 pub fn is_virtual_file_system(magic: i64) -> bool {
 915
 916     match magic {
 917         BINFMTFS_MAGIC |
 918         CGROUP2_SUPER_MAGIC |
 919         CGROUP_SUPER_MAGIC |
 920         CONFIGFS_MAGIC |
 921         DEBUGFS_MAGIC |
 922         DEVPTS_SUPER_MAGIC |
 923         EFIVARFS_MAGIC |
 924         FUSE_CTL_SUPER_MAGIC |
 925         HUGETLBFS_MAGIC |
 926         MQUEUE_MAGIC |
 927         NFSD_MAGIC |
 928         PROC_SUPER_MAGIC |
 929         PSTOREFS_MAGIC |
 930         RPCAUTH_GSSMAGIC |
 931         SECURITYFS_MAGIC |
 932         SELINUX_MAGIC |
 933         SMACK_MAGIC |
 934         SYSFS_MAGIC => true,
 935         _ => false
 936     }
 937 }