1 //! *pxar* format encoder.
3 //! This module contain the code to generate *pxar* archive files.
6 use endian_trait
::Endian
;
7 use std
::collections
::{HashSet, HashMap}
;
10 use super::format_definition
::*;
11 use super::binary_search_tree
::*;
13 use super::exclude_pattern
::*;
15 use crate::tools
::acl
;
16 use crate::tools
::xattr
;
19 use std
::os
::unix
::io
::AsRawFd
;
20 use std
::os
::unix
::ffi
::OsStrExt
;
21 use std
::os
::unix
::io
::RawFd
;
22 use std
::path
::{Path, PathBuf}
;
27 use nix
::fcntl
::OFlag
;
28 use nix
::sys
::stat
::Mode
;
29 use nix
::errno
::Errno
;
30 use nix
::sys
::stat
::FileStat
;
32 use proxmox
::tools
::vec
;
34 /// The format requires to build sorted directory lookup tables in
35 /// memory, so we restrict the number of allowed entries to limit
36 /// maximum memory usage.
37 pub const MAX_DIRECTORY_ENTRIES
: usize = 256*1024;
39 #[derive(Eq, PartialEq, Hash)]
45 pub struct Encoder
<'a
, W
: Write
> {
47 relative_path
: PathBuf
,
51 file_copy_buffer
: Vec
<u8>,
52 device_set
: Option
<HashSet
<u64>>,
54 // Flags set by the user
56 // Flags signaling features supported by the filesystem
57 fs_feature_flags
: u64,
58 hardlinks
: HashMap
<HardLinkInfo
, (PathBuf
, u64)>,
61 impl <'a
, W
: Write
> Encoder
<'a
, W
> {
63 // used for error reporting
64 fn full_path(&self) -> PathBuf
{
65 self.base_path
.join(&self.relative_path
)
68 /// Create archive, write result data to ``writer``.
70 /// The ``device_set`` can be use used to limit included mount points.
72 /// - ``None``: include all mount points
73 /// - ``Some(set)``: only include devices listed in this set (the
74 /// root path device is automathically added to this list, so
75 /// you can pass an empty set if you want to archive a single
79 dir
: &mut nix
::dir
::Dir
,
81 device_set
: Option
<HashSet
<u64>>,
83 skip_lost_and_found
: bool
, // fixme: should be a feature flag ??
85 ) -> Result
<(), Error
> {
87 const FILE_COPY_BUFFER_SIZE
: usize = 1024*1024;
89 let mut file_copy_buffer
= Vec
::with_capacity(FILE_COPY_BUFFER_SIZE
);
90 unsafe { file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE); }
93 // todo: use scandirat??
95 let dir_fd
= dir
.as_raw_fd();
96 let stat
= nix
::sys
::stat
::fstat(dir_fd
)
97 .map_err(|err
| format_err
!("fstat {:?} failed - {}", path
, err
))?
;
99 if !is_directory(&stat
) {
100 bail
!("got unexpected file type {:?} (not a directory)", path
);
103 let mut device_set
= device_set
.clone();
104 if let Some(ref mut set
) = device_set
{
105 set
.insert(stat
.st_dev
);
108 let magic
= detect_fs_type(dir_fd
)?
;
110 if is_virtual_file_system(magic
) {
111 bail
!("backup virtual file systems is disabled!");
114 let fs_feature_flags
= flags
::feature_flags_from_magic(magic
);
118 relative_path
: PathBuf
::new(),
127 hardlinks
: HashMap
::new(),
130 if verbose { println!("{:?}
", me.full_path()); }
132 let mut excludes = Vec::new();
133 if skip_lost_and_found {
134 excludes.push(PxarExcludePattern::from_line(b"**/lost
+found
").unwrap().unwrap());
136 me.encode_dir(dir, &stat, magic, excludes)?;
141 fn write(&mut self, buf: &[u8]) -> Result<(), Error> {
142 self.writer.write_all(buf)?;
143 self.writer_pos += buf.len();
147 fn write_item<T: Endian>(&mut self, item: T) -> Result<(), Error> {
149 let data = item.to_le();
151 let buffer = unsafe { std::slice::from_raw_parts(
152 &data as *const T as *const u8,
153 std::mem::size_of::<T>()
161 fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
162 self.writer.write_all(&self.file_copy_buffer[..size])?;
163 self.writer_pos += size;
167 fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
169 let size = size + (std::mem::size_of::<PxarHeader>() as u64);
170 self.write_item(PxarHeader { size, htype })?;
175 fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
177 let buffer = name.to_bytes_with_nul();
178 self.write_header(PXAR_FILENAME, buffer.len() as u64)?;
184 fn create_entry(&self, stat: &FileStat) -> Result<PxarEntry, Error> {
186 let mode = if is_symlink(&stat) {
187 (libc::S_IFLNK | 0o777) as u64
189 (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
192 let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
194 bail!("got strange
mtime ({}
) from fstat
for {:?}
.", mtime, self.full_path());
198 let entry = PxarEntry {
209 fn read_chattr(&self, fd: RawFd, entry: &mut PxarEntry) -> Result<(), Error> {
211 let mut attr: usize = 0;
213 let res = unsafe { fs::read_attr_fd(fd, &mut attr)};
214 if let Err(err) = res {
215 if let nix::Error::Sys(errno) = err {
216 if errno_is_unsupported(errno) { return Ok(()) };
218 bail!("read_attr_fd failed
for {:?}
- {}
", self.full_path(), err);
221 let flags = flags::feature_flags_from_chattr(attr as u32);
222 entry.flags = entry.flags | flags;
227 fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut PxarEntry) -> Result<(), Error> {
230 if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC {
234 let mut attr: u32 = 0;
236 let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr)};
237 if let Err(err) = res {
238 if let nix::Error::Sys(errno) = err {
239 if errno_is_unsupported(errno) { return Ok(()) };
241 bail!("read_fat_attr_fd failed
for {:?}
- {}
", self.full_path(), err);
244 let flags = flags::feature_flags_from_fat_attr(attr);
245 entry.flags = entry.flags | flags;
250 /// True if all of the given feature flags are set in the Encoder, false otherwise
251 fn has_features(&self, feature_flags: u64) -> bool {
252 (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
255 /// True if at least one of the given feature flags is set in the Encoder, false otherwise
256 fn has_some_features(&self, feature_flags: u64) -> bool {
257 (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
260 fn read_xattrs(&self, fd: RawFd, stat: &FileStat) -> Result<(Vec<PxarXAttr>, Option<PxarFCaps>), Error> {
261 let mut xattrs = Vec::new();
262 let mut fcaps = None;
264 let flags = flags::WITH_XATTRS | flags::WITH_FCAPS;
265 if !self.has_some_features(flags) {
266 return Ok((xattrs, fcaps));
268 // Should never be called on symlinks, just in case check anyway
269 if is_symlink(&stat) {
270 return Ok((xattrs, fcaps));
273 let xattr_names = match xattr::flistxattr(fd) {
275 // Do not bail if the underlying endpoint does not supports xattrs
276 Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
277 // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
278 Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
279 Err(err) => bail!("read_xattrs failed
for {:?}
- {}
", self.full_path(), err),
282 for name in xattr_names.split(|c| *c == b'\0') {
283 // Only extract the relevant extended attributes
284 if !xattr::is_valid_xattr_name(&name) {
288 let value = match xattr::fgetxattr(fd, name) {
290 // Vanished between flistattr and getxattr, this is ok, silently ignore
291 Err(Errno::ENODATA) => continue,
292 Err(err) => bail!("read_xattrs failed
for {:?}
- {}
", self.full_path(), err),
295 if xattr::is_security_capability(&name) {
296 if self.has_features(flags::WITH_FCAPS) {
297 // fcaps are stored in own format within the archive
298 fcaps = Some(PxarFCaps {
302 } else if self.has_features(flags::WITH_XATTRS) {
303 xattrs.push(PxarXAttr {
314 fn read_acl(&self, fd: RawFd, stat: &FileStat, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
322 if !self.has_features(flags::WITH_ACL) {
325 if is_symlink(&stat) {
328 if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
329 bail!("ACL_TYPE_DEFAULT only defined
for directories
.");
332 // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
333 // to create a path for acl_get_file(). acl_get_fd() only allows to get
334 // ACL_TYPE_ACCESS attributes.
335 let proc_path = Path::new("/proc
/self/fd
/").join(fd.to_string());
336 let acl = match acl::ACL::get_file(&proc_path, acl_type) {
338 // Don't bail if underlying endpoint does not support acls
339 Err(Errno::EOPNOTSUPP) => return Ok(ret),
340 // Don't bail if the endpoint cannot carry acls
341 Err(Errno::EBADF) => return Ok(ret),
342 // Don't bail if there is no data
343 Err(Errno::ENODATA) => return Ok(ret),
344 Err(err) => bail!("error
while reading ACL
- {}
", err),
347 self.process_acl(acl, acl_type)
350 fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
351 let mut acl_user = Vec::new();
352 let mut acl_group = Vec::new();
353 let mut acl_group_obj = None;
354 let mut acl_default = None;
355 let mut user_obj_permissions = None;
356 let mut group_obj_permissions = None;
357 let mut other_permissions = None;
358 let mut mask_permissions = None;
360 for entry in &mut acl.entries() {
361 let tag = entry.get_tag_type()?;
362 let permissions = entry.get_permissions()?;
364 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
365 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
366 acl::ACL_OTHER => other_permissions = Some(permissions),
367 acl::ACL_MASK => mask_permissions = Some(permissions),
369 acl_user.push(PxarACLUser {
370 uid: entry.get_qualifier()?,
371 permissions: permissions,
375 acl_group.push(PxarACLGroup {
376 gid: entry.get_qualifier()?,
377 permissions: permissions,
380 _ => bail!("Unexpected ACL tag encountered
!"),
388 acl::ACL_TYPE_ACCESS => {
389 // The mask permissions are mapped to the stat group permissions
390 // in case that the ACL group permissions were set.
391 // Only in that case we need to store the group permissions,
392 // in the other cases they are identical to the stat group permissions.
393 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
394 acl_group_obj = Some(PxarACLGroupObj {
399 acl::ACL_TYPE_DEFAULT => {
400 if user_obj_permissions != None ||
401 group_obj_permissions != None ||
402 other_permissions != None ||
403 mask_permissions != None
405 acl_default = Some(PxarACLDefault {
406 // The value is set to UINT64_MAX as placeholder if one
407 // of the permissions is not set
408 user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
409 group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
410 other_permissions: other_permissions.unwrap_or(std::u64::MAX),
411 mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
415 _ => bail!("Unexpected ACL
type encountered
"),
421 group_obj: acl_group_obj,
422 default: acl_default,
426 /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
427 fn read_quota_project_id(&self, fd: RawFd, magic: i64, stat: &FileStat) -> Result<Option<PxarQuotaProjID>, Error> {
428 if !(is_directory(&stat) || is_reg_file(&stat)) {
431 if !self.has_features(flags::WITH_QUOTA_PROJID) {
436 fs::magic::EXT4_SUPER_MAGIC |
437 fs::magic::XFS_SUPER_MAGIC |
438 fs::magic::FUSE_SUPER_MAGIC |
439 fs::magic::ZFS_SUPER_MAGIC => {
440 let mut fsxattr = fs::FSXAttr::default();
442 fs::fs_ioc_fsgetxattr(fd, &mut fsxattr)
445 // On some FUSE filesystems it can happen that ioctl is not supported.
446 // For these cases projid is set to 0 while the error is ignored.
447 if let Err(err) = res {
448 let errno = err.as_errno().ok_or_else(|| {
449 format_err!("error
while reading quota project id
for {:#?}
", self.full_path())
451 if errno_is_unsupported(errno) {
454 bail!("error
while reading quota project id
for {:#?}
- {}
", self.full_path(), errno);
458 let projid = fsxattr.fsx_projid as u64;
462 return Ok(Some(PxarQuotaProjID { projid }));
465 _ => return Ok(None),
469 fn write_entry(&mut self, entry: PxarEntry) -> Result<(), Error> {
471 self.write_header(PXAR_ENTRY, std::mem::size_of::<PxarEntry>() as u64)?;
472 self.write_item(entry)?;
477 fn write_xattr(&mut self, xattr: PxarXAttr) -> Result<(), Error> {
478 let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
479 self.write_header(PXAR_XATTR, size as u64)?;
480 self.write(xattr.name.as_slice())?;
482 self.write(xattr.value.as_slice())?;
487 fn write_fcaps(&mut self, fcaps: Option<PxarFCaps>) -> Result<(), Error> {
488 if let Some(fcaps) = fcaps {
489 let size = fcaps.data.len();
490 self.write_header(PXAR_FCAPS, size as u64)?;
491 self.write(fcaps.data.as_slice())?;
497 fn write_acl_user(&mut self, acl_user: PxarACLUser) -> Result<(), Error> {
498 self.write_header(PXAR_ACL_USER, std::mem::size_of::<PxarACLUser>() as u64)?;
499 self.write_item(acl_user)?;
504 fn write_acl_group(&mut self, acl_group: PxarACLGroup) -> Result<(), Error> {
505 self.write_header(PXAR_ACL_GROUP, std::mem::size_of::<PxarACLGroup>() as u64)?;
506 self.write_item(acl_group)?;
511 fn write_acl_group_obj(&mut self, acl_group_obj: PxarACLGroupObj) -> Result<(), Error> {
512 self.write_header(PXAR_ACL_GROUP_OBJ, std::mem::size_of::<PxarACLGroupObj>() as u64)?;
513 self.write_item(acl_group_obj)?;
518 fn write_acl_default(&mut self, acl_default: PxarACLDefault) -> Result<(), Error> {
519 self.write_header(PXAR_ACL_DEFAULT, std::mem::size_of::<PxarACLDefault>() as u64)?;
520 self.write_item(acl_default)?;
525 fn write_acl_default_user(&mut self, acl_default_user: PxarACLUser) -> Result<(), Error> {
526 self.write_header(PXAR_ACL_DEFAULT_USER, std::mem::size_of::<PxarACLUser>() as u64)?;
527 self.write_item(acl_default_user)?;
532 fn write_acl_default_group(&mut self, acl_default_group: PxarACLGroup) -> Result<(), Error> {
533 self.write_header(PXAR_ACL_DEFAULT_GROUP, std::mem::size_of::<PxarACLGroup>() as u64)?;
534 self.write_item(acl_default_group)?;
539 fn write_quota_project_id(&mut self, projid: PxarQuotaProjID) -> Result<(), Error> {
540 self.write_header(PXAR_QUOTA_PROJID, std::mem::size_of::<PxarQuotaProjID>() as u64)?;
541 self.write_item(projid)?;
546 fn write_goodbye_table(&mut self, goodbye_offset: usize, goodbye_items: &mut [PxarGoodbyeItem]) -> Result<(), Error> {
548 goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
550 let item_count = goodbye_items.len();
552 let goodbye_table_size = (item_count + 1)*std::mem::size_of::<PxarGoodbyeItem>();
554 self.write_header(PXAR_GOODBYE, goodbye_table_size as u64)?;
556 if self.file_copy_buffer.len() < goodbye_table_size {
557 let need = goodbye_table_size - self.file_copy_buffer.len();
558 self.file_copy_buffer.reserve(need);
559 unsafe { self.file_copy_buffer.set_len(self.file_copy_buffer.capacity()); }
562 let buffer = &mut self.file_copy_buffer;
564 copy_binary_search_tree(item_count, |s, d| {
565 let item = &goodbye_items[s];
566 let offset = d*std::mem::size_of::<PxarGoodbyeItem>();
567 let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
568 dest.offset = u64::to_le(item.offset);
569 dest.size = u64::to_le(item.size);
570 dest.hash = u64::to_le(item.hash);
573 // append PxarGoodbyeTail as last item
574 let offset = item_count*std::mem::size_of::<PxarGoodbyeItem>();
575 let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
576 dest.offset = u64::to_le(goodbye_offset as u64);
577 dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<PxarHeader>()) as u64);
578 dest.hash = u64::to_le(PXAR_GOODBYE_TAIL_MARKER);
580 self.flush_copy_buffer(goodbye_table_size)?;
585 fn encode_dir(&mut self, dir: &mut nix::dir::Dir, dir_stat: &FileStat, magic: i64, match_pattern: Vec<PxarExcludePattern>) -> Result<(), Error> {
587 //println!("encode_dir
: {:?} start {}
", self.full_path(), self.writer_pos);
589 let mut name_list = vec![];
591 let rawfd = dir.as_raw_fd();
593 let dir_start_pos = self.writer_pos;
595 let mut dir_entry = self.create_entry(&dir_stat)?;
597 self.read_chattr(rawfd, &mut dir_entry)?;
598 self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
600 // for each node in the directory tree, the filesystem features are
601 // checked based on the fs magic number.
602 self.fs_feature_flags = flags::feature_flags_from_magic(magic);
604 let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
605 let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
606 let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
607 let projid = self.read_quota_project_id(rawfd, magic, &dir_stat)?;
609 self.write_entry(dir_entry)?;
610 for xattr in xattrs {
611 self.write_xattr(xattr)?;
613 self.write_fcaps(fcaps)?;
615 for user in acl_access.users {
616 self.write_acl_user(user)?;
618 for group in acl_access.groups {
619 self.write_acl_group(group)?;
621 if let Some(group_obj) = acl_access.group_obj {
622 self.write_acl_group_obj(group_obj)?;
625 for default_user in acl_default.users {
626 self.write_acl_default_user(default_user)?;
628 for default_group in acl_default.groups {
629 self.write_acl_default_group(default_group)?;
631 if let Some(default) = acl_default.default {
632 self.write_acl_default(default)?;
634 if let Some(projid) = projid {
635 self.write_quota_project_id(projid)?;
638 let include_children;
639 if is_virtual_file_system(magic) {
640 include_children = false;
642 if let Some(set) = &self.device_set {
643 include_children = set.contains(&dir_stat.st_dev);
645 include_children = true;
649 // Expand the exclude match pattern inherited from the parent by local entries, if present
650 let mut local_match_pattern = match_pattern.clone();
651 let pxar_exclude = match PxarExcludePattern::from_file(rawfd, ".pxarexclude
") {
652 Ok(Some((mut excludes, buffer, stat))) => {
653 local_match_pattern.append(&mut excludes);
657 Err(err) => bail!("error
while reading exclude file
- {}
", err),
660 if include_children {
661 for entry in dir.iter() {
662 let entry = entry.map_err(|err| {
663 format_err!("readir {:?} failed
- {}
", self.full_path(), err)
665 let filename = entry.file_name().to_owned();
667 let name = filename.to_bytes_with_nul();
668 if name == b".\0" || name == b"..\0" {
672 let stat = match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
674 Err(nix::Error::Sys(Errno::ENOENT)) => {
675 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
676 self.report_vanished_file(&self.full_path().join(filename_osstr))?;
679 Err(err) => bail!("fstat {:?} failed
- {}
", self.full_path(), err),
682 match match_exclude_pattern(&filename, &stat, &local_match_pattern) {
683 (MatchType::Exclude, _) => {
684 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
685 eprintln!("matched by
.pxarexclude entry
- skipping
: {:?}
", self.full_path().join(filename_osstr));
687 (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
690 if name_list.len() > MAX_DIRECTORY_ENTRIES {
691 bail!("too many directory items
in {:?}
(> {}
)", self.full_path(), MAX_DIRECTORY_ENTRIES);
695 eprintln!("skip mount point
: {:?}
", self.full_path());
698 name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
700 let mut goodbye_items = vec![];
702 for (filename, stat, exclude_list) in name_list {
703 let start_pos = self.writer_pos;
705 if filename.as_bytes() == b".pxarexclude
" {
706 if let Some((ref content, ref stat)) = pxar_exclude {
707 let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
708 Ok(filefd) => filefd,
709 Err(nix::Error::Sys(Errno::ENOENT)) => {
710 self.report_vanished_file(&self.full_path())?;
714 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
715 bail!("open file {:?} failed
- {}
", self.full_path().join(filename_osstr), err);
719 let child_magic = if dir_stat.st_dev != stat.st_dev {
720 detect_fs_type(filefd)?
725 self.write_filename(&filename)?;
726 self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
731 self.relative_path.push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
733 if self.verbose { println!("{:?}", self.full_path()); }
735 if is_directory(&stat
) {
737 let mut dir
= match nix
::dir
::Dir
::openat(rawfd
, filename
.as_ref(), OFlag
::O_DIRECTORY
|OFlag
::O_NOFOLLOW
, Mode
::empty()) {
739 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
740 self.report_vanished_file(&self.full_path())?
;
743 Err(err
) => bail
!("open dir {:?} failed - {}", self.full_path(), err
),
746 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
747 detect_fs_type(dir
.as_raw_fd())?
752 self.write_filename(&filename
)?
;
753 self.encode_dir(&mut dir
, &stat
, child_magic
, exclude_list
)?
;
755 } else if is_reg_file(&stat
) {
757 let mut hardlink_target
= None
;
759 if stat
.st_nlink
> 1 {
760 let link_info
= HardLinkInfo { st_dev: stat.st_dev, st_ino: stat.st_ino }
;
761 hardlink_target
= self.hardlinks
.get(&link_info
).map(|(v
, offset
)| {
762 let mut target
= v
.clone().into_os_string();
763 target
.push("\0"); // add Nul byte
764 (target
, (start_pos
as u64) - offset
)
766 if hardlink_target
== None
{
767 self.hardlinks
.insert(link_info
, (self.relative_path
.clone(), start_pos
as u64));
771 if let Some((target
, offset
)) = hardlink_target
{
773 self.write_filename(&filename
)?
;
774 self.encode_hardlink(target
.as_bytes(), offset
)?
;
778 let filefd
= match nix
::fcntl
::openat(rawfd
, filename
.as_ref(), OFlag
::O_NOFOLLOW
, Mode
::empty()) {
779 Ok(filefd
) => filefd
,
780 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
781 self.report_vanished_file(&self.full_path())?
;
784 Err(err
) => bail
!("open file {:?} failed - {}", self.full_path(), err
),
787 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
788 detect_fs_type(filefd
)?
793 self.write_filename(&filename
)?
;
794 let res
= self.encode_file(filefd
, &stat
, child_magic
);
795 let _
= nix
::unistd
::close(filefd
); // ignore close errors
799 } else if is_symlink(&stat
) {
800 let mut buffer
= vec
::undefined(libc
::PATH_MAX
as usize);
802 let res
= filename
.with_nix_path(|cstr
| {
803 unsafe { libc::readlinkat(rawfd, cstr.as_ptr(), buffer.as_mut_ptr() as *mut libc::c_char, buffer.len()-1) }
806 match Errno
::result(res
) {
808 buffer
[len
as usize] = 0u8; // add Nul byte
809 self.write_filename(&filename
)?
;
810 self.encode_symlink(&buffer
[..((len
+1) as usize)], &stat
)?
812 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
813 self.report_vanished_file(&self.full_path())?
;
816 Err(err
) => bail
!("readlink {:?} failed - {}", self.full_path(), err
),
818 } else if is_block_dev(&stat
) || is_char_dev(&stat
) {
819 if self.has_features(flags
::WITH_DEVICE_NODES
) {
820 self.write_filename(&filename
)?
;
821 self.encode_device(&stat
)?
;
823 eprintln
!("skip device node: {:?}", self.full_path());
825 } else if is_fifo(&stat
) {
826 if self.has_features(flags
::WITH_FIFOS
) {
827 self.write_filename(&filename
)?
;
828 self.encode_special(&stat
)?
;
830 eprintln
!("skip fifo: {:?}", self.full_path());
832 } else if is_socket(&stat
) {
833 if self.has_features(flags
::WITH_SOCKETS
) {
834 self.write_filename(&filename
)?
;
835 self.encode_special(&stat
)?
;
837 eprintln
!("skip socket: {:?}", self.full_path());
840 bail
!("unsupported file type (mode {:o} {:?})", stat
.st_mode
, self.full_path());
843 let end_pos
= self.writer_pos
;
845 goodbye_items
.push(PxarGoodbyeItem
{
846 offset
: start_pos
as u64,
847 size
: (end_pos
- start_pos
) as u64,
848 hash
: compute_goodbye_hash(filename
.to_bytes()),
851 self.relative_path
.pop();
854 //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
856 // fixup goodby item offsets
857 let goodbye_start
= self.writer_pos
as u64;
858 for item
in &mut goodbye_items
{
859 item
.offset
= goodbye_start
- item
.offset
;
862 let goodbye_offset
= self.writer_pos
- dir_start_pos
;
864 self.write_goodbye_table(goodbye_offset
, &mut goodbye_items
)?
;
866 //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
870 fn encode_file(&mut self, filefd
: RawFd
, stat
: &FileStat
, magic
: i64) -> Result
<(), Error
> {
872 //println!("encode_file: {:?}", self.full_path());
874 let mut entry
= self.create_entry(&stat
)?
;
876 self.read_chattr(filefd
, &mut entry
)?
;
877 self.read_fat_attr(filefd
, magic
, &mut entry
)?
;
878 let (xattrs
, fcaps
) = self.read_xattrs(filefd
, &stat
)?
;
879 let acl_access
= self.read_acl(filefd
, &stat
, acl
::ACL_TYPE_ACCESS
)?
;
880 let projid
= self.read_quota_project_id(filefd
, magic
, &stat
)?
;
882 self.write_entry(entry
)?
;
883 for xattr
in xattrs
{
884 self.write_xattr(xattr
)?
;
886 self.write_fcaps(fcaps
)?
;
887 for user
in acl_access
.users
{
888 self.write_acl_user(user
)?
;
890 for group
in acl_access
.groups
{
891 self.write_acl_group(group
)?
;
893 if let Some(group_obj
) = acl_access
.group_obj
{
894 self.write_acl_group_obj(group_obj
)?
;
896 if let Some(projid
) = projid
{
897 self.write_quota_project_id(projid
)?
;
901 if is_virtual_file_system(magic
) {
902 include_payload
= false;
904 if let Some(ref set
) = &self.device_set
{
905 include_payload
= set
.contains(&stat
.st_dev
);
907 include_payload
= true;
911 if !include_payload
{
912 eprintln
!("skip content: {:?}", self.full_path());
913 self.write_header(PXAR_PAYLOAD
, 0)?
;
917 let size
= stat
.st_size
as u64;
919 self.write_header(PXAR_PAYLOAD
, size
)?
;
921 let mut pos
: u64 = 0;
923 let n
= match nix
::unistd
::read(filefd
, &mut self.file_copy_buffer
) {
925 Err(nix
::Error
::Sys(Errno
::EINTR
)) => continue /* try again */,
926 Err(err
) => bail
!("read {:?} failed - {}", self.full_path(), err
),
930 // Note:: casync format cannot handle that
931 bail
!("detected shrinked file {:?} ({} < {})", self.full_path(), pos
, size
);
936 let mut next
= pos
+ (n
as u64);
938 if next
> size { next = size; }
940 let count
= (next
- pos
) as usize;
942 self.flush_copy_buffer(count
)?
;
946 if pos
>= size { break; }
952 fn encode_device(&mut self, stat
: &FileStat
) -> Result
<(), Error
> {
954 let entry
= self.create_entry(&stat
)?
;
956 self.write_entry(entry
)?
;
958 let major
= unsafe { libc::major(stat.st_rdev) }
as u64;
959 let minor
= unsafe { libc::minor(stat.st_rdev) }
as u64;
961 //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
963 self.write_header(PXAR_DEVICE
, std
::mem
::size_of
::<PxarDevice
>() as u64)?
;
964 self.write_item(PxarDevice { major, minor }
)?
;
970 fn encode_special(&mut self, stat
: &FileStat
) -> Result
<(), Error
> {
972 let entry
= self.create_entry(&stat
)?
;
974 self.write_entry(entry
)?
;
979 fn encode_symlink(&mut self, target
: &[u8], stat
: &FileStat
) -> Result
<(), Error
> {
981 //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
983 let entry
= self.create_entry(&stat
)?
;
984 self.write_entry(entry
)?
;
986 self.write_header(PXAR_SYMLINK
, target
.len() as u64)?
;
992 fn encode_hardlink(&mut self, target
: &[u8], offset
: u64) -> Result
<(), Error
> {
994 //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
996 // Note: HARDLINK replaces an ENTRY.
997 self.write_header(PXAR_FORMAT_HARDLINK
, (target
.len() as u64) + 8)?
;
998 self.write_item(offset
)?
;
1004 fn encode_pxar_exclude(&mut self, filefd
: RawFd
, stat
: &FileStat
, magic
: i64, content
: &[u8]) -> Result
<(), Error
> {
1005 let mut entry
= self.create_entry(&stat
)?
;
1007 self.read_chattr(filefd
, &mut entry
)?
;
1008 self.read_fat_attr(filefd
, magic
, &mut entry
)?
;
1009 let (xattrs
, fcaps
) = self.read_xattrs(filefd
, &stat
)?
;
1010 let acl_access
= self.read_acl(filefd
, &stat
, acl
::ACL_TYPE_ACCESS
)?
;
1011 let projid
= self.read_quota_project_id(filefd
, magic
, &stat
)?
;
1013 self.write_entry(entry
)?
;
1014 for xattr
in xattrs
{
1015 self.write_xattr(xattr
)?
;
1017 self.write_fcaps(fcaps
)?
;
1018 for user
in acl_access
.users
{
1019 self.write_acl_user(user
)?
;
1021 for group
in acl_access
.groups
{
1022 self.write_acl_group(group
)?
;
1024 if let Some(group_obj
) = acl_access
.group_obj
{
1025 self.write_acl_group_obj(group_obj
)?
;
1027 if let Some(projid
) = projid
{
1028 self.write_quota_project_id(projid
)?
;
1031 let include_payload
;
1032 if is_virtual_file_system(magic
) {
1033 include_payload
= false;
1035 if let Some(set
) = &self.device_set
{
1036 include_payload
= set
.contains(&stat
.st_dev
);
1038 include_payload
= true;
1042 if !include_payload
{
1043 eprintln
!("skip content: {:?}", self.full_path());
1044 self.write_header(PXAR_PAYLOAD
, 0)?
;
1048 let size
= content
.len();
1049 self.write_header(PXAR_PAYLOAD
, size
as u64)?
;
1050 self.writer
.write_all(content
)?
;
1051 self.writer_pos
+= size
;
1056 // the report_XXX method may raise and error - depending on encoder configuration
1058 fn report_vanished_file(&self, path
: &Path
) -> Result
<(), Error
> {
1060 eprintln
!("WARNING: detected vanished file {:?}", path
);
1066 // If there is a match, an updated PxarExcludePattern list to pass to the matched child is returned.
1067 fn match_exclude_pattern(
1070 match_pattern
: &Vec
<PxarExcludePattern
>
1071 ) -> (MatchType
, Vec
<PxarExcludePattern
>) {
1072 let mut child_pattern
= Vec
::new();
1073 let mut match_state
= MatchType
::None
;
1075 for pattern
in match_pattern
{
1076 match pattern
.matches_filename(filename
, is_directory(&stat
)) {
1077 MatchType
::None
=> {}
,
1078 MatchType
::Exclude
=> match_state
= MatchType
::Exclude
,
1079 MatchType
::Include
=> match_state
= MatchType
::Include
,
1080 MatchType
::PartialExclude
=> {
1081 if match_state
!= MatchType
::Exclude
&& match_state
!= MatchType
::Include
{
1082 match_state
= MatchType
::PartialExclude
;
1084 child_pattern
.push(pattern
.get_rest_pattern());
1086 MatchType
::PartialInclude
=> {
1087 if match_state
!= MatchType
::Exclude
&& match_state
!= MatchType
::Include
{
1088 match_state
= MatchType
::PartialInclude
;
1090 child_pattern
.push(pattern
.get_rest_pattern());
1095 (match_state
, child_pattern
)
1098 fn errno_is_unsupported(errno
: Errno
) -> bool
{
1101 Errno
::ENOTTY
| Errno
::ENOSYS
| Errno
::EBADF
| Errno
::EOPNOTSUPP
| Errno
::EINVAL
=> {
1108 fn detect_fs_type(fd
: RawFd
) -> Result
<i64, Error
> {
1109 let mut fs_stat
: libc
::statfs
= unsafe { std::mem::uninitialized() }
;
1110 let res
= unsafe { libc::fstatfs(fd, &mut fs_stat) }
;
1111 Errno
::result(res
)?
;
1117 pub fn is_temporary_file_system(magic
: i64) -> bool
{
1119 magic
== RAMFS_MAGIC
|| magic
== TMPFS_MAGIC
1122 pub fn is_virtual_file_system(magic
: i64) -> bool
{
1127 CGROUP2_SUPER_MAGIC
|
1128 CGROUP_SUPER_MAGIC
|
1131 DEVPTS_SUPER_MAGIC
|
1133 FUSE_CTL_SUPER_MAGIC
|
1143 SYSFS_MAGIC
=> true,