1 //! *pxar* format encoder.
3 //! This module contain the code to generate *pxar* archive files.
4 use std
::collections
::{HashMap, HashSet}
;
5 use std
::ffi
::{CStr, CString}
;
7 use std
::os
::unix
::ffi
::OsStrExt
;
8 use std
::os
::unix
::io
::AsRawFd
;
9 use std
::os
::unix
::io
::RawFd
;
10 use std
::path
::{Path, PathBuf}
;
12 use endian_trait
::Endian
;
13 use anyhow
::{bail, format_err, Error}
;
14 use nix
::errno
::Errno
;
15 use nix
::fcntl
::OFlag
;
16 use nix
::sys
::stat
::FileStat
;
17 use nix
::sys
::stat
::Mode
;
20 use proxmox
::tools
::vec
;
22 use super::binary_search_tree
::*;
23 use super::catalog
::BackupCatalogWriter
;
25 use super::format_definition
::*;
27 use super::match_pattern
::{MatchPattern, MatchPatternSlice, MatchType}
;
28 use crate::tools
::acl
;
30 use crate::tools
::xattr
;
32 #[derive(Eq, PartialEq, Hash)]
38 pub struct Encoder
<'a
, W
: Write
, C
: BackupCatalogWriter
> {
40 relative_path
: PathBuf
,
43 catalog
: Option
<&'a
mut C
>,
45 file_copy_buffer
: Vec
<u8>,
46 device_set
: Option
<HashSet
<u64>>,
48 // Flags set by the user
50 // Flags signaling features supported by the filesystem
51 fs_feature_flags
: u64,
52 hardlinks
: HashMap
<HardLinkInfo
, (PathBuf
, u64)>,
57 impl<'a
, W
: Write
, C
: BackupCatalogWriter
> Encoder
<'a
, W
, C
> {
58 // used for error reporting
59 fn full_path(&self) -> PathBuf
{
60 self.base_path
.join(&self.relative_path
)
63 /// Create archive, write result data to ``writer``.
65 /// The ``device_set`` can be use used to limit included mount points.
67 /// - ``None``: include all mount points
68 /// - ``Some(set)``: only include devices listed in this set (the
69 /// root path device is automathically added to this list, so
70 /// you can pass an empty set if you want to archive a single
74 dir
: &mut nix
::dir
::Dir
,
76 catalog
: Option
<&'a
mut C
>,
77 device_set
: Option
<HashSet
<u64>>,
79 skip_lost_and_found
: bool
, // fixme: should be a feature flag ??
81 mut excludes
: Vec
<MatchPattern
>,
83 ) -> Result
<(), Error
> {
84 const FILE_COPY_BUFFER_SIZE
: usize = 1024 * 1024;
86 let mut file_copy_buffer
= Vec
::with_capacity(FILE_COPY_BUFFER_SIZE
);
88 file_copy_buffer
.set_len(FILE_COPY_BUFFER_SIZE
);
91 // todo: use scandirat??
93 let dir_fd
= dir
.as_raw_fd();
94 let stat
= nix
::sys
::stat
::fstat(dir_fd
)
95 .map_err(|err
| format_err
!("fstat {:?} failed - {}", path
, err
))?
;
97 if !is_directory(&stat
) {
98 bail
!("got unexpected file type {:?} (not a directory)", path
);
101 let mut device_set
= device_set
.clone();
102 if let Some(ref mut set
) = device_set
{
103 set
.insert(stat
.st_dev
);
106 let magic
= detect_fs_type(dir_fd
)?
;
108 if is_virtual_file_system(magic
) {
109 bail
!("backup virtual file systems is disabled!");
112 let fs_feature_flags
= flags
::feature_flags_from_magic(magic
);
116 relative_path
: PathBuf
::new(),
126 hardlinks
: HashMap
::new(),
132 println
!("{:?}", me
.full_path());
135 if skip_lost_and_found
{
136 excludes
.push(MatchPattern
::from_line(b
"**/lost+found").unwrap().unwrap());
138 let mut exclude_slices
= Vec
::new();
139 for excl
in &excludes
{
140 exclude_slices
.push(excl
.as_slice());
143 me
.encode_dir(dir
, &stat
, magic
, exclude_slices
)?
;
148 fn write(&mut self, buf
: &[u8]) -> Result
<(), Error
> {
149 self.writer
.write_all(buf
)?
;
150 self.writer_pos
+= buf
.len();
154 fn write_item
<T
: Endian
>(&mut self, item
: T
) -> Result
<(), Error
> {
155 let data
= item
.to_le();
157 let buffer
= unsafe {
158 std
::slice
::from_raw_parts(&data
as *const T
as *const u8, std
::mem
::size_of
::<T
>())
166 fn flush_copy_buffer(&mut self, size
: usize) -> Result
<(), Error
> {
167 self.writer
.write_all(&self.file_copy_buffer
[..size
])?
;
168 self.writer_pos
+= size
;
172 fn write_header(&mut self, htype
: u64, size
: u64) -> Result
<(), Error
> {
173 let size
= size
+ (std
::mem
::size_of
::<PxarHeader
>() as u64);
174 self.write_item(PxarHeader { size, htype }
)?
;
179 fn write_filename(&mut self, name
: &CStr
) -> Result
<(), Error
> {
180 let buffer
= name
.to_bytes_with_nul();
181 self.write_header(PXAR_FILENAME
, buffer
.len() as u64)?
;
187 fn create_entry(&self, stat
: &FileStat
) -> Result
<PxarEntry
, Error
> {
188 let mode
= if is_symlink(&stat
) {
189 (libc
::S_IFLNK
| 0o777) as u64
191 (stat
.st_mode
& (libc
::S_IFMT
| 0o7777)) as u64
194 let mtime
= stat
.st_mtime
* 1_000_000_000 + stat
.st_mtime_nsec
;
196 bail
!("got strange mtime ({}) from fstat for {:?}.", mtime
, self.full_path());
199 let entry
= PxarEntry
{
210 fn read_chattr(&self, fd
: RawFd
, entry
: &mut PxarEntry
) -> Result
<(), Error
> {
211 let mut attr
: usize = 0;
213 let res
= unsafe { fs::read_attr_fd(fd, &mut attr) }
;
214 if let Err(err
) = res
{
215 if let nix
::Error
::Sys(errno
) = err
{
216 if errno_is_unsupported(errno
) {
220 bail
!("read_attr_fd failed for {:?} - {}", self.full_path(), err
);
223 let flags
= flags
::feature_flags_from_chattr(attr
as u32);
224 entry
.flags
|= flags
;
229 fn read_fat_attr(&self, fd
: RawFd
, magic
: i64, entry
: &mut PxarEntry
) -> Result
<(), Error
> {
230 use proxmox
::sys
::linux
::magic
::*;
232 if magic
!= MSDOS_SUPER_MAGIC
&& magic
!= FUSE_SUPER_MAGIC
{
236 let mut attr
: u32 = 0;
238 let res
= unsafe { fs::read_fat_attr_fd(fd, &mut attr) }
;
239 if let Err(err
) = res
{
240 if let nix
::Error
::Sys(errno
) = err
{
241 if errno_is_unsupported(errno
) {
245 bail
!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err
);
248 let flags
= flags
::feature_flags_from_fat_attr(attr
);
249 entry
.flags
|= flags
;
254 /// True if all of the given feature flags are set in the Encoder, false otherwise
255 fn has_features(&self, feature_flags
: u64) -> bool
{
256 (self.feature_flags
& self.fs_feature_flags
& feature_flags
) == feature_flags
259 /// True if at least one of the given feature flags is set in the Encoder, false otherwise
260 fn has_some_features(&self, feature_flags
: u64) -> bool
{
261 (self.feature_flags
& self.fs_feature_flags
& feature_flags
) != 0
268 ) -> Result
<(Vec
<PxarXAttr
>, Option
<PxarFCaps
>), Error
> {
269 let mut xattrs
= Vec
::new();
270 let mut fcaps
= None
;
272 let flags
= flags
::WITH_XATTRS
| flags
::WITH_FCAPS
;
273 if !self.has_some_features(flags
) {
274 return Ok((xattrs
, fcaps
));
276 // Should never be called on symlinks, just in case check anyway
277 if is_symlink(&stat
) {
278 return Ok((xattrs
, fcaps
));
281 let xattr_names
= match xattr
::flistxattr(fd
) {
283 // Do not bail if the underlying endpoint does not supports xattrs
284 Err(Errno
::EOPNOTSUPP
) => return Ok((xattrs
, fcaps
)),
285 // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
286 Err(Errno
::EBADF
) => return Ok((xattrs
, fcaps
)),
287 Err(err
) => bail
!("read_xattrs failed for {:?} - {}", self.full_path(), err
),
290 for name
in &xattr_names
{
291 // Only extract the relevant extended attributes
292 if !xattr
::is_valid_xattr_name(&name
) {
296 let value
= match xattr
::fgetxattr(fd
, name
) {
298 // Vanished between flistattr and getxattr, this is ok, silently ignore
299 Err(Errno
::ENODATA
) => continue,
300 Err(err
) => bail
!("read_xattrs failed for {:?} - {}", self.full_path(), err
),
303 if xattr
::is_security_capability(&name
) {
304 if self.has_features(flags
::WITH_FCAPS
) {
305 // fcaps are stored in own format within the archive
306 fcaps
= Some(PxarFCaps { data: value }
);
308 } else if self.has_features(flags
::WITH_XATTRS
) {
309 xattrs
.push(PxarXAttr
{
310 name
: name
.to_bytes().to_vec(),
324 acl_type
: acl
::ACLType
,
325 ) -> Result
<PxarACL
, Error
> {
333 if !self.has_features(flags
::WITH_ACL
) {
336 if is_symlink(&stat
) {
339 if acl_type
== acl
::ACL_TYPE_DEFAULT
&& !is_directory(&stat
) {
340 bail
!("ACL_TYPE_DEFAULT only defined for directories.");
343 // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
344 // to create a path for acl_get_file(). acl_get_fd() only allows to get
345 // ACL_TYPE_ACCESS attributes.
346 let proc_path
= Path
::new("/proc/self/fd/").join(fd
.to_string());
347 let acl
= match acl
::ACL
::get_file(&proc_path
, acl_type
) {
349 // Don't bail if underlying endpoint does not support acls
350 Err(Errno
::EOPNOTSUPP
) => return Ok(ret
),
351 // Don't bail if the endpoint cannot carry acls
352 Err(Errno
::EBADF
) => return Ok(ret
),
353 // Don't bail if there is no data
354 Err(Errno
::ENODATA
) => return Ok(ret
),
355 Err(err
) => bail
!("error while reading ACL - {}", err
),
358 self.process_acl(acl
, acl_type
)
361 fn process_acl(&self, acl
: acl
::ACL
, acl_type
: acl
::ACLType
) -> Result
<PxarACL
, Error
> {
362 let mut acl_user
= Vec
::new();
363 let mut acl_group
= Vec
::new();
364 let mut acl_group_obj
= None
;
365 let mut acl_default
= None
;
366 let mut user_obj_permissions
= None
;
367 let mut group_obj_permissions
= None
;
368 let mut other_permissions
= None
;
369 let mut mask_permissions
= None
;
371 for entry
in &mut acl
.entries() {
372 let tag
= entry
.get_tag_type()?
;
373 let permissions
= entry
.get_permissions()?
;
375 acl
::ACL_USER_OBJ
=> user_obj_permissions
= Some(permissions
),
376 acl
::ACL_GROUP_OBJ
=> group_obj_permissions
= Some(permissions
),
377 acl
::ACL_OTHER
=> other_permissions
= Some(permissions
),
378 acl
::ACL_MASK
=> mask_permissions
= Some(permissions
),
380 acl_user
.push(PxarACLUser
{
381 uid
: entry
.get_qualifier()?
,
386 acl_group
.push(PxarACLGroup
{
387 gid
: entry
.get_qualifier()?
,
391 _
=> bail
!("Unexpected ACL tag encountered!"),
399 acl
::ACL_TYPE_ACCESS
=> {
400 // The mask permissions are mapped to the stat group permissions
401 // in case that the ACL group permissions were set.
402 // Only in that case we need to store the group permissions,
403 // in the other cases they are identical to the stat group permissions.
404 if let (Some(gop
), Some(_
)) = (group_obj_permissions
, mask_permissions
) {
405 acl_group_obj
= Some(PxarACLGroupObj { permissions: gop }
);
408 acl
::ACL_TYPE_DEFAULT
=> {
409 if user_obj_permissions
!= None
410 || group_obj_permissions
!= None
411 || other_permissions
!= None
412 || mask_permissions
!= None
414 acl_default
= Some(PxarACLDefault
{
415 // The value is set to UINT64_MAX as placeholder if one
416 // of the permissions is not set
417 user_obj_permissions
: user_obj_permissions
.unwrap_or(std
::u64::MAX
),
418 group_obj_permissions
: group_obj_permissions
.unwrap_or(std
::u64::MAX
),
419 other_permissions
: other_permissions
.unwrap_or(std
::u64::MAX
),
420 mask_permissions
: mask_permissions
.unwrap_or(std
::u64::MAX
),
424 _
=> bail
!("Unexpected ACL type encountered"),
430 group_obj
: acl_group_obj
,
431 default: acl_default
,
435 /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
436 fn read_quota_project_id(
441 ) -> Result
<Option
<PxarQuotaProjID
>, Error
> {
442 if !(is_directory(&stat
) || is_reg_file(&stat
)) {
445 if !self.has_features(flags
::WITH_QUOTA_PROJID
) {
449 use proxmox
::sys
::linux
::magic
::*;
452 EXT4_SUPER_MAGIC
| XFS_SUPER_MAGIC
| FUSE_SUPER_MAGIC
| ZFS_SUPER_MAGIC
=> {
453 let mut fsxattr
= fs
::FSXAttr
::default();
454 let res
= unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) }
;
456 // On some FUSE filesystems it can happen that ioctl is not supported.
457 // For these cases projid is set to 0 while the error is ignored.
458 if let Err(err
) = res
{
459 let errno
= err
.as_errno().ok_or_else(|| {
461 "error while reading quota project id for {:#?}",
465 if errno_is_unsupported(errno
) {
469 "error while reading quota project id for {:#?} - {}",
476 let projid
= fsxattr
.fsx_projid
as u64;
480 Ok(Some(PxarQuotaProjID { projid }
))
487 fn write_entry(&mut self, entry
: PxarEntry
) -> Result
<(), Error
> {
488 self.write_header(PXAR_ENTRY
, std
::mem
::size_of
::<PxarEntry
>() as u64)?
;
489 self.write_item(entry
)?
;
494 fn write_xattr(&mut self, xattr
: PxarXAttr
) -> Result
<(), Error
> {
495 let size
= xattr
.name
.len() + xattr
.value
.len() + 1; // +1 for '\0' separating name and value
496 self.write_header(PXAR_XATTR
, size
as u64)?
;
497 self.write(xattr
.name
.as_slice())?
;
499 self.write(xattr
.value
.as_slice())?
;
504 fn write_fcaps(&mut self, fcaps
: Option
<PxarFCaps
>) -> Result
<(), Error
> {
505 if let Some(fcaps
) = fcaps
{
506 let size
= fcaps
.data
.len();
507 self.write_header(PXAR_FCAPS
, size
as u64)?
;
508 self.write(fcaps
.data
.as_slice())?
;
514 fn write_acl_user(&mut self, acl_user
: PxarACLUser
) -> Result
<(), Error
> {
515 self.write_header(PXAR_ACL_USER
, std
::mem
::size_of
::<PxarACLUser
>() as u64)?
;
516 self.write_item(acl_user
)?
;
521 fn write_acl_group(&mut self, acl_group
: PxarACLGroup
) -> Result
<(), Error
> {
522 self.write_header(PXAR_ACL_GROUP
, std
::mem
::size_of
::<PxarACLGroup
>() as u64)?
;
523 self.write_item(acl_group
)?
;
528 fn write_acl_group_obj(&mut self, acl_group_obj
: PxarACLGroupObj
) -> Result
<(), Error
> {
531 std
::mem
::size_of
::<PxarACLGroupObj
>() as u64,
533 self.write_item(acl_group_obj
)?
;
538 fn write_acl_default(&mut self, acl_default
: PxarACLDefault
) -> Result
<(), Error
> {
541 std
::mem
::size_of
::<PxarACLDefault
>() as u64,
543 self.write_item(acl_default
)?
;
548 fn write_acl_default_user(&mut self, acl_default_user
: PxarACLUser
) -> Result
<(), Error
> {
550 PXAR_ACL_DEFAULT_USER
,
551 std
::mem
::size_of
::<PxarACLUser
>() as u64,
553 self.write_item(acl_default_user
)?
;
558 fn write_acl_default_group(&mut self, acl_default_group
: PxarACLGroup
) -> Result
<(), Error
> {
560 PXAR_ACL_DEFAULT_GROUP
,
561 std
::mem
::size_of
::<PxarACLGroup
>() as u64,
563 self.write_item(acl_default_group
)?
;
568 fn write_quota_project_id(&mut self, projid
: PxarQuotaProjID
) -> Result
<(), Error
> {
571 std
::mem
::size_of
::<PxarQuotaProjID
>() as u64,
573 self.write_item(projid
)?
;
578 fn write_goodbye_table(
580 goodbye_offset
: usize,
581 goodbye_items
: &mut [PxarGoodbyeItem
],
582 ) -> Result
<(), Error
> {
583 goodbye_items
.sort_unstable_by(|a
, b
| a
.hash
.cmp(&b
.hash
));
585 let item_count
= goodbye_items
.len();
587 let goodbye_table_size
= (item_count
+ 1) * std
::mem
::size_of
::<PxarGoodbyeItem
>();
589 self.write_header(PXAR_GOODBYE
, goodbye_table_size
as u64)?
;
591 if self.file_copy_buffer
.len() < goodbye_table_size
{
592 let need
= goodbye_table_size
- self.file_copy_buffer
.len();
593 self.file_copy_buffer
.reserve(need
);
595 self.file_copy_buffer
596 .set_len(self.file_copy_buffer
.capacity());
600 let buffer
= &mut self.file_copy_buffer
;
602 copy_binary_search_tree(item_count
, |s
, d
| {
603 let item
= &goodbye_items
[s
];
604 let offset
= d
* std
::mem
::size_of
::<PxarGoodbyeItem
>();
606 crate::tools
::map_struct_mut
::<PxarGoodbyeItem
>(&mut buffer
[offset
..]).unwrap();
607 dest
.offset
= u64::to_le(item
.offset
);
608 dest
.size
= u64::to_le(item
.size
);
609 dest
.hash
= u64::to_le(item
.hash
);
612 // append PxarGoodbyeTail as last item
613 let offset
= item_count
* std
::mem
::size_of
::<PxarGoodbyeItem
>();
614 let dest
= crate::tools
::map_struct_mut
::<PxarGoodbyeItem
>(&mut buffer
[offset
..]).unwrap();
615 dest
.offset
= u64::to_le(goodbye_offset
as u64);
616 dest
.size
= u64::to_le((goodbye_table_size
+ std
::mem
::size_of
::<PxarHeader
>()) as u64);
617 dest
.hash
= u64::to_le(PXAR_GOODBYE_TAIL_MARKER
);
619 self.flush_copy_buffer(goodbye_table_size
)?
;
626 dir
: &mut nix
::dir
::Dir
,
629 match_pattern
: Vec
<MatchPatternSlice
>,
630 ) -> Result
<(), Error
> {
631 //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
633 let mut name_list
= Vec
::new();
635 let rawfd
= dir
.as_raw_fd();
637 let dir_start_pos
= self.writer_pos
;
639 let is_root
= dir_start_pos
== 0;
641 let mut dir_entry
= self.create_entry(&dir_stat
)?
;
643 self.read_chattr(rawfd
, &mut dir_entry
)?
;
644 self.read_fat_attr(rawfd
, magic
, &mut dir_entry
)?
;
646 // for each node in the directory tree, the filesystem features are
647 // checked based on the fs magic number.
648 self.fs_feature_flags
= flags
::feature_flags_from_magic(magic
);
650 let (xattrs
, fcaps
) = self.read_xattrs(rawfd
, &dir_stat
)?
;
651 let acl_access
= self.read_acl(rawfd
, &dir_stat
, acl
::ACL_TYPE_ACCESS
)?
;
652 let acl_default
= self.read_acl(rawfd
, &dir_stat
, acl
::ACL_TYPE_DEFAULT
)?
;
653 let projid
= self.read_quota_project_id(rawfd
, magic
, &dir_stat
)?
;
655 self.write_entry(dir_entry
)?
;
656 for xattr
in xattrs
{
657 self.write_xattr(xattr
)?
;
659 self.write_fcaps(fcaps
)?
;
661 for user
in acl_access
.users
{
662 self.write_acl_user(user
)?
;
664 for group
in acl_access
.groups
{
665 self.write_acl_group(group
)?
;
667 if let Some(group_obj
) = acl_access
.group_obj
{
668 self.write_acl_group_obj(group_obj
)?
;
671 for default_user
in acl_default
.users
{
672 self.write_acl_default_user(default_user
)?
;
674 for default_group
in acl_default
.groups
{
675 self.write_acl_default_group(default_group
)?
;
677 if let Some(default) = acl_default
.default {
678 self.write_acl_default(default)?
;
680 if let Some(projid
) = projid
{
681 self.write_quota_project_id(projid
)?
;
684 let include_children
;
685 if is_virtual_file_system(magic
) {
686 include_children
= false;
687 } else if let Some(set
) = &self.device_set
{
688 include_children
= set
.contains(&dir_stat
.st_dev
);
690 include_children
= true;
693 // Expand the exclude match pattern inherited from the parent by local entries, if present
694 let mut local_match_pattern
= match_pattern
.clone();
695 let (pxar_exclude
, excludes
) = match MatchPattern
::from_file(rawfd
, ".pxarexclude") {
696 Ok(Some((excludes
, buffer
, stat
))) => {
697 (Some((buffer
, stat
)), excludes
)
699 Ok(None
) => (None
, Vec
::new()),
700 Err(nix
::Error
::Sys(Errno
::EACCES
)) => {
701 // No permission to read .pxarexclude, ignore its contents.
703 "ignoring match patterns in {:?}: open file failed - EACCES",
704 self.full_path().join(".pxarexclude"),
708 Err(err
) => bail
!("error while reading exclude file - {}", err
),
710 for excl
in &excludes
{
711 local_match_pattern
.push(excl
.as_slice());
714 if include_children
{
715 // Exclude patterns passed via the CLI are stored as '.pxarexclude-cli'
716 // in the root directory of the archive.
717 if is_root
&& !match_pattern
.is_empty() {
718 let filename
= CString
::new(".pxarexclude-cli")?
;
719 name_list
.push((filename
, *dir_stat
, match_pattern
.clone()));
722 for entry
in dir
.iter() {
724 .map_err(|err
| format_err
!("readir {:?} failed - {}", self.full_path(), err
))?
;
725 let filename
= entry
.file_name().to_owned();
727 let name
= filename
.to_bytes_with_nul();
728 if name
== b
".\0" || name
== b
"..\0" {
731 // Do not store a ".pxarexclude-cli" file found in the archive root,
732 // as this would confilict with new cli passed exclude patterns,
734 if is_root
&& name
== b
".pxarexclude-cli\0" {
735 eprintln
!("skip existing '.pxarexclude-cli' in archive root.");
739 let stat
= match nix
::sys
::stat
::fstatat(
742 nix
::fcntl
::AtFlags
::AT_SYMLINK_NOFOLLOW
,
745 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
746 let filename_osstr
= std
::ffi
::OsStr
::from_bytes(filename
.to_bytes());
747 self.report_vanished_file(&self.full_path().join(filename_osstr
))?
;
750 Err(err
) => bail
!("fstat {:?} failed - {}", self.full_path(), err
),
753 match MatchPatternSlice
::match_filename_exclude(
756 &local_match_pattern
,
758 (MatchType
::Positive
, _
) => {
759 let filename_osstr
= std
::ffi
::OsStr
::from_bytes(filename
.to_bytes());
761 "matched by exclude pattern - skipping: {:?}",
762 self.full_path().join(filename_osstr
)
765 (_
, child_pattern
) => {
766 self.entry_counter
+= 1;
767 name_list
.push((filename
, stat
, child_pattern
));
771 if self.entry_counter
> self.entry_max
{
773 "exceeded max number of entries (> {})",
779 eprintln
!("skip mount point: {:?}", self.full_path());
782 name_list
.sort_unstable_by(|a
, b
| a
.0.cmp(&b
.0));
783 let num_entries
= name_list
.len();
785 let mut goodbye_items
= Vec
::with_capacity(num_entries
);
787 for (filename
, stat
, exclude_list
) in name_list
{
788 let start_pos
= self.writer_pos
;
790 if filename
.as_bytes() == b
".pxarexclude" {
791 // pxar_exclude is none in case of error EACCES.
792 if let Some((ref content
, ref stat
)) = pxar_exclude
{
793 let filefd
= match nix
::fcntl
::openat(
799 Ok(filefd
) => filefd
,
800 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
801 self.report_vanished_file(&self.full_path())?
;
804 Err(nix
::Error
::Sys(Errno
::EACCES
)) => {
805 let filename_osstr
= std
::ffi
::OsStr
::from_bytes(filename
.to_bytes());
807 "skipping {:?}: open file failed - EACCES",
808 self.full_path().join(filename_osstr
),
813 let filename_osstr
= std
::ffi
::OsStr
::from_bytes(filename
.to_bytes());
815 "open file {:?} failed - {}",
816 self.full_path().join(filename_osstr
),
822 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
823 detect_fs_type(filefd
)?
828 self.write_filename(&filename
)?
;
829 if let Some(ref mut catalog
) = self.catalog
{
830 catalog
.add_file(&filename
, stat
.st_size
as u64, stat
.st_mtime
as u64)?
;
832 self.encode_pxar_exclude(filefd
, stat
, child_magic
, content
)?
;
837 if is_root
&& filename
.as_bytes() == b
".pxarexclude-cli" {
838 // '.pxarexclude-cli' is used to store the exclude MatchPatterns
839 // passed via the cli in the root directory of the archive.
840 self.write_filename(&filename
)?
;
841 let content
= MatchPatternSlice
::to_bytes(&exclude_list
);
842 if let Some(ref mut catalog
) = self.catalog
{
843 catalog
.add_file(&filename
, content
.len() as u64, 0)?
;
845 self.encode_pxar_exclude_cli(stat
.st_uid
, stat
.st_gid
, 0, &content
)?
;
850 .push(std
::ffi
::OsStr
::from_bytes(filename
.as_bytes()));
853 println
!("{:?}", self.full_path());
856 if is_directory(&stat
) {
857 let mut dir
= match nix
::dir
::Dir
::openat(
860 OFlag
::O_DIRECTORY
| OFlag
::O_NOFOLLOW
,
864 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
865 self.report_vanished_file(&self.full_path())?
;
866 self.relative_path
.pop();
869 Err(nix
::Error
::Sys(Errno
::EACCES
)) => {
871 "skipping {:?}: open dir failed - EACCES",
874 self.relative_path
.pop();
877 Err(err
) => bail
!("open dir {:?} failed - {}", self.full_path(), err
),
880 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
881 detect_fs_type(dir
.as_raw_fd())?
886 self.write_filename(&filename
)?
;
887 if let Some(ref mut catalog
) = self.catalog
{
888 catalog
.start_directory(&filename
)?
;
890 self.encode_dir(&mut dir
, &stat
, child_magic
, exclude_list
)?
;
891 if let Some(ref mut catalog
) = self.catalog
{
892 catalog
.end_directory()?
;
894 } else if is_reg_file(&stat
) {
895 let mut hardlink_target
= None
;
897 if stat
.st_nlink
> 1 {
898 let link_info
= HardLinkInfo
{
902 hardlink_target
= self.hardlinks
.get(&link_info
).map(|(v
, offset
)| {
903 let mut target
= v
.clone().into_os_string();
904 target
.push("\0"); // add Nul byte
905 (target
, (start_pos
as u64) - offset
)
907 if hardlink_target
== None
{
909 .insert(link_info
, (self.relative_path
.clone(), start_pos
as u64));
913 if let Some((target
, offset
)) = hardlink_target
{
914 if let Some(ref mut catalog
) = self.catalog
{
915 catalog
.add_hardlink(&filename
)?
;
917 self.write_filename(&filename
)?
;
918 self.encode_hardlink(target
.as_bytes(), offset
)?
;
920 let filefd
= match nix
::fcntl
::openat(
926 Ok(filefd
) => filefd
,
927 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
928 self.report_vanished_file(&self.full_path())?
;
929 self.relative_path
.pop();
932 Err(nix
::Error
::Sys(Errno
::EACCES
)) => {
934 "skipping {:?}: open file failed - EACCES",
937 self.relative_path
.pop();
940 Err(err
) => bail
!("open file {:?} failed - {}", self.full_path(), err
),
943 if let Some(ref mut catalog
) = self.catalog
{
944 catalog
.add_file(&filename
, stat
.st_size
as u64, stat
.st_mtime
as u64)?
;
946 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
947 detect_fs_type(filefd
)?
952 self.write_filename(&filename
)?
;
953 let res
= self.encode_file(filefd
, &stat
, child_magic
);
954 let _
= nix
::unistd
::close(filefd
); // ignore close errors
957 } else if is_symlink(&stat
) {
958 let mut buffer
= vec
::undefined(libc
::PATH_MAX
as usize);
960 let res
= filename
.with_nix_path(|cstr
| unsafe {
964 buffer
.as_mut_ptr() as *mut libc
::c_char
,
969 match Errno
::result(res
) {
971 if let Some(ref mut catalog
) = self.catalog
{
972 catalog
.add_symlink(&filename
)?
;
974 buffer
[len
as usize] = 0u8; // add Nul byte
975 self.write_filename(&filename
)?
;
976 self.encode_symlink(&buffer
[..((len
+ 1) as usize)], &stat
)?
978 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
979 self.report_vanished_file(&self.full_path())?
;
980 self.relative_path
.pop();
983 Err(err
) => bail
!("readlink {:?} failed - {}", self.full_path(), err
),
985 } else if is_block_dev(&stat
) || is_char_dev(&stat
) {
986 if self.has_features(flags
::WITH_DEVICE_NODES
) {
987 if let Some(ref mut catalog
) = self.catalog
{
988 if is_block_dev(&stat
) {
989 catalog
.add_block_device(&filename
)?
;
991 catalog
.add_char_device(&filename
)?
;
994 self.write_filename(&filename
)?
;
995 self.encode_device(&stat
)?
;
997 eprintln
!("skip device node: {:?}", self.full_path());
998 self.relative_path
.pop();
1001 } else if is_fifo(&stat
) {
1002 if self.has_features(flags
::WITH_FIFOS
) {
1003 if let Some(ref mut catalog
) = self.catalog
{
1004 catalog
.add_fifo(&filename
)?
;
1006 self.write_filename(&filename
)?
;
1007 self.encode_special(&stat
)?
;
1009 eprintln
!("skip fifo: {:?}", self.full_path());
1010 self.relative_path
.pop();
1013 } else if is_socket(&stat
) {
1014 if self.has_features(flags
::WITH_SOCKETS
) {
1015 if let Some(ref mut catalog
) = self.catalog
{
1016 catalog
.add_socket(&filename
)?
;
1018 self.write_filename(&filename
)?
;
1019 self.encode_special(&stat
)?
;
1021 eprintln
!("skip socket: {:?}", self.full_path());
1022 self.relative_path
.pop();
1027 "unsupported file type (mode {:o} {:?})",
1033 let end_pos
= self.writer_pos
;
1035 goodbye_items
.push(PxarGoodbyeItem
{
1036 offset
: start_pos
as u64,
1037 size
: (end_pos
- start_pos
) as u64,
1038 hash
: compute_goodbye_hash(filename
.to_bytes()),
1041 self.relative_path
.pop();
1044 //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
1046 // fixup goodby item offsets
1047 let goodbye_start
= self.writer_pos
as u64;
1048 for item
in &mut goodbye_items
{
1049 item
.offset
= goodbye_start
- item
.offset
;
1052 let goodbye_offset
= self.writer_pos
- dir_start_pos
;
1054 self.write_goodbye_table(goodbye_offset
, &mut goodbye_items
)?
;
1055 self.entry_counter
-= num_entries
;
1057 //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
1061 fn encode_file(&mut self, filefd
: RawFd
, stat
: &FileStat
, magic
: i64) -> Result
<(), Error
> {
1062 //println!("encode_file: {:?}", self.full_path());
1064 let mut entry
= self.create_entry(&stat
)?
;
1066 self.read_chattr(filefd
, &mut entry
)?
;
1067 self.read_fat_attr(filefd
, magic
, &mut entry
)?
;
1068 let (xattrs
, fcaps
) = self.read_xattrs(filefd
, &stat
)?
;
1069 let acl_access
= self.read_acl(filefd
, &stat
, acl
::ACL_TYPE_ACCESS
)?
;
1070 let projid
= self.read_quota_project_id(filefd
, magic
, &stat
)?
;
1072 self.write_entry(entry
)?
;
1073 for xattr
in xattrs
{
1074 self.write_xattr(xattr
)?
;
1076 self.write_fcaps(fcaps
)?
;
1077 for user
in acl_access
.users
{
1078 self.write_acl_user(user
)?
;
1080 for group
in acl_access
.groups
{
1081 self.write_acl_group(group
)?
;
1083 if let Some(group_obj
) = acl_access
.group_obj
{
1084 self.write_acl_group_obj(group_obj
)?
;
1086 if let Some(projid
) = projid
{
1087 self.write_quota_project_id(projid
)?
;
1090 let include_payload
;
1091 if is_virtual_file_system(magic
) {
1092 include_payload
= false;
1093 } else if let Some(ref set
) = &self.device_set
{
1094 include_payload
= set
.contains(&stat
.st_dev
);
1096 include_payload
= true;
1099 if !include_payload
{
1100 eprintln
!("skip content: {:?}", self.full_path());
1101 self.write_header(PXAR_PAYLOAD
, 0)?
;
1105 let size
= stat
.st_size
as u64;
1107 self.write_header(PXAR_PAYLOAD
, size
)?
;
1109 let mut pos
: u64 = 0;
1111 let n
= match nix
::unistd
::read(filefd
, &mut self.file_copy_buffer
) {
1113 Err(nix
::Error
::Sys(Errno
::EINTR
)) => continue, /* try again */
1114 Err(err
) => bail
!("read {:?} failed - {}", self.full_path(), err
),
1118 // Note:: casync format cannot handle that
1120 "detected shrunk file {:?} ({} < {})",
1129 let mut next
= pos
+ (n
as u64);
1135 let count
= (next
- pos
) as usize;
1137 self.flush_copy_buffer(count
)?
;
1149 fn encode_device(&mut self, stat
: &FileStat
) -> Result
<(), Error
> {
1150 let entry
= self.create_entry(&stat
)?
;
1152 self.write_entry(entry
)?
;
1154 let major
= unsafe { libc::major(stat.st_rdev) }
as u64;
1155 let minor
= unsafe { libc::minor(stat.st_rdev) }
as u64;
1157 //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
1159 self.write_header(PXAR_DEVICE
, std
::mem
::size_of
::<PxarDevice
>() as u64)?
;
1160 self.write_item(PxarDevice { major, minor }
)?
;
1166 fn encode_special(&mut self, stat
: &FileStat
) -> Result
<(), Error
> {
1167 let entry
= self.create_entry(&stat
)?
;
1169 self.write_entry(entry
)?
;
1174 fn encode_symlink(&mut self, target
: &[u8], stat
: &FileStat
) -> Result
<(), Error
> {
1175 //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
1177 let entry
= self.create_entry(&stat
)?
;
1178 self.write_entry(entry
)?
;
1180 self.write_header(PXAR_SYMLINK
, target
.len() as u64)?
;
1181 self.write(target
)?
;
1186 fn encode_hardlink(&mut self, target
: &[u8], offset
: u64) -> Result
<(), Error
> {
1187 //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
1189 // Note: HARDLINK replaces an ENTRY.
1190 self.write_header(PXAR_FORMAT_HARDLINK
, (target
.len() as u64) + 8)?
;
1191 self.write_item(offset
)?
;
1192 self.write(target
)?
;
1197 fn encode_pxar_exclude(
1203 ) -> Result
<(), Error
> {
1204 let mut entry
= self.create_entry(&stat
)?
;
1206 self.read_chattr(filefd
, &mut entry
)?
;
1207 self.read_fat_attr(filefd
, magic
, &mut entry
)?
;
1208 let (xattrs
, fcaps
) = self.read_xattrs(filefd
, &stat
)?
;
1209 let acl_access
= self.read_acl(filefd
, &stat
, acl
::ACL_TYPE_ACCESS
)?
;
1210 let projid
= self.read_quota_project_id(filefd
, magic
, &stat
)?
;
1212 self.write_entry(entry
)?
;
1213 for xattr
in xattrs
{
1214 self.write_xattr(xattr
)?
;
1216 self.write_fcaps(fcaps
)?
;
1217 for user
in acl_access
.users
{
1218 self.write_acl_user(user
)?
;
1220 for group
in acl_access
.groups
{
1221 self.write_acl_group(group
)?
;
1223 if let Some(group_obj
) = acl_access
.group_obj
{
1224 self.write_acl_group_obj(group_obj
)?
;
1226 if let Some(projid
) = projid
{
1227 self.write_quota_project_id(projid
)?
;
1230 let include_payload
;
1231 if is_virtual_file_system(magic
) {
1232 include_payload
= false;
1233 } else if let Some(set
) = &self.device_set
{
1234 include_payload
= set
.contains(&stat
.st_dev
);
1236 include_payload
= true;
1239 if !include_payload
{
1240 eprintln
!("skip content: {:?}", self.full_path());
1241 self.write_header(PXAR_PAYLOAD
, 0)?
;
1245 let size
= content
.len();
1246 self.write_header(PXAR_PAYLOAD
, size
as u64)?
;
1247 self.writer
.write_all(content
)?
;
1248 self.writer_pos
+= size
;
1253 /// Encodes the excude match patterns passed via cli as file in the archive.
1254 fn encode_pxar_exclude_cli(
1260 ) -> Result
<(), Error
> {
1261 let entry
= PxarEntry
{
1262 mode
: (libc
::S_IFREG
| 0o600) as u64,
1268 self.write_entry(entry
)?
;
1269 let size
= content
.len();
1270 self.write_header(PXAR_PAYLOAD
, size
as u64)?
;
1271 self.writer
.write_all(content
)?
;
1272 self.writer_pos
+= size
;
1277 // the report_XXX method may raise and error - depending on encoder configuration
1279 fn report_vanished_file(&self, path
: &Path
) -> Result
<(), Error
> {
1280 eprintln
!("WARNING: detected vanished file {:?}", path
);
1286 fn errno_is_unsupported(errno
: Errno
) -> bool
{
1288 Errno
::ENOTTY
| Errno
::ENOSYS
| Errno
::EBADF
| Errno
::EOPNOTSUPP
| Errno
::EINVAL
=> true,
1293 fn detect_fs_type(fd
: RawFd
) -> Result
<i64, Error
> {
1294 let mut fs_stat
= std
::mem
::MaybeUninit
::uninit();
1295 let res
= unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) }
;
1296 Errno
::result(res
)?
;
1297 let fs_stat
= unsafe { fs_stat.assume_init() }
;
1303 pub fn is_temporary_file_system(magic
: i64) -> bool
{
1304 use proxmox
::sys
::linux
::magic
::*;
1305 magic
== RAMFS_MAGIC
|| magic
== TMPFS_MAGIC
1308 pub fn is_virtual_file_system(magic
: i64) -> bool
{
1309 use proxmox
::sys
::linux
::magic
::*;
1313 CGROUP2_SUPER_MAGIC
|
1314 CGROUP_SUPER_MAGIC
|
1317 DEVPTS_SUPER_MAGIC
|
1319 FUSE_CTL_SUPER_MAGIC
|
1329 SYSFS_MAGIC
=> true,