1 //! *pxar* format encoder.
3 //! This module contain the code to generate *pxar* archive files.
4 use std
::collections
::{HashMap, HashSet}
;
5 use std
::ffi
::{CStr, CString}
;
7 use std
::os
::unix
::ffi
::OsStrExt
;
8 use std
::os
::unix
::io
::AsRawFd
;
9 use std
::os
::unix
::io
::RawFd
;
10 use std
::path
::{Path, PathBuf}
;
12 use endian_trait
::Endian
;
14 use nix
::errno
::Errno
;
15 use nix
::fcntl
::OFlag
;
16 use nix
::sys
::stat
::FileStat
;
17 use nix
::sys
::stat
::Mode
;
20 use proxmox
::tools
::vec
;
22 use super::binary_search_tree
::*;
23 use super::catalog
::BackupCatalogWriter
;
25 use super::format_definition
::*;
27 use super::match_pattern
::{MatchPattern, MatchType}
;
28 use crate::tools
::acl
;
30 use crate::tools
::xattr
;
32 /// The format requires to build sorted directory lookup tables in
33 /// memory, so we restrict the number of allowed entries to limit
34 /// maximum memory usage.
35 pub const MAX_DIRECTORY_ENTRIES
: usize = 256 * 1024;
37 #[derive(Eq, PartialEq, Hash)]
43 pub struct Encoder
<'a
, W
: Write
, C
: BackupCatalogWriter
> {
45 relative_path
: PathBuf
,
48 catalog
: Option
<&'a
mut C
>,
50 file_copy_buffer
: Vec
<u8>,
51 device_set
: Option
<HashSet
<u64>>,
53 // Flags set by the user
55 // Flags signaling features supported by the filesystem
56 fs_feature_flags
: u64,
57 hardlinks
: HashMap
<HardLinkInfo
, (PathBuf
, u64)>,
60 impl<'a
, W
: Write
, C
: BackupCatalogWriter
> Encoder
<'a
, W
, C
> {
61 // used for error reporting
62 fn full_path(&self) -> PathBuf
{
63 self.base_path
.join(&self.relative_path
)
66 /// Create archive, write result data to ``writer``.
68 /// The ``device_set`` can be use used to limit included mount points.
70 /// - ``None``: include all mount points
71 /// - ``Some(set)``: only include devices listed in this set (the
72 /// root path device is automathically added to this list, so
73 /// you can pass an empty set if you want to archive a single
77 dir
: &mut nix
::dir
::Dir
,
79 catalog
: Option
<&'a
mut C
>,
80 device_set
: Option
<HashSet
<u64>>,
82 skip_lost_and_found
: bool
, // fixme: should be a feature flag ??
84 mut excludes
: Vec
<MatchPattern
>,
85 ) -> Result
<(), Error
> {
86 const FILE_COPY_BUFFER_SIZE
: usize = 1024 * 1024;
88 let mut file_copy_buffer
= Vec
::with_capacity(FILE_COPY_BUFFER_SIZE
);
90 file_copy_buffer
.set_len(FILE_COPY_BUFFER_SIZE
);
93 // todo: use scandirat??
95 let dir_fd
= dir
.as_raw_fd();
96 let stat
= nix
::sys
::stat
::fstat(dir_fd
)
97 .map_err(|err
| format_err
!("fstat {:?} failed - {}", path
, err
))?
;
99 if !is_directory(&stat
) {
100 bail
!("got unexpected file type {:?} (not a directory)", path
);
103 let mut device_set
= device_set
.clone();
104 if let Some(ref mut set
) = device_set
{
105 set
.insert(stat
.st_dev
);
108 let magic
= detect_fs_type(dir_fd
)?
;
110 if is_virtual_file_system(magic
) {
111 bail
!("backup virtual file systems is disabled!");
114 let fs_feature_flags
= flags
::feature_flags_from_magic(magic
);
118 relative_path
: PathBuf
::new(),
128 hardlinks
: HashMap
::new(),
132 println
!("{:?}", me
.full_path());
135 if skip_lost_and_found
{
136 excludes
.push(MatchPattern
::from_line(b
"**/lost+found").unwrap().unwrap());
139 me
.encode_dir(dir
, &stat
, magic
, excludes
)?
;
144 fn write(&mut self, buf
: &[u8]) -> Result
<(), Error
> {
145 self.writer
.write_all(buf
)?
;
146 self.writer_pos
+= buf
.len();
150 fn write_item
<T
: Endian
>(&mut self, item
: T
) -> Result
<(), Error
> {
151 let data
= item
.to_le();
153 let buffer
= unsafe {
154 std
::slice
::from_raw_parts(&data
as *const T
as *const u8, std
::mem
::size_of
::<T
>())
162 fn flush_copy_buffer(&mut self, size
: usize) -> Result
<(), Error
> {
163 self.writer
.write_all(&self.file_copy_buffer
[..size
])?
;
164 self.writer_pos
+= size
;
168 fn write_header(&mut self, htype
: u64, size
: u64) -> Result
<(), Error
> {
169 let size
= size
+ (std
::mem
::size_of
::<PxarHeader
>() as u64);
170 self.write_item(PxarHeader { size, htype }
)?
;
175 fn write_filename(&mut self, name
: &CStr
) -> Result
<(), Error
> {
176 let buffer
= name
.to_bytes_with_nul();
177 self.write_header(PXAR_FILENAME
, buffer
.len() as u64)?
;
183 fn create_entry(&self, stat
: &FileStat
) -> Result
<PxarEntry
, Error
> {
184 let mode
= if is_symlink(&stat
) {
185 (libc
::S_IFLNK
| 0o777) as u64
187 (stat
.st_mode
& (libc
::S_IFMT
| 0o7777)) as u64
190 let mtime
= stat
.st_mtime
* 1_000_000_000 + stat
.st_mtime_nsec
;
192 bail
!("got strange mtime ({}) from fstat for {:?}.", mtime
, self.full_path());
195 let entry
= PxarEntry
{
206 fn read_chattr(&self, fd
: RawFd
, entry
: &mut PxarEntry
) -> Result
<(), Error
> {
207 let mut attr
: usize = 0;
209 let res
= unsafe { fs::read_attr_fd(fd, &mut attr) }
;
210 if let Err(err
) = res
{
211 if let nix
::Error
::Sys(errno
) = err
{
212 if errno_is_unsupported(errno
) {
216 bail
!("read_attr_fd failed for {:?} - {}", self.full_path(), err
);
219 let flags
= flags
::feature_flags_from_chattr(attr
as u32);
220 entry
.flags
= entry
.flags
| flags
;
225 fn read_fat_attr(&self, fd
: RawFd
, magic
: i64, entry
: &mut PxarEntry
) -> Result
<(), Error
> {
226 use proxmox
::sys
::linux
::magic
::*;
228 if magic
!= MSDOS_SUPER_MAGIC
&& magic
!= FUSE_SUPER_MAGIC
{
232 let mut attr
: u32 = 0;
234 let res
= unsafe { fs::read_fat_attr_fd(fd, &mut attr) }
;
235 if let Err(err
) = res
{
236 if let nix
::Error
::Sys(errno
) = err
{
237 if errno_is_unsupported(errno
) {
241 bail
!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err
);
244 let flags
= flags
::feature_flags_from_fat_attr(attr
);
245 entry
.flags
= entry
.flags
| flags
;
250 /// True if all of the given feature flags are set in the Encoder, false otherwise
251 fn has_features(&self, feature_flags
: u64) -> bool
{
252 (self.feature_flags
& self.fs_feature_flags
& feature_flags
) == feature_flags
255 /// True if at least one of the given feature flags is set in the Encoder, false otherwise
256 fn has_some_features(&self, feature_flags
: u64) -> bool
{
257 (self.feature_flags
& self.fs_feature_flags
& feature_flags
) != 0
264 ) -> Result
<(Vec
<PxarXAttr
>, Option
<PxarFCaps
>), Error
> {
265 let mut xattrs
= Vec
::new();
266 let mut fcaps
= None
;
268 let flags
= flags
::WITH_XATTRS
| flags
::WITH_FCAPS
;
269 if !self.has_some_features(flags
) {
270 return Ok((xattrs
, fcaps
));
272 // Should never be called on symlinks, just in case check anyway
273 if is_symlink(&stat
) {
274 return Ok((xattrs
, fcaps
));
277 let xattr_names
= match xattr
::flistxattr(fd
) {
279 // Do not bail if the underlying endpoint does not supports xattrs
280 Err(Errno
::EOPNOTSUPP
) => return Ok((xattrs
, fcaps
)),
281 // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
282 Err(Errno
::EBADF
) => return Ok((xattrs
, fcaps
)),
283 Err(err
) => bail
!("read_xattrs failed for {:?} - {}", self.full_path(), err
),
286 for name
in xattr_names
.split(|c
| *c
== b'
\0'
) {
287 // Only extract the relevant extended attributes
288 if !xattr
::is_valid_xattr_name(&name
) {
292 let value
= match xattr
::fgetxattr(fd
, name
) {
294 // Vanished between flistattr and getxattr, this is ok, silently ignore
295 Err(Errno
::ENODATA
) => continue,
296 Err(err
) => bail
!("read_xattrs failed for {:?} - {}", self.full_path(), err
),
299 if xattr
::is_security_capability(&name
) {
300 if self.has_features(flags
::WITH_FCAPS
) {
301 // fcaps are stored in own format within the archive
302 fcaps
= Some(PxarFCaps { data: value }
);
304 } else if self.has_features(flags
::WITH_XATTRS
) {
305 xattrs
.push(PxarXAttr
{
320 acl_type
: acl
::ACLType
,
321 ) -> Result
<PxarACL
, Error
> {
329 if !self.has_features(flags
::WITH_ACL
) {
332 if is_symlink(&stat
) {
335 if acl_type
== acl
::ACL_TYPE_DEFAULT
&& !is_directory(&stat
) {
336 bail
!("ACL_TYPE_DEFAULT only defined for directories.");
339 // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
340 // to create a path for acl_get_file(). acl_get_fd() only allows to get
341 // ACL_TYPE_ACCESS attributes.
342 let proc_path
= Path
::new("/proc/self/fd/").join(fd
.to_string());
343 let acl
= match acl
::ACL
::get_file(&proc_path
, acl_type
) {
345 // Don't bail if underlying endpoint does not support acls
346 Err(Errno
::EOPNOTSUPP
) => return Ok(ret
),
347 // Don't bail if the endpoint cannot carry acls
348 Err(Errno
::EBADF
) => return Ok(ret
),
349 // Don't bail if there is no data
350 Err(Errno
::ENODATA
) => return Ok(ret
),
351 Err(err
) => bail
!("error while reading ACL - {}", err
),
354 self.process_acl(acl
, acl_type
)
357 fn process_acl(&self, acl
: acl
::ACL
, acl_type
: acl
::ACLType
) -> Result
<PxarACL
, Error
> {
358 let mut acl_user
= Vec
::new();
359 let mut acl_group
= Vec
::new();
360 let mut acl_group_obj
= None
;
361 let mut acl_default
= None
;
362 let mut user_obj_permissions
= None
;
363 let mut group_obj_permissions
= None
;
364 let mut other_permissions
= None
;
365 let mut mask_permissions
= None
;
367 for entry
in &mut acl
.entries() {
368 let tag
= entry
.get_tag_type()?
;
369 let permissions
= entry
.get_permissions()?
;
371 acl
::ACL_USER_OBJ
=> user_obj_permissions
= Some(permissions
),
372 acl
::ACL_GROUP_OBJ
=> group_obj_permissions
= Some(permissions
),
373 acl
::ACL_OTHER
=> other_permissions
= Some(permissions
),
374 acl
::ACL_MASK
=> mask_permissions
= Some(permissions
),
376 acl_user
.push(PxarACLUser
{
377 uid
: entry
.get_qualifier()?
,
382 acl_group
.push(PxarACLGroup
{
383 gid
: entry
.get_qualifier()?
,
387 _
=> bail
!("Unexpected ACL tag encountered!"),
395 acl
::ACL_TYPE_ACCESS
=> {
396 // The mask permissions are mapped to the stat group permissions
397 // in case that the ACL group permissions were set.
398 // Only in that case we need to store the group permissions,
399 // in the other cases they are identical to the stat group permissions.
400 if let (Some(gop
), Some(_
)) = (group_obj_permissions
, mask_permissions
) {
401 acl_group_obj
= Some(PxarACLGroupObj { permissions: gop }
);
404 acl
::ACL_TYPE_DEFAULT
=> {
405 if user_obj_permissions
!= None
406 || group_obj_permissions
!= None
407 || other_permissions
!= None
408 || mask_permissions
!= None
410 acl_default
= Some(PxarACLDefault
{
411 // The value is set to UINT64_MAX as placeholder if one
412 // of the permissions is not set
413 user_obj_permissions
: user_obj_permissions
.unwrap_or(std
::u64::MAX
),
414 group_obj_permissions
: group_obj_permissions
.unwrap_or(std
::u64::MAX
),
415 other_permissions
: other_permissions
.unwrap_or(std
::u64::MAX
),
416 mask_permissions
: mask_permissions
.unwrap_or(std
::u64::MAX
),
420 _
=> bail
!("Unexpected ACL type encountered"),
426 group_obj
: acl_group_obj
,
427 default: acl_default
,
431 /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
432 fn read_quota_project_id(
437 ) -> Result
<Option
<PxarQuotaProjID
>, Error
> {
438 if !(is_directory(&stat
) || is_reg_file(&stat
)) {
441 if !self.has_features(flags
::WITH_QUOTA_PROJID
) {
445 use proxmox
::sys
::linux
::magic
::*;
448 EXT4_SUPER_MAGIC
| XFS_SUPER_MAGIC
| FUSE_SUPER_MAGIC
| ZFS_SUPER_MAGIC
=> {
449 let mut fsxattr
= fs
::FSXAttr
::default();
450 let res
= unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) }
;
452 // On some FUSE filesystems it can happen that ioctl is not supported.
453 // For these cases projid is set to 0 while the error is ignored.
454 if let Err(err
) = res
{
455 let errno
= err
.as_errno().ok_or_else(|| {
457 "error while reading quota project id for {:#?}",
461 if errno_is_unsupported(errno
) {
465 "error while reading quota project id for {:#?} - {}",
472 let projid
= fsxattr
.fsx_projid
as u64;
476 Ok(Some(PxarQuotaProjID { projid }
))
483 fn write_entry(&mut self, entry
: PxarEntry
) -> Result
<(), Error
> {
484 self.write_header(PXAR_ENTRY
, std
::mem
::size_of
::<PxarEntry
>() as u64)?
;
485 self.write_item(entry
)?
;
490 fn write_xattr(&mut self, xattr
: PxarXAttr
) -> Result
<(), Error
> {
491 let size
= xattr
.name
.len() + xattr
.value
.len() + 1; // +1 for '\0' separating name and value
492 self.write_header(PXAR_XATTR
, size
as u64)?
;
493 self.write(xattr
.name
.as_slice())?
;
495 self.write(xattr
.value
.as_slice())?
;
500 fn write_fcaps(&mut self, fcaps
: Option
<PxarFCaps
>) -> Result
<(), Error
> {
501 if let Some(fcaps
) = fcaps
{
502 let size
= fcaps
.data
.len();
503 self.write_header(PXAR_FCAPS
, size
as u64)?
;
504 self.write(fcaps
.data
.as_slice())?
;
510 fn write_acl_user(&mut self, acl_user
: PxarACLUser
) -> Result
<(), Error
> {
511 self.write_header(PXAR_ACL_USER
, std
::mem
::size_of
::<PxarACLUser
>() as u64)?
;
512 self.write_item(acl_user
)?
;
517 fn write_acl_group(&mut self, acl_group
: PxarACLGroup
) -> Result
<(), Error
> {
518 self.write_header(PXAR_ACL_GROUP
, std
::mem
::size_of
::<PxarACLGroup
>() as u64)?
;
519 self.write_item(acl_group
)?
;
524 fn write_acl_group_obj(&mut self, acl_group_obj
: PxarACLGroupObj
) -> Result
<(), Error
> {
527 std
::mem
::size_of
::<PxarACLGroupObj
>() as u64,
529 self.write_item(acl_group_obj
)?
;
534 fn write_acl_default(&mut self, acl_default
: PxarACLDefault
) -> Result
<(), Error
> {
537 std
::mem
::size_of
::<PxarACLDefault
>() as u64,
539 self.write_item(acl_default
)?
;
544 fn write_acl_default_user(&mut self, acl_default_user
: PxarACLUser
) -> Result
<(), Error
> {
546 PXAR_ACL_DEFAULT_USER
,
547 std
::mem
::size_of
::<PxarACLUser
>() as u64,
549 self.write_item(acl_default_user
)?
;
554 fn write_acl_default_group(&mut self, acl_default_group
: PxarACLGroup
) -> Result
<(), Error
> {
556 PXAR_ACL_DEFAULT_GROUP
,
557 std
::mem
::size_of
::<PxarACLGroup
>() as u64,
559 self.write_item(acl_default_group
)?
;
564 fn write_quota_project_id(&mut self, projid
: PxarQuotaProjID
) -> Result
<(), Error
> {
567 std
::mem
::size_of
::<PxarQuotaProjID
>() as u64,
569 self.write_item(projid
)?
;
574 fn write_goodbye_table(
576 goodbye_offset
: usize,
577 goodbye_items
: &mut [PxarGoodbyeItem
],
578 ) -> Result
<(), Error
> {
579 goodbye_items
.sort_unstable_by(|a
, b
| a
.hash
.cmp(&b
.hash
));
581 let item_count
= goodbye_items
.len();
583 let goodbye_table_size
= (item_count
+ 1) * std
::mem
::size_of
::<PxarGoodbyeItem
>();
585 self.write_header(PXAR_GOODBYE
, goodbye_table_size
as u64)?
;
587 if self.file_copy_buffer
.len() < goodbye_table_size
{
588 let need
= goodbye_table_size
- self.file_copy_buffer
.len();
589 self.file_copy_buffer
.reserve(need
);
591 self.file_copy_buffer
592 .set_len(self.file_copy_buffer
.capacity());
596 let buffer
= &mut self.file_copy_buffer
;
598 copy_binary_search_tree(item_count
, |s
, d
| {
599 let item
= &goodbye_items
[s
];
600 let offset
= d
* std
::mem
::size_of
::<PxarGoodbyeItem
>();
602 crate::tools
::map_struct_mut
::<PxarGoodbyeItem
>(&mut buffer
[offset
..]).unwrap();
603 dest
.offset
= u64::to_le(item
.offset
);
604 dest
.size
= u64::to_le(item
.size
);
605 dest
.hash
= u64::to_le(item
.hash
);
608 // append PxarGoodbyeTail as last item
609 let offset
= item_count
* std
::mem
::size_of
::<PxarGoodbyeItem
>();
610 let dest
= crate::tools
::map_struct_mut
::<PxarGoodbyeItem
>(&mut buffer
[offset
..]).unwrap();
611 dest
.offset
= u64::to_le(goodbye_offset
as u64);
612 dest
.size
= u64::to_le((goodbye_table_size
+ std
::mem
::size_of
::<PxarHeader
>()) as u64);
613 dest
.hash
= u64::to_le(PXAR_GOODBYE_TAIL_MARKER
);
615 self.flush_copy_buffer(goodbye_table_size
)?
;
622 dir
: &mut nix
::dir
::Dir
,
625 match_pattern
: Vec
<MatchPattern
>,
626 ) -> Result
<(), Error
> {
627 //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
629 let mut name_list
= vec
![];
631 let rawfd
= dir
.as_raw_fd();
633 let dir_start_pos
= self.writer_pos
;
635 let is_root
= dir_start_pos
== 0;
637 let mut dir_entry
= self.create_entry(&dir_stat
)?
;
639 self.read_chattr(rawfd
, &mut dir_entry
)?
;
640 self.read_fat_attr(rawfd
, magic
, &mut dir_entry
)?
;
642 // for each node in the directory tree, the filesystem features are
643 // checked based on the fs magic number.
644 self.fs_feature_flags
= flags
::feature_flags_from_magic(magic
);
646 let (xattrs
, fcaps
) = self.read_xattrs(rawfd
, &dir_stat
)?
;
647 let acl_access
= self.read_acl(rawfd
, &dir_stat
, acl
::ACL_TYPE_ACCESS
)?
;
648 let acl_default
= self.read_acl(rawfd
, &dir_stat
, acl
::ACL_TYPE_DEFAULT
)?
;
649 let projid
= self.read_quota_project_id(rawfd
, magic
, &dir_stat
)?
;
651 self.write_entry(dir_entry
)?
;
652 for xattr
in xattrs
{
653 self.write_xattr(xattr
)?
;
655 self.write_fcaps(fcaps
)?
;
657 for user
in acl_access
.users
{
658 self.write_acl_user(user
)?
;
660 for group
in acl_access
.groups
{
661 self.write_acl_group(group
)?
;
663 if let Some(group_obj
) = acl_access
.group_obj
{
664 self.write_acl_group_obj(group_obj
)?
;
667 for default_user
in acl_default
.users
{
668 self.write_acl_default_user(default_user
)?
;
670 for default_group
in acl_default
.groups
{
671 self.write_acl_default_group(default_group
)?
;
673 if let Some(default) = acl_default
.default {
674 self.write_acl_default(default)?
;
676 if let Some(projid
) = projid
{
677 self.write_quota_project_id(projid
)?
;
680 let include_children
;
681 if is_virtual_file_system(magic
) {
682 include_children
= false;
683 } else if let Some(set
) = &self.device_set
{
684 include_children
= set
.contains(&dir_stat
.st_dev
);
686 include_children
= true;
689 // Expand the exclude match pattern inherited from the parent by local entries, if present
690 let mut local_match_pattern
= match_pattern
.clone();
691 let pxar_exclude
= match MatchPattern
::from_file(rawfd
, ".pxarexclude") {
692 Ok(Some((mut excludes
, buffer
, stat
))) => {
693 local_match_pattern
.append(&mut excludes
);
697 Err(err
) => bail
!("error while reading exclude file - {}", err
),
700 if include_children
{
701 // Exclude patterns passed via the CLI are stored as '.pxarexclude-cli'
702 // in the root directory of the archive.
703 if is_root
&& match_pattern
.len() > 0 {
704 let filename
= CString
::new(".pxarexclude-cli")?
;
705 name_list
.push((filename
, dir_stat
.clone(), match_pattern
.clone()));
708 for entry
in dir
.iter() {
710 .map_err(|err
| format_err
!("readir {:?} failed - {}", self.full_path(), err
))?
;
711 let filename
= entry
.file_name().to_owned();
713 let name
= filename
.to_bytes_with_nul();
714 if name
== b
".\0" || name
== b
"..\0" {
717 // Do not store a ".pxarexclude-cli" file found in the archive root,
718 // as this would confilict with new cli passed exclude patterns,
720 if is_root
&& name
== b
".pxarexclude-cli\0" {
721 eprintln
!("skip existing '.pxarexclude-cli' in archive root.");
725 let stat
= match nix
::sys
::stat
::fstatat(
728 nix
::fcntl
::AtFlags
::AT_SYMLINK_NOFOLLOW
,
731 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
732 let filename_osstr
= std
::ffi
::OsStr
::from_bytes(filename
.to_bytes());
733 self.report_vanished_file(&self.full_path().join(filename_osstr
))?
;
736 Err(err
) => bail
!("fstat {:?} failed - {}", self.full_path(), err
),
739 match match_filename(&filename
, &stat
, &local_match_pattern
)?
{
740 (MatchType
::Positive
, _
) => {
741 let filename_osstr
= std
::ffi
::OsStr
::from_bytes(filename
.to_bytes());
743 "matched by .pxarexclude entry - skipping: {:?}",
744 self.full_path().join(filename_osstr
)
747 (_
, child_pattern
) => name_list
.push((filename
, stat
, child_pattern
)),
750 if name_list
.len() > MAX_DIRECTORY_ENTRIES
{
752 "too many directory items in {:?} (> {})",
754 MAX_DIRECTORY_ENTRIES
759 eprintln
!("skip mount point: {:?}", self.full_path());
762 name_list
.sort_unstable_by(|a
, b
| a
.0.cmp(&b
.0));
764 let mut goodbye_items
= vec
![];
766 for (filename
, stat
, exclude_list
) in name_list
{
767 let start_pos
= self.writer_pos
;
769 if filename
.as_bytes() == b
".pxarexclude" {
770 if let Some((ref content
, ref stat
)) = pxar_exclude
{
771 let filefd
= match nix
::fcntl
::openat(
777 Ok(filefd
) => filefd
,
778 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
779 self.report_vanished_file(&self.full_path())?
;
783 let filename_osstr
= std
::ffi
::OsStr
::from_bytes(filename
.to_bytes());
785 "open file {:?} failed - {}",
786 self.full_path().join(filename_osstr
),
792 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
793 detect_fs_type(filefd
)?
798 self.write_filename(&filename
)?
;
799 if let Some(ref mut catalog
) = self.catalog
{
800 catalog
.add_file(&filename
, stat
.st_size
as u64, stat
.st_mtime
as u64)?
;
802 self.encode_pxar_exclude(filefd
, stat
, child_magic
, content
)?
;
807 if is_root
&& filename
.as_bytes() == b
".pxarexclude-cli" {
808 // '.pxarexclude-cli' is used to store the exclude MatchPatterns
809 // passed via the cli in the root directory of the archive.
810 self.write_filename(&filename
)?
;
811 let content
= MatchPattern
::to_bytes(&exclude_list
);
812 if let Some(ref mut catalog
) = self.catalog
{
813 catalog
.add_file(&filename
, content
.len() as u64, 0)?
;
815 self.encode_pxar_exclude_cli(stat
.st_uid
, stat
.st_gid
, 0, &content
)?
;
820 .push(std
::ffi
::OsStr
::from_bytes(filename
.as_bytes()));
823 println
!("{:?}", self.full_path());
826 if is_directory(&stat
) {
827 let mut dir
= match nix
::dir
::Dir
::openat(
830 OFlag
::O_DIRECTORY
| OFlag
::O_NOFOLLOW
,
834 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
835 self.report_vanished_file(&self.full_path())?
;
838 Err(err
) => bail
!("open dir {:?} failed - {}", self.full_path(), err
),
841 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
842 detect_fs_type(dir
.as_raw_fd())?
847 self.write_filename(&filename
)?
;
848 if let Some(ref mut catalog
) = self.catalog
{
849 catalog
.start_directory(&filename
)?
;
851 self.encode_dir(&mut dir
, &stat
, child_magic
, exclude_list
)?
;
852 if let Some(ref mut catalog
) = self.catalog
{
853 catalog
.end_directory()?
;
855 } else if is_reg_file(&stat
) {
856 let mut hardlink_target
= None
;
858 if stat
.st_nlink
> 1 {
859 let link_info
= HardLinkInfo
{
863 hardlink_target
= self.hardlinks
.get(&link_info
).map(|(v
, offset
)| {
864 let mut target
= v
.clone().into_os_string();
865 target
.push("\0"); // add Nul byte
866 (target
, (start_pos
as u64) - offset
)
868 if hardlink_target
== None
{
870 .insert(link_info
, (self.relative_path
.clone(), start_pos
as u64));
874 if let Some((target
, offset
)) = hardlink_target
{
875 if let Some(ref mut catalog
) = self.catalog
{
876 catalog
.add_hardlink(&filename
)?
;
878 self.write_filename(&filename
)?
;
879 self.encode_hardlink(target
.as_bytes(), offset
)?
;
881 let filefd
= match nix
::fcntl
::openat(
887 Ok(filefd
) => filefd
,
888 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
889 self.report_vanished_file(&self.full_path())?
;
892 Err(err
) => bail
!("open file {:?} failed - {}", self.full_path(), err
),
895 if let Some(ref mut catalog
) = self.catalog
{
896 catalog
.add_file(&filename
, stat
.st_size
as u64, stat
.st_mtime
as u64)?
;
898 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
899 detect_fs_type(filefd
)?
904 self.write_filename(&filename
)?
;
905 let res
= self.encode_file(filefd
, &stat
, child_magic
);
906 let _
= nix
::unistd
::close(filefd
); // ignore close errors
909 } else if is_symlink(&stat
) {
910 let mut buffer
= vec
::undefined(libc
::PATH_MAX
as usize);
912 let res
= filename
.with_nix_path(|cstr
| unsafe {
916 buffer
.as_mut_ptr() as *mut libc
::c_char
,
921 match Errno
::result(res
) {
923 if let Some(ref mut catalog
) = self.catalog
{
924 catalog
.add_symlink(&filename
)?
;
926 buffer
[len
as usize] = 0u8; // add Nul byte
927 self.write_filename(&filename
)?
;
928 self.encode_symlink(&buffer
[..((len
+ 1) as usize)], &stat
)?
930 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
931 self.report_vanished_file(&self.full_path())?
;
934 Err(err
) => bail
!("readlink {:?} failed - {}", self.full_path(), err
),
936 } else if is_block_dev(&stat
) || is_char_dev(&stat
) {
937 if self.has_features(flags
::WITH_DEVICE_NODES
) {
938 if let Some(ref mut catalog
) = self.catalog
{
939 if is_block_dev(&stat
) {
940 catalog
.add_block_device(&filename
)?
;
942 catalog
.add_char_device(&filename
)?
;
945 self.write_filename(&filename
)?
;
946 self.encode_device(&stat
)?
;
948 eprintln
!("skip device node: {:?}", self.full_path());
950 } else if is_fifo(&stat
) {
951 if self.has_features(flags
::WITH_FIFOS
) {
952 if let Some(ref mut catalog
) = self.catalog
{
953 catalog
.add_fifo(&filename
)?
;
955 self.write_filename(&filename
)?
;
956 self.encode_special(&stat
)?
;
958 eprintln
!("skip fifo: {:?}", self.full_path());
960 } else if is_socket(&stat
) {
961 if self.has_features(flags
::WITH_SOCKETS
) {
962 if let Some(ref mut catalog
) = self.catalog
{
963 catalog
.add_socket(&filename
)?
;
965 self.write_filename(&filename
)?
;
966 self.encode_special(&stat
)?
;
968 eprintln
!("skip socket: {:?}", self.full_path());
972 "unsupported file type (mode {:o} {:?})",
978 let end_pos
= self.writer_pos
;
980 goodbye_items
.push(PxarGoodbyeItem
{
981 offset
: start_pos
as u64,
982 size
: (end_pos
- start_pos
) as u64,
983 hash
: compute_goodbye_hash(filename
.to_bytes()),
986 self.relative_path
.pop();
989 //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
991 // fixup goodby item offsets
992 let goodbye_start
= self.writer_pos
as u64;
993 for item
in &mut goodbye_items
{
994 item
.offset
= goodbye_start
- item
.offset
;
997 let goodbye_offset
= self.writer_pos
- dir_start_pos
;
999 self.write_goodbye_table(goodbye_offset
, &mut goodbye_items
)?
;
1001 //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
1005 fn encode_file(&mut self, filefd
: RawFd
, stat
: &FileStat
, magic
: i64) -> Result
<(), Error
> {
1006 //println!("encode_file: {:?}", self.full_path());
1008 let mut entry
= self.create_entry(&stat
)?
;
1010 self.read_chattr(filefd
, &mut entry
)?
;
1011 self.read_fat_attr(filefd
, magic
, &mut entry
)?
;
1012 let (xattrs
, fcaps
) = self.read_xattrs(filefd
, &stat
)?
;
1013 let acl_access
= self.read_acl(filefd
, &stat
, acl
::ACL_TYPE_ACCESS
)?
;
1014 let projid
= self.read_quota_project_id(filefd
, magic
, &stat
)?
;
1016 self.write_entry(entry
)?
;
1017 for xattr
in xattrs
{
1018 self.write_xattr(xattr
)?
;
1020 self.write_fcaps(fcaps
)?
;
1021 for user
in acl_access
.users
{
1022 self.write_acl_user(user
)?
;
1024 for group
in acl_access
.groups
{
1025 self.write_acl_group(group
)?
;
1027 if let Some(group_obj
) = acl_access
.group_obj
{
1028 self.write_acl_group_obj(group_obj
)?
;
1030 if let Some(projid
) = projid
{
1031 self.write_quota_project_id(projid
)?
;
1034 let include_payload
;
1035 if is_virtual_file_system(magic
) {
1036 include_payload
= false;
1037 } else if let Some(ref set
) = &self.device_set
{
1038 include_payload
= set
.contains(&stat
.st_dev
);
1040 include_payload
= true;
1043 if !include_payload
{
1044 eprintln
!("skip content: {:?}", self.full_path());
1045 self.write_header(PXAR_PAYLOAD
, 0)?
;
1049 let size
= stat
.st_size
as u64;
1051 self.write_header(PXAR_PAYLOAD
, size
)?
;
1053 let mut pos
: u64 = 0;
1055 let n
= match nix
::unistd
::read(filefd
, &mut self.file_copy_buffer
) {
1057 Err(nix
::Error
::Sys(Errno
::EINTR
)) => continue, /* try again */
1058 Err(err
) => bail
!("read {:?} failed - {}", self.full_path(), err
),
1062 // Note:: casync format cannot handle that
1064 "detected shrinked file {:?} ({} < {})",
1073 let mut next
= pos
+ (n
as u64);
1079 let count
= (next
- pos
) as usize;
1081 self.flush_copy_buffer(count
)?
;
1093 fn encode_device(&mut self, stat
: &FileStat
) -> Result
<(), Error
> {
1094 let entry
= self.create_entry(&stat
)?
;
1096 self.write_entry(entry
)?
;
1098 let major
= unsafe { libc::major(stat.st_rdev) }
as u64;
1099 let minor
= unsafe { libc::minor(stat.st_rdev) }
as u64;
1101 //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
1103 self.write_header(PXAR_DEVICE
, std
::mem
::size_of
::<PxarDevice
>() as u64)?
;
1104 self.write_item(PxarDevice { major, minor }
)?
;
1110 fn encode_special(&mut self, stat
: &FileStat
) -> Result
<(), Error
> {
1111 let entry
= self.create_entry(&stat
)?
;
1113 self.write_entry(entry
)?
;
1118 fn encode_symlink(&mut self, target
: &[u8], stat
: &FileStat
) -> Result
<(), Error
> {
1119 //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
1121 let entry
= self.create_entry(&stat
)?
;
1122 self.write_entry(entry
)?
;
1124 self.write_header(PXAR_SYMLINK
, target
.len() as u64)?
;
1125 self.write(target
)?
;
1130 fn encode_hardlink(&mut self, target
: &[u8], offset
: u64) -> Result
<(), Error
> {
1131 //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
1133 // Note: HARDLINK replaces an ENTRY.
1134 self.write_header(PXAR_FORMAT_HARDLINK
, (target
.len() as u64) + 8)?
;
1135 self.write_item(offset
)?
;
1136 self.write(target
)?
;
1141 fn encode_pxar_exclude(
1147 ) -> Result
<(), Error
> {
1148 let mut entry
= self.create_entry(&stat
)?
;
1150 self.read_chattr(filefd
, &mut entry
)?
;
1151 self.read_fat_attr(filefd
, magic
, &mut entry
)?
;
1152 let (xattrs
, fcaps
) = self.read_xattrs(filefd
, &stat
)?
;
1153 let acl_access
= self.read_acl(filefd
, &stat
, acl
::ACL_TYPE_ACCESS
)?
;
1154 let projid
= self.read_quota_project_id(filefd
, magic
, &stat
)?
;
1156 self.write_entry(entry
)?
;
1157 for xattr
in xattrs
{
1158 self.write_xattr(xattr
)?
;
1160 self.write_fcaps(fcaps
)?
;
1161 for user
in acl_access
.users
{
1162 self.write_acl_user(user
)?
;
1164 for group
in acl_access
.groups
{
1165 self.write_acl_group(group
)?
;
1167 if let Some(group_obj
) = acl_access
.group_obj
{
1168 self.write_acl_group_obj(group_obj
)?
;
1170 if let Some(projid
) = projid
{
1171 self.write_quota_project_id(projid
)?
;
1174 let include_payload
;
1175 if is_virtual_file_system(magic
) {
1176 include_payload
= false;
1177 } else if let Some(set
) = &self.device_set
{
1178 include_payload
= set
.contains(&stat
.st_dev
);
1180 include_payload
= true;
1183 if !include_payload
{
1184 eprintln
!("skip content: {:?}", self.full_path());
1185 self.write_header(PXAR_PAYLOAD
, 0)?
;
1189 let size
= content
.len();
1190 self.write_header(PXAR_PAYLOAD
, size
as u64)?
;
1191 self.writer
.write_all(content
)?
;
1192 self.writer_pos
+= size
;
1197 /// Encodes the excude match patterns passed via cli as file in the archive.
1198 fn encode_pxar_exclude_cli(
1204 ) -> Result
<(), Error
> {
1205 let entry
= PxarEntry
{
1206 mode
: (libc
::S_IFREG
| 0o600) as u64,
1212 self.write_entry(entry
)?
;
1213 let size
= content
.len();
1214 self.write_header(PXAR_PAYLOAD
, size
as u64)?
;
1215 self.writer
.write_all(content
)?
;
1216 self.writer_pos
+= size
;
1221 // the report_XXX method may raise and error - depending on encoder configuration
1223 fn report_vanished_file(&self, path
: &Path
) -> Result
<(), Error
> {
1224 eprintln
!("WARNING: detected vanished file {:?}", path
);
1230 // If there is a match, an updated MatchPattern list to pass to the matched child is returned.
1234 match_pattern
: &Vec
<MatchPattern
>,
1235 ) -> Result
<(MatchType
, Vec
<MatchPattern
>), Error
> {
1236 let mut child_pattern
= Vec
::new();
1237 let mut match_state
= MatchType
::None
;
1239 for pattern
in match_pattern
{
1240 match pattern
.matches_filename(filename
, is_directory(&stat
))?
{
1241 MatchType
::None
=> {}
1242 MatchType
::Positive
=> match_state
= MatchType
::Positive
,
1243 MatchType
::Negative
=> match_state
= MatchType
::Negative
,
1245 if match_state
!= MatchType
::Positive
&& match_state
!= MatchType
::Negative
{
1246 match_state
= match_type
;
1248 child_pattern
.push(pattern
.get_rest_pattern());
1253 Ok((match_state
, child_pattern
))
1256 fn errno_is_unsupported(errno
: Errno
) -> bool
{
1258 Errno
::ENOTTY
| Errno
::ENOSYS
| Errno
::EBADF
| Errno
::EOPNOTSUPP
| Errno
::EINVAL
=> true,
1263 fn detect_fs_type(fd
: RawFd
) -> Result
<i64, Error
> {
1264 let mut fs_stat
= std
::mem
::MaybeUninit
::uninit();
1265 let res
= unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) }
;
1266 Errno
::result(res
)?
;
1267 let fs_stat
= unsafe { fs_stat.assume_init() }
;
1273 pub fn is_temporary_file_system(magic
: i64) -> bool
{
1274 use proxmox
::sys
::linux
::magic
::*;
1275 magic
== RAMFS_MAGIC
|| magic
== TMPFS_MAGIC
1278 pub fn is_virtual_file_system(magic
: i64) -> bool
{
1279 use proxmox
::sys
::linux
::magic
::*;
1283 CGROUP2_SUPER_MAGIC
|
1284 CGROUP_SUPER_MAGIC
|
1287 DEVPTS_SUPER_MAGIC
|
1289 FUSE_CTL_SUPER_MAGIC
|
1299 SYSFS_MAGIC
=> true,