1 //! *pxar* format encoder.
3 //! This module contain the code to generate *pxar* archive files.
4 use std
::collections
::{HashMap, HashSet}
;
7 use std
::os
::unix
::ffi
::OsStrExt
;
8 use std
::os
::unix
::io
::AsRawFd
;
9 use std
::os
::unix
::io
::RawFd
;
10 use std
::path
::{Path, PathBuf}
;
12 use endian_trait
::Endian
;
14 use nix
::errno
::Errno
;
15 use nix
::fcntl
::OFlag
;
16 use nix
::sys
::stat
::FileStat
;
17 use nix
::sys
::stat
::Mode
;
20 use proxmox
::tools
::vec
;
22 use super::binary_search_tree
::*;
23 use super::catalog
::BackupCatalogWriter
;
25 use super::format_definition
::*;
27 use super::match_pattern
::{MatchPattern, MatchType}
;
28 use crate::tools
::acl
;
30 use crate::tools
::xattr
;
32 /// The format requires to build sorted directory lookup tables in
33 /// memory, so we restrict the number of allowed entries to limit
34 /// maximum memory usage.
35 pub const MAX_DIRECTORY_ENTRIES
: usize = 256 * 1024;
37 #[derive(Eq, PartialEq, Hash)]
43 pub struct Encoder
<'a
, W
: Write
, C
: BackupCatalogWriter
> {
45 relative_path
: PathBuf
,
48 catalog
: Option
<&'a
mut C
>,
50 file_copy_buffer
: Vec
<u8>,
51 device_set
: Option
<HashSet
<u64>>,
53 // Flags set by the user
55 // Flags signaling features supported by the filesystem
56 fs_feature_flags
: u64,
57 hardlinks
: HashMap
<HardLinkInfo
, (PathBuf
, u64)>,
60 impl<'a
, W
: Write
, C
: BackupCatalogWriter
> Encoder
<'a
, W
, C
> {
61 // used for error reporting
62 fn full_path(&self) -> PathBuf
{
63 self.base_path
.join(&self.relative_path
)
66 /// Create archive, write result data to ``writer``.
68 /// The ``device_set`` can be use used to limit included mount points.
70 /// - ``None``: include all mount points
71 /// - ``Some(set)``: only include devices listed in this set (the
72 /// root path device is automathically added to this list, so
73 /// you can pass an empty set if you want to archive a single
77 dir
: &mut nix
::dir
::Dir
,
79 catalog
: Option
<&'a
mut C
>,
80 device_set
: Option
<HashSet
<u64>>,
82 skip_lost_and_found
: bool
, // fixme: should be a feature flag ??
84 ) -> Result
<(), Error
> {
85 const FILE_COPY_BUFFER_SIZE
: usize = 1024 * 1024;
87 let mut file_copy_buffer
= Vec
::with_capacity(FILE_COPY_BUFFER_SIZE
);
89 file_copy_buffer
.set_len(FILE_COPY_BUFFER_SIZE
);
92 // todo: use scandirat??
94 let dir_fd
= dir
.as_raw_fd();
95 let stat
= nix
::sys
::stat
::fstat(dir_fd
)
96 .map_err(|err
| format_err
!("fstat {:?} failed - {}", path
, err
))?
;
98 if !is_directory(&stat
) {
99 bail
!("got unexpected file type {:?} (not a directory)", path
);
102 let mut device_set
= device_set
.clone();
103 if let Some(ref mut set
) = device_set
{
104 set
.insert(stat
.st_dev
);
107 let magic
= detect_fs_type(dir_fd
)?
;
109 if is_virtual_file_system(magic
) {
110 bail
!("backup virtual file systems is disabled!");
113 let fs_feature_flags
= flags
::feature_flags_from_magic(magic
);
117 relative_path
: PathBuf
::new(),
127 hardlinks
: HashMap
::new(),
131 println
!("{:?}", me
.full_path());
134 let mut excludes
= Vec
::new();
135 if skip_lost_and_found
{
136 excludes
.push(MatchPattern
::from_line(b
"**/lost+found").unwrap().unwrap());
138 me
.encode_dir(dir
, &stat
, magic
, excludes
)?
;
143 fn write(&mut self, buf
: &[u8]) -> Result
<(), Error
> {
144 self.writer
.write_all(buf
)?
;
145 self.writer_pos
+= buf
.len();
149 fn write_item
<T
: Endian
>(&mut self, item
: T
) -> Result
<(), Error
> {
150 let data
= item
.to_le();
152 let buffer
= unsafe {
153 std
::slice
::from_raw_parts(&data
as *const T
as *const u8, std
::mem
::size_of
::<T
>())
161 fn flush_copy_buffer(&mut self, size
: usize) -> Result
<(), Error
> {
162 self.writer
.write_all(&self.file_copy_buffer
[..size
])?
;
163 self.writer_pos
+= size
;
167 fn write_header(&mut self, htype
: u64, size
: u64) -> Result
<(), Error
> {
168 let size
= size
+ (std
::mem
::size_of
::<PxarHeader
>() as u64);
169 self.write_item(PxarHeader { size, htype }
)?
;
174 fn write_filename(&mut self, name
: &CStr
) -> Result
<(), Error
> {
175 let buffer
= name
.to_bytes_with_nul();
176 self.write_header(PXAR_FILENAME
, buffer
.len() as u64)?
;
182 fn create_entry(&self, stat
: &FileStat
) -> Result
<PxarEntry
, Error
> {
183 let mode
= if is_symlink(&stat
) {
184 (libc
::S_IFLNK
| 0o777) as u64
186 (stat
.st_mode
& (libc
::S_IFMT
| 0o7777)) as u64
189 let mtime
= stat
.st_mtime
* 1_000_000_000 + stat
.st_mtime_nsec
;
191 bail
!("got strange mtime ({}) from fstat for {:?}.", mtime
, self.full_path());
194 let entry
= PxarEntry
{
205 fn read_chattr(&self, fd
: RawFd
, entry
: &mut PxarEntry
) -> Result
<(), Error
> {
206 let mut attr
: usize = 0;
208 let res
= unsafe { fs::read_attr_fd(fd, &mut attr) }
;
209 if let Err(err
) = res
{
210 if let nix
::Error
::Sys(errno
) = err
{
211 if errno_is_unsupported(errno
) {
215 bail
!("read_attr_fd failed for {:?} - {}", self.full_path(), err
);
218 let flags
= flags
::feature_flags_from_chattr(attr
as u32);
219 entry
.flags
= entry
.flags
| flags
;
224 fn read_fat_attr(&self, fd
: RawFd
, magic
: i64, entry
: &mut PxarEntry
) -> Result
<(), Error
> {
225 use proxmox
::sys
::linux
::magic
::*;
227 if magic
!= MSDOS_SUPER_MAGIC
&& magic
!= FUSE_SUPER_MAGIC
{
231 let mut attr
: u32 = 0;
233 let res
= unsafe { fs::read_fat_attr_fd(fd, &mut attr) }
;
234 if let Err(err
) = res
{
235 if let nix
::Error
::Sys(errno
) = err
{
236 if errno_is_unsupported(errno
) {
240 bail
!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err
);
243 let flags
= flags
::feature_flags_from_fat_attr(attr
);
244 entry
.flags
= entry
.flags
| flags
;
249 /// True if all of the given feature flags are set in the Encoder, false otherwise
250 fn has_features(&self, feature_flags
: u64) -> bool
{
251 (self.feature_flags
& self.fs_feature_flags
& feature_flags
) == feature_flags
254 /// True if at least one of the given feature flags is set in the Encoder, false otherwise
255 fn has_some_features(&self, feature_flags
: u64) -> bool
{
256 (self.feature_flags
& self.fs_feature_flags
& feature_flags
) != 0
263 ) -> Result
<(Vec
<PxarXAttr
>, Option
<PxarFCaps
>), Error
> {
264 let mut xattrs
= Vec
::new();
265 let mut fcaps
= None
;
267 let flags
= flags
::WITH_XATTRS
| flags
::WITH_FCAPS
;
268 if !self.has_some_features(flags
) {
269 return Ok((xattrs
, fcaps
));
271 // Should never be called on symlinks, just in case check anyway
272 if is_symlink(&stat
) {
273 return Ok((xattrs
, fcaps
));
276 let xattr_names
= match xattr
::flistxattr(fd
) {
278 // Do not bail if the underlying endpoint does not supports xattrs
279 Err(Errno
::EOPNOTSUPP
) => return Ok((xattrs
, fcaps
)),
280 // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
281 Err(Errno
::EBADF
) => return Ok((xattrs
, fcaps
)),
282 Err(err
) => bail
!("read_xattrs failed for {:?} - {}", self.full_path(), err
),
285 for name
in xattr_names
.split(|c
| *c
== b'
\0'
) {
286 // Only extract the relevant extended attributes
287 if !xattr
::is_valid_xattr_name(&name
) {
291 let value
= match xattr
::fgetxattr(fd
, name
) {
293 // Vanished between flistattr and getxattr, this is ok, silently ignore
294 Err(Errno
::ENODATA
) => continue,
295 Err(err
) => bail
!("read_xattrs failed for {:?} - {}", self.full_path(), err
),
298 if xattr
::is_security_capability(&name
) {
299 if self.has_features(flags
::WITH_FCAPS
) {
300 // fcaps are stored in own format within the archive
301 fcaps
= Some(PxarFCaps { data: value }
);
303 } else if self.has_features(flags
::WITH_XATTRS
) {
304 xattrs
.push(PxarXAttr
{
319 acl_type
: acl
::ACLType
,
320 ) -> Result
<PxarACL
, Error
> {
328 if !self.has_features(flags
::WITH_ACL
) {
331 if is_symlink(&stat
) {
334 if acl_type
== acl
::ACL_TYPE_DEFAULT
&& !is_directory(&stat
) {
335 bail
!("ACL_TYPE_DEFAULT only defined for directories.");
338 // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
339 // to create a path for acl_get_file(). acl_get_fd() only allows to get
340 // ACL_TYPE_ACCESS attributes.
341 let proc_path
= Path
::new("/proc/self/fd/").join(fd
.to_string());
342 let acl
= match acl
::ACL
::get_file(&proc_path
, acl_type
) {
344 // Don't bail if underlying endpoint does not support acls
345 Err(Errno
::EOPNOTSUPP
) => return Ok(ret
),
346 // Don't bail if the endpoint cannot carry acls
347 Err(Errno
::EBADF
) => return Ok(ret
),
348 // Don't bail if there is no data
349 Err(Errno
::ENODATA
) => return Ok(ret
),
350 Err(err
) => bail
!("error while reading ACL - {}", err
),
353 self.process_acl(acl
, acl_type
)
356 fn process_acl(&self, acl
: acl
::ACL
, acl_type
: acl
::ACLType
) -> Result
<PxarACL
, Error
> {
357 let mut acl_user
= Vec
::new();
358 let mut acl_group
= Vec
::new();
359 let mut acl_group_obj
= None
;
360 let mut acl_default
= None
;
361 let mut user_obj_permissions
= None
;
362 let mut group_obj_permissions
= None
;
363 let mut other_permissions
= None
;
364 let mut mask_permissions
= None
;
366 for entry
in &mut acl
.entries() {
367 let tag
= entry
.get_tag_type()?
;
368 let permissions
= entry
.get_permissions()?
;
370 acl
::ACL_USER_OBJ
=> user_obj_permissions
= Some(permissions
),
371 acl
::ACL_GROUP_OBJ
=> group_obj_permissions
= Some(permissions
),
372 acl
::ACL_OTHER
=> other_permissions
= Some(permissions
),
373 acl
::ACL_MASK
=> mask_permissions
= Some(permissions
),
375 acl_user
.push(PxarACLUser
{
376 uid
: entry
.get_qualifier()?
,
381 acl_group
.push(PxarACLGroup
{
382 gid
: entry
.get_qualifier()?
,
386 _
=> bail
!("Unexpected ACL tag encountered!"),
394 acl
::ACL_TYPE_ACCESS
=> {
395 // The mask permissions are mapped to the stat group permissions
396 // in case that the ACL group permissions were set.
397 // Only in that case we need to store the group permissions,
398 // in the other cases they are identical to the stat group permissions.
399 if let (Some(gop
), Some(_
)) = (group_obj_permissions
, mask_permissions
) {
400 acl_group_obj
= Some(PxarACLGroupObj { permissions: gop }
);
403 acl
::ACL_TYPE_DEFAULT
=> {
404 if user_obj_permissions
!= None
405 || group_obj_permissions
!= None
406 || other_permissions
!= None
407 || mask_permissions
!= None
409 acl_default
= Some(PxarACLDefault
{
410 // The value is set to UINT64_MAX as placeholder if one
411 // of the permissions is not set
412 user_obj_permissions
: user_obj_permissions
.unwrap_or(std
::u64::MAX
),
413 group_obj_permissions
: group_obj_permissions
.unwrap_or(std
::u64::MAX
),
414 other_permissions
: other_permissions
.unwrap_or(std
::u64::MAX
),
415 mask_permissions
: mask_permissions
.unwrap_or(std
::u64::MAX
),
419 _
=> bail
!("Unexpected ACL type encountered"),
425 group_obj
: acl_group_obj
,
426 default: acl_default
,
430 /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
431 fn read_quota_project_id(
436 ) -> Result
<Option
<PxarQuotaProjID
>, Error
> {
437 if !(is_directory(&stat
) || is_reg_file(&stat
)) {
440 if !self.has_features(flags
::WITH_QUOTA_PROJID
) {
444 use proxmox
::sys
::linux
::magic
::*;
447 EXT4_SUPER_MAGIC
| XFS_SUPER_MAGIC
| FUSE_SUPER_MAGIC
| ZFS_SUPER_MAGIC
=> {
448 let mut fsxattr
= fs
::FSXAttr
::default();
449 let res
= unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) }
;
451 // On some FUSE filesystems it can happen that ioctl is not supported.
452 // For these cases projid is set to 0 while the error is ignored.
453 if let Err(err
) = res
{
454 let errno
= err
.as_errno().ok_or_else(|| {
456 "error while reading quota project id for {:#?}",
460 if errno_is_unsupported(errno
) {
464 "error while reading quota project id for {:#?} - {}",
471 let projid
= fsxattr
.fsx_projid
as u64;
475 return Ok(Some(PxarQuotaProjID { projid }
));
478 _
=> return Ok(None
),
482 fn write_entry(&mut self, entry
: PxarEntry
) -> Result
<(), Error
> {
483 self.write_header(PXAR_ENTRY
, std
::mem
::size_of
::<PxarEntry
>() as u64)?
;
484 self.write_item(entry
)?
;
489 fn write_xattr(&mut self, xattr
: PxarXAttr
) -> Result
<(), Error
> {
490 let size
= xattr
.name
.len() + xattr
.value
.len() + 1; // +1 for '\0' separating name and value
491 self.write_header(PXAR_XATTR
, size
as u64)?
;
492 self.write(xattr
.name
.as_slice())?
;
494 self.write(xattr
.value
.as_slice())?
;
499 fn write_fcaps(&mut self, fcaps
: Option
<PxarFCaps
>) -> Result
<(), Error
> {
500 if let Some(fcaps
) = fcaps
{
501 let size
= fcaps
.data
.len();
502 self.write_header(PXAR_FCAPS
, size
as u64)?
;
503 self.write(fcaps
.data
.as_slice())?
;
509 fn write_acl_user(&mut self, acl_user
: PxarACLUser
) -> Result
<(), Error
> {
510 self.write_header(PXAR_ACL_USER
, std
::mem
::size_of
::<PxarACLUser
>() as u64)?
;
511 self.write_item(acl_user
)?
;
516 fn write_acl_group(&mut self, acl_group
: PxarACLGroup
) -> Result
<(), Error
> {
517 self.write_header(PXAR_ACL_GROUP
, std
::mem
::size_of
::<PxarACLGroup
>() as u64)?
;
518 self.write_item(acl_group
)?
;
523 fn write_acl_group_obj(&mut self, acl_group_obj
: PxarACLGroupObj
) -> Result
<(), Error
> {
526 std
::mem
::size_of
::<PxarACLGroupObj
>() as u64,
528 self.write_item(acl_group_obj
)?
;
533 fn write_acl_default(&mut self, acl_default
: PxarACLDefault
) -> Result
<(), Error
> {
536 std
::mem
::size_of
::<PxarACLDefault
>() as u64,
538 self.write_item(acl_default
)?
;
543 fn write_acl_default_user(&mut self, acl_default_user
: PxarACLUser
) -> Result
<(), Error
> {
545 PXAR_ACL_DEFAULT_USER
,
546 std
::mem
::size_of
::<PxarACLUser
>() as u64,
548 self.write_item(acl_default_user
)?
;
553 fn write_acl_default_group(&mut self, acl_default_group
: PxarACLGroup
) -> Result
<(), Error
> {
555 PXAR_ACL_DEFAULT_GROUP
,
556 std
::mem
::size_of
::<PxarACLGroup
>() as u64,
558 self.write_item(acl_default_group
)?
;
563 fn write_quota_project_id(&mut self, projid
: PxarQuotaProjID
) -> Result
<(), Error
> {
566 std
::mem
::size_of
::<PxarQuotaProjID
>() as u64,
568 self.write_item(projid
)?
;
573 fn write_goodbye_table(
575 goodbye_offset
: usize,
576 goodbye_items
: &mut [PxarGoodbyeItem
],
577 ) -> Result
<(), Error
> {
578 goodbye_items
.sort_unstable_by(|a
, b
| a
.hash
.cmp(&b
.hash
));
580 let item_count
= goodbye_items
.len();
582 let goodbye_table_size
= (item_count
+ 1) * std
::mem
::size_of
::<PxarGoodbyeItem
>();
584 self.write_header(PXAR_GOODBYE
, goodbye_table_size
as u64)?
;
586 if self.file_copy_buffer
.len() < goodbye_table_size
{
587 let need
= goodbye_table_size
- self.file_copy_buffer
.len();
588 self.file_copy_buffer
.reserve(need
);
590 self.file_copy_buffer
591 .set_len(self.file_copy_buffer
.capacity());
595 let buffer
= &mut self.file_copy_buffer
;
597 copy_binary_search_tree(item_count
, |s
, d
| {
598 let item
= &goodbye_items
[s
];
599 let offset
= d
* std
::mem
::size_of
::<PxarGoodbyeItem
>();
601 crate::tools
::map_struct_mut
::<PxarGoodbyeItem
>(&mut buffer
[offset
..]).unwrap();
602 dest
.offset
= u64::to_le(item
.offset
);
603 dest
.size
= u64::to_le(item
.size
);
604 dest
.hash
= u64::to_le(item
.hash
);
607 // append PxarGoodbyeTail as last item
608 let offset
= item_count
* std
::mem
::size_of
::<PxarGoodbyeItem
>();
609 let dest
= crate::tools
::map_struct_mut
::<PxarGoodbyeItem
>(&mut buffer
[offset
..]).unwrap();
610 dest
.offset
= u64::to_le(goodbye_offset
as u64);
611 dest
.size
= u64::to_le((goodbye_table_size
+ std
::mem
::size_of
::<PxarHeader
>()) as u64);
612 dest
.hash
= u64::to_le(PXAR_GOODBYE_TAIL_MARKER
);
614 self.flush_copy_buffer(goodbye_table_size
)?
;
621 dir
: &mut nix
::dir
::Dir
,
624 match_pattern
: Vec
<MatchPattern
>,
625 ) -> Result
<(), Error
> {
626 //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
628 let mut name_list
= vec
![];
630 let rawfd
= dir
.as_raw_fd();
632 let dir_start_pos
= self.writer_pos
;
634 let mut dir_entry
= self.create_entry(&dir_stat
)?
;
636 self.read_chattr(rawfd
, &mut dir_entry
)?
;
637 self.read_fat_attr(rawfd
, magic
, &mut dir_entry
)?
;
639 // for each node in the directory tree, the filesystem features are
640 // checked based on the fs magic number.
641 self.fs_feature_flags
= flags
::feature_flags_from_magic(magic
);
643 let (xattrs
, fcaps
) = self.read_xattrs(rawfd
, &dir_stat
)?
;
644 let acl_access
= self.read_acl(rawfd
, &dir_stat
, acl
::ACL_TYPE_ACCESS
)?
;
645 let acl_default
= self.read_acl(rawfd
, &dir_stat
, acl
::ACL_TYPE_DEFAULT
)?
;
646 let projid
= self.read_quota_project_id(rawfd
, magic
, &dir_stat
)?
;
648 self.write_entry(dir_entry
)?
;
649 for xattr
in xattrs
{
650 self.write_xattr(xattr
)?
;
652 self.write_fcaps(fcaps
)?
;
654 for user
in acl_access
.users
{
655 self.write_acl_user(user
)?
;
657 for group
in acl_access
.groups
{
658 self.write_acl_group(group
)?
;
660 if let Some(group_obj
) = acl_access
.group_obj
{
661 self.write_acl_group_obj(group_obj
)?
;
664 for default_user
in acl_default
.users
{
665 self.write_acl_default_user(default_user
)?
;
667 for default_group
in acl_default
.groups
{
668 self.write_acl_default_group(default_group
)?
;
670 if let Some(default) = acl_default
.default {
671 self.write_acl_default(default)?
;
673 if let Some(projid
) = projid
{
674 self.write_quota_project_id(projid
)?
;
677 let include_children
;
678 if is_virtual_file_system(magic
) {
679 include_children
= false;
681 if let Some(set
) = &self.device_set
{
682 include_children
= set
.contains(&dir_stat
.st_dev
);
684 include_children
= true;
688 // Expand the exclude match pattern inherited from the parent by local entries, if present
689 let mut local_match_pattern
= match_pattern
.clone();
690 let pxar_exclude
= match MatchPattern
::from_file(rawfd
, ".pxarexclude") {
691 Ok(Some((mut excludes
, buffer
, stat
))) => {
692 local_match_pattern
.append(&mut excludes
);
696 Err(err
) => bail
!("error while reading exclude file - {}", err
),
699 if include_children
{
700 for entry
in dir
.iter() {
702 .map_err(|err
| format_err
!("readir {:?} failed - {}", self.full_path(), err
))?
;
703 let filename
= entry
.file_name().to_owned();
705 let name
= filename
.to_bytes_with_nul();
706 if name
== b
".\0" || name
== b
"..\0" {
710 let stat
= match nix
::sys
::stat
::fstatat(
713 nix
::fcntl
::AtFlags
::AT_SYMLINK_NOFOLLOW
,
716 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
717 let filename_osstr
= std
::ffi
::OsStr
::from_bytes(filename
.to_bytes());
718 self.report_vanished_file(&self.full_path().join(filename_osstr
))?
;
721 Err(err
) => bail
!("fstat {:?} failed - {}", self.full_path(), err
),
724 match match_filename(&filename
, &stat
, &local_match_pattern
)?
{
725 (MatchType
::Positive
, _
) => {
726 let filename_osstr
= std
::ffi
::OsStr
::from_bytes(filename
.to_bytes());
728 "matched by .pxarexclude entry - skipping: {:?}",
729 self.full_path().join(filename_osstr
)
732 (_
, child_pattern
) => name_list
.push((filename
, stat
, child_pattern
)),
735 if name_list
.len() > MAX_DIRECTORY_ENTRIES
{
737 "too many directory items in {:?} (> {})",
739 MAX_DIRECTORY_ENTRIES
744 eprintln
!("skip mount point: {:?}", self.full_path());
747 name_list
.sort_unstable_by(|a
, b
| a
.0.cmp(&b
.0));
749 let mut goodbye_items
= vec
![];
751 for (filename
, stat
, exclude_list
) in name_list
{
752 let start_pos
= self.writer_pos
;
754 if filename
.as_bytes() == b
".pxarexclude" {
755 if let Some((ref content
, ref stat
)) = pxar_exclude
{
756 let filefd
= match nix
::fcntl
::openat(
762 Ok(filefd
) => filefd
,
763 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
764 self.report_vanished_file(&self.full_path())?
;
768 let filename_osstr
= std
::ffi
::OsStr
::from_bytes(filename
.to_bytes());
770 "open file {:?} failed - {}",
771 self.full_path().join(filename_osstr
),
777 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
778 detect_fs_type(filefd
)?
783 self.write_filename(&filename
)?
;
784 if let Some(ref mut catalog
) = self.catalog
{
785 catalog
.add_file(&filename
, stat
.st_size
as u64, stat
.st_mtime
as u64)?
;
787 self.encode_pxar_exclude(filefd
, stat
, child_magic
, content
)?
;
793 .push(std
::ffi
::OsStr
::from_bytes(filename
.as_bytes()));
796 println
!("{:?}", self.full_path());
799 if is_directory(&stat
) {
800 let mut dir
= match nix
::dir
::Dir
::openat(
803 OFlag
::O_DIRECTORY
| OFlag
::O_NOFOLLOW
,
807 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
808 self.report_vanished_file(&self.full_path())?
;
811 Err(err
) => bail
!("open dir {:?} failed - {}", self.full_path(), err
),
814 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
815 detect_fs_type(dir
.as_raw_fd())?
820 self.write_filename(&filename
)?
;
821 if let Some(ref mut catalog
) = self.catalog
{
822 catalog
.start_directory(&filename
)?
;
824 self.encode_dir(&mut dir
, &stat
, child_magic
, exclude_list
)?
;
825 if let Some(ref mut catalog
) = self.catalog
{
826 catalog
.end_directory()?
;
828 } else if is_reg_file(&stat
) {
829 let mut hardlink_target
= None
;
831 if stat
.st_nlink
> 1 {
832 let link_info
= HardLinkInfo
{
836 hardlink_target
= self.hardlinks
.get(&link_info
).map(|(v
, offset
)| {
837 let mut target
= v
.clone().into_os_string();
838 target
.push("\0"); // add Nul byte
839 (target
, (start_pos
as u64) - offset
)
841 if hardlink_target
== None
{
843 .insert(link_info
, (self.relative_path
.clone(), start_pos
as u64));
847 if let Some((target
, offset
)) = hardlink_target
{
848 if let Some(ref mut catalog
) = self.catalog
{
849 catalog
.add_hardlink(&filename
)?
;
851 self.write_filename(&filename
)?
;
852 self.encode_hardlink(target
.as_bytes(), offset
)?
;
854 let filefd
= match nix
::fcntl
::openat(
860 Ok(filefd
) => filefd
,
861 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
862 self.report_vanished_file(&self.full_path())?
;
865 Err(err
) => bail
!("open file {:?} failed - {}", self.full_path(), err
),
868 if let Some(ref mut catalog
) = self.catalog
{
869 catalog
.add_file(&filename
, stat
.st_size
as u64, stat
.st_mtime
as u64)?
;
871 let child_magic
= if dir_stat
.st_dev
!= stat
.st_dev
{
872 detect_fs_type(filefd
)?
877 self.write_filename(&filename
)?
;
878 let res
= self.encode_file(filefd
, &stat
, child_magic
);
879 let _
= nix
::unistd
::close(filefd
); // ignore close errors
882 } else if is_symlink(&stat
) {
883 let mut buffer
= vec
::undefined(libc
::PATH_MAX
as usize);
885 let res
= filename
.with_nix_path(|cstr
| unsafe {
889 buffer
.as_mut_ptr() as *mut libc
::c_char
,
894 match Errno
::result(res
) {
896 if let Some(ref mut catalog
) = self.catalog
{
897 catalog
.add_symlink(&filename
)?
;
899 buffer
[len
as usize] = 0u8; // add Nul byte
900 self.write_filename(&filename
)?
;
901 self.encode_symlink(&buffer
[..((len
+ 1) as usize)], &stat
)?
903 Err(nix
::Error
::Sys(Errno
::ENOENT
)) => {
904 self.report_vanished_file(&self.full_path())?
;
907 Err(err
) => bail
!("readlink {:?} failed - {}", self.full_path(), err
),
909 } else if is_block_dev(&stat
) || is_char_dev(&stat
) {
910 if self.has_features(flags
::WITH_DEVICE_NODES
) {
911 if let Some(ref mut catalog
) = self.catalog
{
912 if is_block_dev(&stat
) {
913 catalog
.add_block_device(&filename
)?
;
915 catalog
.add_char_device(&filename
)?
;
918 self.write_filename(&filename
)?
;
919 self.encode_device(&stat
)?
;
921 eprintln
!("skip device node: {:?}", self.full_path());
923 } else if is_fifo(&stat
) {
924 if self.has_features(flags
::WITH_FIFOS
) {
925 if let Some(ref mut catalog
) = self.catalog
{
926 catalog
.add_fifo(&filename
)?
;
928 self.write_filename(&filename
)?
;
929 self.encode_special(&stat
)?
;
931 eprintln
!("skip fifo: {:?}", self.full_path());
933 } else if is_socket(&stat
) {
934 if self.has_features(flags
::WITH_SOCKETS
) {
935 if let Some(ref mut catalog
) = self.catalog
{
936 catalog
.add_socket(&filename
)?
;
938 self.write_filename(&filename
)?
;
939 self.encode_special(&stat
)?
;
941 eprintln
!("skip socket: {:?}", self.full_path());
945 "unsupported file type (mode {:o} {:?})",
951 let end_pos
= self.writer_pos
;
953 goodbye_items
.push(PxarGoodbyeItem
{
954 offset
: start_pos
as u64,
955 size
: (end_pos
- start_pos
) as u64,
956 hash
: compute_goodbye_hash(filename
.to_bytes()),
959 self.relative_path
.pop();
962 //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
964 // fixup goodby item offsets
965 let goodbye_start
= self.writer_pos
as u64;
966 for item
in &mut goodbye_items
{
967 item
.offset
= goodbye_start
- item
.offset
;
970 let goodbye_offset
= self.writer_pos
- dir_start_pos
;
972 self.write_goodbye_table(goodbye_offset
, &mut goodbye_items
)?
;
974 //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
978 fn encode_file(&mut self, filefd
: RawFd
, stat
: &FileStat
, magic
: i64) -> Result
<(), Error
> {
979 //println!("encode_file: {:?}", self.full_path());
981 let mut entry
= self.create_entry(&stat
)?
;
983 self.read_chattr(filefd
, &mut entry
)?
;
984 self.read_fat_attr(filefd
, magic
, &mut entry
)?
;
985 let (xattrs
, fcaps
) = self.read_xattrs(filefd
, &stat
)?
;
986 let acl_access
= self.read_acl(filefd
, &stat
, acl
::ACL_TYPE_ACCESS
)?
;
987 let projid
= self.read_quota_project_id(filefd
, magic
, &stat
)?
;
989 self.write_entry(entry
)?
;
990 for xattr
in xattrs
{
991 self.write_xattr(xattr
)?
;
993 self.write_fcaps(fcaps
)?
;
994 for user
in acl_access
.users
{
995 self.write_acl_user(user
)?
;
997 for group
in acl_access
.groups
{
998 self.write_acl_group(group
)?
;
1000 if let Some(group_obj
) = acl_access
.group_obj
{
1001 self.write_acl_group_obj(group_obj
)?
;
1003 if let Some(projid
) = projid
{
1004 self.write_quota_project_id(projid
)?
;
1007 let include_payload
;
1008 if is_virtual_file_system(magic
) {
1009 include_payload
= false;
1011 if let Some(ref set
) = &self.device_set
{
1012 include_payload
= set
.contains(&stat
.st_dev
);
1014 include_payload
= true;
1018 if !include_payload
{
1019 eprintln
!("skip content: {:?}", self.full_path());
1020 self.write_header(PXAR_PAYLOAD
, 0)?
;
1024 let size
= stat
.st_size
as u64;
1026 self.write_header(PXAR_PAYLOAD
, size
)?
;
1028 let mut pos
: u64 = 0;
1030 let n
= match nix
::unistd
::read(filefd
, &mut self.file_copy_buffer
) {
1032 Err(nix
::Error
::Sys(Errno
::EINTR
)) => continue, /* try again */
1033 Err(err
) => bail
!("read {:?} failed - {}", self.full_path(), err
),
1037 // Note:: casync format cannot handle that
1039 "detected shrinked file {:?} ({} < {})",
1048 let mut next
= pos
+ (n
as u64);
1054 let count
= (next
- pos
) as usize;
1056 self.flush_copy_buffer(count
)?
;
1068 fn encode_device(&mut self, stat
: &FileStat
) -> Result
<(), Error
> {
1069 let entry
= self.create_entry(&stat
)?
;
1071 self.write_entry(entry
)?
;
1073 let major
= unsafe { libc::major(stat.st_rdev) }
as u64;
1074 let minor
= unsafe { libc::minor(stat.st_rdev) }
as u64;
1076 //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
1078 self.write_header(PXAR_DEVICE
, std
::mem
::size_of
::<PxarDevice
>() as u64)?
;
1079 self.write_item(PxarDevice { major, minor }
)?
;
1085 fn encode_special(&mut self, stat
: &FileStat
) -> Result
<(), Error
> {
1086 let entry
= self.create_entry(&stat
)?
;
1088 self.write_entry(entry
)?
;
1093 fn encode_symlink(&mut self, target
: &[u8], stat
: &FileStat
) -> Result
<(), Error
> {
1094 //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
1096 let entry
= self.create_entry(&stat
)?
;
1097 self.write_entry(entry
)?
;
1099 self.write_header(PXAR_SYMLINK
, target
.len() as u64)?
;
1100 self.write(target
)?
;
1105 fn encode_hardlink(&mut self, target
: &[u8], offset
: u64) -> Result
<(), Error
> {
1106 //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
1108 // Note: HARDLINK replaces an ENTRY.
1109 self.write_header(PXAR_FORMAT_HARDLINK
, (target
.len() as u64) + 8)?
;
1110 self.write_item(offset
)?
;
1111 self.write(target
)?
;
1116 fn encode_pxar_exclude(
1122 ) -> Result
<(), Error
> {
1123 let mut entry
= self.create_entry(&stat
)?
;
1125 self.read_chattr(filefd
, &mut entry
)?
;
1126 self.read_fat_attr(filefd
, magic
, &mut entry
)?
;
1127 let (xattrs
, fcaps
) = self.read_xattrs(filefd
, &stat
)?
;
1128 let acl_access
= self.read_acl(filefd
, &stat
, acl
::ACL_TYPE_ACCESS
)?
;
1129 let projid
= self.read_quota_project_id(filefd
, magic
, &stat
)?
;
1131 self.write_entry(entry
)?
;
1132 for xattr
in xattrs
{
1133 self.write_xattr(xattr
)?
;
1135 self.write_fcaps(fcaps
)?
;
1136 for user
in acl_access
.users
{
1137 self.write_acl_user(user
)?
;
1139 for group
in acl_access
.groups
{
1140 self.write_acl_group(group
)?
;
1142 if let Some(group_obj
) = acl_access
.group_obj
{
1143 self.write_acl_group_obj(group_obj
)?
;
1145 if let Some(projid
) = projid
{
1146 self.write_quota_project_id(projid
)?
;
1149 let include_payload
;
1150 if is_virtual_file_system(magic
) {
1151 include_payload
= false;
1153 if let Some(set
) = &self.device_set
{
1154 include_payload
= set
.contains(&stat
.st_dev
);
1156 include_payload
= true;
1160 if !include_payload
{
1161 eprintln
!("skip content: {:?}", self.full_path());
1162 self.write_header(PXAR_PAYLOAD
, 0)?
;
1166 let size
= content
.len();
1167 self.write_header(PXAR_PAYLOAD
, size
as u64)?
;
1168 self.writer
.write_all(content
)?
;
1169 self.writer_pos
+= size
;
1174 // the report_XXX method may raise and error - depending on encoder configuration
1176 fn report_vanished_file(&self, path
: &Path
) -> Result
<(), Error
> {
1177 eprintln
!("WARNING: detected vanished file {:?}", path
);
1183 // If there is a match, an updated MatchPattern list to pass to the matched child is returned.
1187 match_pattern
: &Vec
<MatchPattern
>,
1188 ) -> Result
<(MatchType
, Vec
<MatchPattern
>), Error
> {
1189 let mut child_pattern
= Vec
::new();
1190 let mut match_state
= MatchType
::None
;
1192 for pattern
in match_pattern
{
1193 match pattern
.matches_filename(filename
, is_directory(&stat
))?
{
1194 MatchType
::None
=> {}
1195 MatchType
::Positive
=> match_state
= MatchType
::Positive
,
1196 MatchType
::Negative
=> match_state
= MatchType
::Negative
,
1198 if match_state
!= MatchType
::Positive
&& match_state
!= MatchType
::Negative
{
1199 match_state
= match_type
;
1201 child_pattern
.push(pattern
.get_rest_pattern());
1206 Ok((match_state
, child_pattern
))
1209 fn errno_is_unsupported(errno
: Errno
) -> bool
{
1211 Errno
::ENOTTY
| Errno
::ENOSYS
| Errno
::EBADF
| Errno
::EOPNOTSUPP
| Errno
::EINVAL
=> true,
1216 fn detect_fs_type(fd
: RawFd
) -> Result
<i64, Error
> {
1217 let mut fs_stat
= std
::mem
::MaybeUninit
::uninit();
1218 let res
= unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) }
;
1219 Errno
::result(res
)?
;
1220 let fs_stat
= unsafe { fs_stat.assume_init() }
;
1226 pub fn is_temporary_file_system(magic
: i64) -> bool
{
1227 use proxmox
::sys
::linux
::magic
::*;
1228 magic
== RAMFS_MAGIC
|| magic
== TMPFS_MAGIC
1231 pub fn is_virtual_file_system(magic
: i64) -> bool
{
1232 use proxmox
::sys
::linux
::magic
::*;
1236 CGROUP2_SUPER_MAGIC
|
1237 CGROUP_SUPER_MAGIC
|
1240 DEVPTS_SUPER_MAGIC
|
1242 FUSE_CTL_SUPER_MAGIC
|
1252 SYSFS_MAGIC
=> true,