]> git.proxmox.com Git - proxmox-backup.git/blob - src/pxar/encoder.rs
f438e39a7ff5e0c78b189f62f82e3b9dfb5a04c3
[proxmox-backup.git] / src / pxar / encoder.rs
1 //! *pxar* format encoder.
2 //!
3 //! This module contain the code to generate *pxar* archive files.
4 use std::collections::{HashMap, HashSet};
5 use std::ffi::{CStr, CString};
6 use std::io::Write;
7 use std::os::unix::ffi::OsStrExt;
8 use std::os::unix::io::AsRawFd;
9 use std::os::unix::io::RawFd;
10 use std::path::{Path, PathBuf};
11
12 use endian_trait::Endian;
13 use anyhow::{bail, format_err, Error};
14 use nix::errno::Errno;
15 use nix::fcntl::OFlag;
16 use nix::sys::stat::FileStat;
17 use nix::sys::stat::Mode;
18 use nix::NixPath;
19
20 use proxmox::tools::vec;
21
22 use super::binary_search_tree::*;
23 use super::catalog::BackupCatalogWriter;
24 use super::flags;
25 use super::format_definition::*;
26 use super::helper::*;
27 use super::match_pattern::{MatchPattern, MatchPatternSlice, MatchType};
28 use crate::tools::acl;
29 use crate::tools::fs;
30 use crate::tools::xattr;
31
32 #[derive(Eq, PartialEq, Hash)]
33 struct HardLinkInfo {
34 st_dev: u64,
35 st_ino: u64,
36 }
37
38 pub struct Encoder<'a, W: Write, C: BackupCatalogWriter> {
39 base_path: PathBuf,
40 relative_path: PathBuf,
41 writer: &'a mut W,
42 writer_pos: usize,
43 catalog: Option<&'a mut C>,
44 _size: usize,
45 file_copy_buffer: Vec<u8>,
46 device_set: Option<HashSet<u64>>,
47 verbose: bool,
48 // Flags set by the user
49 feature_flags: u64,
50 // Flags signaling features supported by the filesystem
51 fs_feature_flags: u64,
52 hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
53 entry_counter: usize,
54 entry_max: usize,
55 }
56
57 impl<'a, W: Write, C: BackupCatalogWriter> Encoder<'a, W, C> {
58 // used for error reporting
59 fn full_path(&self) -> PathBuf {
60 self.base_path.join(&self.relative_path)
61 }
62
63 /// Create archive, write result data to ``writer``.
64 ///
65 /// The ``device_set`` can be use used to limit included mount points.
66 ///
67 /// - ``None``: include all mount points
68 /// - ``Some(set)``: only include devices listed in this set (the
69 /// root path device is automathically added to this list, so
70 /// you can pass an empty set if you want to archive a single
71 /// mount point.)
72 pub fn encode(
73 path: PathBuf,
74 dir: &mut nix::dir::Dir,
75 writer: &'a mut W,
76 catalog: Option<&'a mut C>,
77 device_set: Option<HashSet<u64>>,
78 verbose: bool,
79 skip_lost_and_found: bool, // fixme: should be a feature flag ??
80 feature_flags: u64,
81 mut excludes: Vec<MatchPattern>,
82 entry_max: usize,
83 ) -> Result<(), Error> {
84 const FILE_COPY_BUFFER_SIZE: usize = 1024 * 1024;
85
86 let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
87 unsafe {
88 file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE);
89 }
90
91 // todo: use scandirat??
92
93 let dir_fd = dir.as_raw_fd();
94 let stat = nix::sys::stat::fstat(dir_fd)
95 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
96
97 if !is_directory(&stat) {
98 bail!("got unexpected file type {:?} (not a directory)", path);
99 }
100
101 let mut device_set = device_set.clone();
102 if let Some(ref mut set) = device_set {
103 set.insert(stat.st_dev);
104 }
105
106 let magic = detect_fs_type(dir_fd)?;
107
108 if is_virtual_file_system(magic) {
109 bail!("backup virtual file systems is disabled!");
110 }
111
112 let fs_feature_flags = flags::feature_flags_from_magic(magic);
113
114 let mut me = Self {
115 base_path: path,
116 relative_path: PathBuf::new(),
117 writer,
118 writer_pos: 0,
119 catalog,
120 _size: 0,
121 file_copy_buffer,
122 device_set,
123 verbose,
124 feature_flags,
125 fs_feature_flags,
126 hardlinks: HashMap::new(),
127 entry_counter: 0,
128 entry_max,
129 };
130
131 if verbose {
132 println!("{:?}", me.full_path());
133 }
134
135 if skip_lost_and_found {
136 excludes.push(MatchPattern::from_line(b"**/lost+found").unwrap().unwrap());
137 }
138 let mut exclude_slices = Vec::new();
139 for excl in &excludes {
140 exclude_slices.push(excl.as_slice());
141 }
142
143 me.encode_dir(dir, &stat, magic, exclude_slices)?;
144
145 Ok(())
146 }
147
148 fn write(&mut self, buf: &[u8]) -> Result<(), Error> {
149 self.writer.write_all(buf)?;
150 self.writer_pos += buf.len();
151 Ok(())
152 }
153
154 fn write_item<T: Endian>(&mut self, item: T) -> Result<(), Error> {
155 let data = item.to_le();
156
157 let buffer = unsafe {
158 std::slice::from_raw_parts(&data as *const T as *const u8, std::mem::size_of::<T>())
159 };
160
161 self.write(buffer)?;
162
163 Ok(())
164 }
165
166 fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
167 self.writer.write_all(&self.file_copy_buffer[..size])?;
168 self.writer_pos += size;
169 Ok(())
170 }
171
172 fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
173 let size = size + (std::mem::size_of::<PxarHeader>() as u64);
174 self.write_item(PxarHeader { size, htype })?;
175
176 Ok(())
177 }
178
179 fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
180 let buffer = name.to_bytes_with_nul();
181 self.write_header(PXAR_FILENAME, buffer.len() as u64)?;
182 self.write(buffer)?;
183
184 Ok(())
185 }
186
187 fn create_entry(&self, stat: &FileStat) -> Result<PxarEntry, Error> {
188 let mode = if is_symlink(&stat) {
189 (libc::S_IFLNK | 0o777) as u64
190 } else {
191 (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
192 };
193
194 let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
195 if mtime < 0 {
196 bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
197 }
198
199 let entry = PxarEntry {
200 mode,
201 flags: 0,
202 uid: stat.st_uid,
203 gid: stat.st_gid,
204 mtime: mtime as u64,
205 };
206
207 Ok(entry)
208 }
209
210 fn read_chattr(&self, fd: RawFd, entry: &mut PxarEntry) -> Result<(), Error> {
211 let mut attr: usize = 0;
212
213 let res = unsafe { fs::read_attr_fd(fd, &mut attr) };
214 if let Err(err) = res {
215 if let nix::Error::Sys(errno) = err {
216 if errno_is_unsupported(errno) {
217 return Ok(());
218 };
219 }
220 bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
221 }
222
223 let flags = flags::feature_flags_from_chattr(attr as u32);
224 entry.flags |= flags;
225
226 Ok(())
227 }
228
229 fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut PxarEntry) -> Result<(), Error> {
230 use proxmox::sys::linux::magic::*;
231
232 if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC {
233 return Ok(());
234 }
235
236 let mut attr: u32 = 0;
237
238 let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr) };
239 if let Err(err) = res {
240 if let nix::Error::Sys(errno) = err {
241 if errno_is_unsupported(errno) {
242 return Ok(());
243 };
244 }
245 bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
246 }
247
248 let flags = flags::feature_flags_from_fat_attr(attr);
249 entry.flags |= flags;
250
251 Ok(())
252 }
253
254 /// True if all of the given feature flags are set in the Encoder, false otherwise
255 fn has_features(&self, feature_flags: u64) -> bool {
256 (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
257 }
258
259 /// True if at least one of the given feature flags is set in the Encoder, false otherwise
260 fn has_some_features(&self, feature_flags: u64) -> bool {
261 (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
262 }
263
264 fn read_xattrs(
265 &self,
266 fd: RawFd,
267 stat: &FileStat,
268 ) -> Result<(Vec<PxarXAttr>, Option<PxarFCaps>), Error> {
269 let mut xattrs = Vec::new();
270 let mut fcaps = None;
271
272 let flags = flags::WITH_XATTRS | flags::WITH_FCAPS;
273 if !self.has_some_features(flags) {
274 return Ok((xattrs, fcaps));
275 }
276 // Should never be called on symlinks, just in case check anyway
277 if is_symlink(&stat) {
278 return Ok((xattrs, fcaps));
279 }
280
281 let xattr_names = match xattr::flistxattr(fd) {
282 Ok(names) => names,
283 // Do not bail if the underlying endpoint does not supports xattrs
284 Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
285 // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
286 Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
287 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
288 };
289
290 for name in &xattr_names {
291 // Only extract the relevant extended attributes
292 if !xattr::is_valid_xattr_name(&name) {
293 continue;
294 }
295
296 let value = match xattr::fgetxattr(fd, name) {
297 Ok(value) => value,
298 // Vanished between flistattr and getxattr, this is ok, silently ignore
299 Err(Errno::ENODATA) => continue,
300 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
301 };
302
303 if xattr::is_security_capability(&name) {
304 if self.has_features(flags::WITH_FCAPS) {
305 // fcaps are stored in own format within the archive
306 fcaps = Some(PxarFCaps { data: value });
307 }
308 } else if self.has_features(flags::WITH_XATTRS) {
309 xattrs.push(PxarXAttr {
310 name: name.to_bytes().to_vec(),
311 value,
312 });
313 }
314 }
315 xattrs.sort();
316
317 Ok((xattrs, fcaps))
318 }
319
320 fn read_acl(
321 &self,
322 fd: RawFd,
323 stat: &FileStat,
324 acl_type: acl::ACLType,
325 ) -> Result<PxarACL, Error> {
326 let ret = PxarACL {
327 users: Vec::new(),
328 groups: Vec::new(),
329 group_obj: None,
330 default: None,
331 };
332
333 if !self.has_features(flags::WITH_ACL) {
334 return Ok(ret);
335 }
336 if is_symlink(&stat) {
337 return Ok(ret);
338 }
339 if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
340 bail!("ACL_TYPE_DEFAULT only defined for directories.");
341 }
342
343 // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
344 // to create a path for acl_get_file(). acl_get_fd() only allows to get
345 // ACL_TYPE_ACCESS attributes.
346 let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
347 let acl = match acl::ACL::get_file(&proc_path, acl_type) {
348 Ok(acl) => acl,
349 // Don't bail if underlying endpoint does not support acls
350 Err(Errno::EOPNOTSUPP) => return Ok(ret),
351 // Don't bail if the endpoint cannot carry acls
352 Err(Errno::EBADF) => return Ok(ret),
353 // Don't bail if there is no data
354 Err(Errno::ENODATA) => return Ok(ret),
355 Err(err) => bail!("error while reading ACL - {}", err),
356 };
357
358 self.process_acl(acl, acl_type)
359 }
360
361 fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
362 let mut acl_user = Vec::new();
363 let mut acl_group = Vec::new();
364 let mut acl_group_obj = None;
365 let mut acl_default = None;
366 let mut user_obj_permissions = None;
367 let mut group_obj_permissions = None;
368 let mut other_permissions = None;
369 let mut mask_permissions = None;
370
371 for entry in &mut acl.entries() {
372 let tag = entry.get_tag_type()?;
373 let permissions = entry.get_permissions()?;
374 match tag {
375 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
376 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
377 acl::ACL_OTHER => other_permissions = Some(permissions),
378 acl::ACL_MASK => mask_permissions = Some(permissions),
379 acl::ACL_USER => {
380 acl_user.push(PxarACLUser {
381 uid: entry.get_qualifier()?,
382 permissions,
383 });
384 }
385 acl::ACL_GROUP => {
386 acl_group.push(PxarACLGroup {
387 gid: entry.get_qualifier()?,
388 permissions,
389 });
390 }
391 _ => bail!("Unexpected ACL tag encountered!"),
392 }
393 }
394
395 acl_user.sort();
396 acl_group.sort();
397
398 match acl_type {
399 acl::ACL_TYPE_ACCESS => {
400 // The mask permissions are mapped to the stat group permissions
401 // in case that the ACL group permissions were set.
402 // Only in that case we need to store the group permissions,
403 // in the other cases they are identical to the stat group permissions.
404 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
405 acl_group_obj = Some(PxarACLGroupObj { permissions: gop });
406 }
407 }
408 acl::ACL_TYPE_DEFAULT => {
409 if user_obj_permissions != None
410 || group_obj_permissions != None
411 || other_permissions != None
412 || mask_permissions != None
413 {
414 acl_default = Some(PxarACLDefault {
415 // The value is set to UINT64_MAX as placeholder if one
416 // of the permissions is not set
417 user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
418 group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
419 other_permissions: other_permissions.unwrap_or(std::u64::MAX),
420 mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
421 });
422 }
423 }
424 _ => bail!("Unexpected ACL type encountered"),
425 }
426
427 Ok(PxarACL {
428 users: acl_user,
429 groups: acl_group,
430 group_obj: acl_group_obj,
431 default: acl_default,
432 })
433 }
434
435 /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
436 fn read_quota_project_id(
437 &self,
438 fd: RawFd,
439 magic: i64,
440 stat: &FileStat,
441 ) -> Result<Option<PxarQuotaProjID>, Error> {
442 if !(is_directory(&stat) || is_reg_file(&stat)) {
443 return Ok(None);
444 }
445 if !self.has_features(flags::WITH_QUOTA_PROJID) {
446 return Ok(None);
447 }
448
449 use proxmox::sys::linux::magic::*;
450
451 match magic {
452 EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC | ZFS_SUPER_MAGIC => {
453 let mut fsxattr = fs::FSXAttr::default();
454 let res = unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) };
455
456 // On some FUSE filesystems it can happen that ioctl is not supported.
457 // For these cases projid is set to 0 while the error is ignored.
458 if let Err(err) = res {
459 let errno = err.as_errno().ok_or_else(|| {
460 format_err!(
461 "error while reading quota project id for {:#?}",
462 self.full_path()
463 )
464 })?;
465 if errno_is_unsupported(errno) {
466 return Ok(None);
467 } else {
468 bail!(
469 "error while reading quota project id for {:#?} - {}",
470 self.full_path(),
471 errno
472 );
473 }
474 }
475
476 let projid = fsxattr.fsx_projid as u64;
477 if projid == 0 {
478 Ok(None)
479 } else {
480 Ok(Some(PxarQuotaProjID { projid }))
481 }
482 }
483 _ => Ok(None),
484 }
485 }
486
487 fn write_entry(&mut self, entry: PxarEntry) -> Result<(), Error> {
488 self.write_header(PXAR_ENTRY, std::mem::size_of::<PxarEntry>() as u64)?;
489 self.write_item(entry)?;
490
491 Ok(())
492 }
493
494 fn write_xattr(&mut self, xattr: PxarXAttr) -> Result<(), Error> {
495 let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
496 self.write_header(PXAR_XATTR, size as u64)?;
497 self.write(xattr.name.as_slice())?;
498 self.write(&[0])?;
499 self.write(xattr.value.as_slice())?;
500
501 Ok(())
502 }
503
504 fn write_fcaps(&mut self, fcaps: Option<PxarFCaps>) -> Result<(), Error> {
505 if let Some(fcaps) = fcaps {
506 let size = fcaps.data.len();
507 self.write_header(PXAR_FCAPS, size as u64)?;
508 self.write(fcaps.data.as_slice())?;
509 }
510
511 Ok(())
512 }
513
514 fn write_acl_user(&mut self, acl_user: PxarACLUser) -> Result<(), Error> {
515 self.write_header(PXAR_ACL_USER, std::mem::size_of::<PxarACLUser>() as u64)?;
516 self.write_item(acl_user)?;
517
518 Ok(())
519 }
520
521 fn write_acl_group(&mut self, acl_group: PxarACLGroup) -> Result<(), Error> {
522 self.write_header(PXAR_ACL_GROUP, std::mem::size_of::<PxarACLGroup>() as u64)?;
523 self.write_item(acl_group)?;
524
525 Ok(())
526 }
527
528 fn write_acl_group_obj(&mut self, acl_group_obj: PxarACLGroupObj) -> Result<(), Error> {
529 self.write_header(
530 PXAR_ACL_GROUP_OBJ,
531 std::mem::size_of::<PxarACLGroupObj>() as u64,
532 )?;
533 self.write_item(acl_group_obj)?;
534
535 Ok(())
536 }
537
538 fn write_acl_default(&mut self, acl_default: PxarACLDefault) -> Result<(), Error> {
539 self.write_header(
540 PXAR_ACL_DEFAULT,
541 std::mem::size_of::<PxarACLDefault>() as u64,
542 )?;
543 self.write_item(acl_default)?;
544
545 Ok(())
546 }
547
548 fn write_acl_default_user(&mut self, acl_default_user: PxarACLUser) -> Result<(), Error> {
549 self.write_header(
550 PXAR_ACL_DEFAULT_USER,
551 std::mem::size_of::<PxarACLUser>() as u64,
552 )?;
553 self.write_item(acl_default_user)?;
554
555 Ok(())
556 }
557
558 fn write_acl_default_group(&mut self, acl_default_group: PxarACLGroup) -> Result<(), Error> {
559 self.write_header(
560 PXAR_ACL_DEFAULT_GROUP,
561 std::mem::size_of::<PxarACLGroup>() as u64,
562 )?;
563 self.write_item(acl_default_group)?;
564
565 Ok(())
566 }
567
568 fn write_quota_project_id(&mut self, projid: PxarQuotaProjID) -> Result<(), Error> {
569 self.write_header(
570 PXAR_QUOTA_PROJID,
571 std::mem::size_of::<PxarQuotaProjID>() as u64,
572 )?;
573 self.write_item(projid)?;
574
575 Ok(())
576 }
577
578 fn write_goodbye_table(
579 &mut self,
580 goodbye_offset: usize,
581 goodbye_items: &mut [PxarGoodbyeItem],
582 ) -> Result<(), Error> {
583 goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
584
585 let item_count = goodbye_items.len();
586
587 let goodbye_table_size = (item_count + 1) * std::mem::size_of::<PxarGoodbyeItem>();
588
589 self.write_header(PXAR_GOODBYE, goodbye_table_size as u64)?;
590
591 if self.file_copy_buffer.len() < goodbye_table_size {
592 let need = goodbye_table_size - self.file_copy_buffer.len();
593 self.file_copy_buffer.reserve(need);
594 unsafe {
595 self.file_copy_buffer
596 .set_len(self.file_copy_buffer.capacity());
597 }
598 }
599
600 let buffer = &mut self.file_copy_buffer;
601
602 copy_binary_search_tree(item_count, |s, d| {
603 let item = &goodbye_items[s];
604 let offset = d * std::mem::size_of::<PxarGoodbyeItem>();
605 let dest =
606 crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
607 dest.offset = u64::to_le(item.offset);
608 dest.size = u64::to_le(item.size);
609 dest.hash = u64::to_le(item.hash);
610 });
611
612 // append PxarGoodbyeTail as last item
613 let offset = item_count * std::mem::size_of::<PxarGoodbyeItem>();
614 let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
615 dest.offset = u64::to_le(goodbye_offset as u64);
616 dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<PxarHeader>()) as u64);
617 dest.hash = u64::to_le(PXAR_GOODBYE_TAIL_MARKER);
618
619 self.flush_copy_buffer(goodbye_table_size)?;
620
621 Ok(())
622 }
623
624 fn encode_dir(
625 &mut self,
626 dir: &mut nix::dir::Dir,
627 dir_stat: &FileStat,
628 magic: i64,
629 match_pattern: Vec<MatchPatternSlice>,
630 ) -> Result<(), Error> {
631 //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
632
633 let mut name_list = Vec::new();
634
635 let rawfd = dir.as_raw_fd();
636
637 let dir_start_pos = self.writer_pos;
638
639 let is_root = dir_start_pos == 0;
640
641 let mut dir_entry = self.create_entry(&dir_stat)?;
642
643 self.read_chattr(rawfd, &mut dir_entry)?;
644 self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
645
646 // for each node in the directory tree, the filesystem features are
647 // checked based on the fs magic number.
648 self.fs_feature_flags = flags::feature_flags_from_magic(magic);
649
650 let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
651 let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
652 let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
653 let projid = self.read_quota_project_id(rawfd, magic, &dir_stat)?;
654
655 self.write_entry(dir_entry)?;
656 for xattr in xattrs {
657 self.write_xattr(xattr)?;
658 }
659 self.write_fcaps(fcaps)?;
660
661 for user in acl_access.users {
662 self.write_acl_user(user)?;
663 }
664 for group in acl_access.groups {
665 self.write_acl_group(group)?;
666 }
667 if let Some(group_obj) = acl_access.group_obj {
668 self.write_acl_group_obj(group_obj)?;
669 }
670
671 for default_user in acl_default.users {
672 self.write_acl_default_user(default_user)?;
673 }
674 for default_group in acl_default.groups {
675 self.write_acl_default_group(default_group)?;
676 }
677 if let Some(default) = acl_default.default {
678 self.write_acl_default(default)?;
679 }
680 if let Some(projid) = projid {
681 self.write_quota_project_id(projid)?;
682 }
683
684 let include_children;
685 if is_virtual_file_system(magic) {
686 include_children = false;
687 } else if let Some(set) = &self.device_set {
688 include_children = set.contains(&dir_stat.st_dev);
689 } else {
690 include_children = true;
691 }
692
693 // Expand the exclude match pattern inherited from the parent by local entries, if present
694 let mut local_match_pattern = match_pattern.clone();
695 let (pxar_exclude, excludes) = match MatchPattern::from_file(rawfd, ".pxarexclude") {
696 Ok(Some((excludes, buffer, stat))) => {
697 (Some((buffer, stat)), excludes)
698 }
699 Ok(None) => (None, Vec::new()),
700 Err(nix::Error::Sys(Errno::EACCES)) => {
701 // No permission to read .pxarexclude, ignore its contents.
702 eprintln!(
703 "ignoring match patterns in {:?}: open file failed - EACCES",
704 self.full_path().join(".pxarexclude"),
705 );
706 (None, Vec::new())
707 }
708 Err(err) => bail!("error while reading exclude file - {}", err),
709 };
710 for excl in &excludes {
711 local_match_pattern.push(excl.as_slice());
712 }
713
714 if include_children {
715 // Exclude patterns passed via the CLI are stored as '.pxarexclude-cli'
716 // in the root directory of the archive.
717 if is_root && !match_pattern.is_empty() {
718 let filename = CString::new(".pxarexclude-cli")?;
719 name_list.push((filename, *dir_stat, match_pattern.clone()));
720 }
721
722 for entry in dir.iter() {
723 let entry = entry
724 .map_err(|err| format_err!("readir {:?} failed - {}", self.full_path(), err))?;
725 let filename = entry.file_name().to_owned();
726
727 let name = filename.to_bytes_with_nul();
728 if name == b".\0" || name == b"..\0" {
729 continue;
730 }
731 // Do not store a ".pxarexclude-cli" file found in the archive root,
732 // as this would confilict with new cli passed exclude patterns,
733 // if present.
734 if is_root && name == b".pxarexclude-cli\0" {
735 eprintln!("skip existing '.pxarexclude-cli' in archive root.");
736 continue;
737 }
738
739 let stat = match nix::sys::stat::fstatat(
740 rawfd,
741 filename.as_ref(),
742 nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW,
743 ) {
744 Ok(stat) => stat,
745 Err(nix::Error::Sys(Errno::ENOENT)) => {
746 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
747 self.report_vanished_file(&self.full_path().join(filename_osstr))?;
748 continue;
749 }
750 Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
751 };
752
753 match MatchPatternSlice::match_filename_exclude(
754 &filename,
755 is_directory(&stat),
756 &local_match_pattern,
757 )? {
758 (MatchType::Positive, _) => {
759 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
760 eprintln!(
761 "matched by exclude pattern - skipping: {:?}",
762 self.full_path().join(filename_osstr)
763 );
764 }
765 (_, child_pattern) => {
766 self.entry_counter += 1;
767 name_list.push((filename, stat, child_pattern));
768 }
769 }
770
771 if self.entry_counter > self.entry_max {
772 bail!(
773 "exceeded max number of entries (> {})",
774 self.entry_max
775 );
776 }
777 }
778 } else {
779 eprintln!("skip mount point: {:?}", self.full_path());
780 }
781
782 name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
783 let num_entries = name_list.len();
784
785 let mut goodbye_items = Vec::with_capacity(num_entries);
786
787 for (filename, stat, exclude_list) in name_list {
788 let start_pos = self.writer_pos;
789
790 if filename.as_bytes() == b".pxarexclude" {
791 // pxar_exclude is none in case of error EACCES.
792 if let Some((ref content, ref stat)) = pxar_exclude {
793 let filefd = match nix::fcntl::openat(
794 rawfd,
795 filename.as_ref(),
796 OFlag::O_NOFOLLOW,
797 Mode::empty(),
798 ) {
799 Ok(filefd) => filefd,
800 Err(nix::Error::Sys(Errno::ENOENT)) => {
801 self.report_vanished_file(&self.full_path())?;
802 continue;
803 }
804 Err(nix::Error::Sys(Errno::EACCES)) => {
805 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
806 eprintln!(
807 "skipping {:?}: open file failed - EACCES",
808 self.full_path().join(filename_osstr),
809 );
810 continue;
811 }
812 Err(err) => {
813 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
814 bail!(
815 "open file {:?} failed - {}",
816 self.full_path().join(filename_osstr),
817 err
818 );
819 }
820 };
821
822 let child_magic = if dir_stat.st_dev != stat.st_dev {
823 detect_fs_type(filefd)?
824 } else {
825 magic
826 };
827
828 self.write_filename(&filename)?;
829 if let Some(ref mut catalog) = self.catalog {
830 catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
831 }
832 self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
833 }
834 continue;
835 }
836
837 if is_root && filename.as_bytes() == b".pxarexclude-cli" {
838 // '.pxarexclude-cli' is used to store the exclude MatchPatterns
839 // passed via the cli in the root directory of the archive.
840 self.write_filename(&filename)?;
841 let content = MatchPatternSlice::to_bytes(&exclude_list);
842 if let Some(ref mut catalog) = self.catalog {
843 catalog.add_file(&filename, content.len() as u64, 0)?;
844 }
845 self.encode_pxar_exclude_cli(stat.st_uid, stat.st_gid, 0, &content)?;
846 continue;
847 }
848
849 self.relative_path
850 .push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
851
852 if self.verbose {
853 println!("{:?}", self.full_path());
854 }
855
856 if is_directory(&stat) {
857 let mut dir = match nix::dir::Dir::openat(
858 rawfd,
859 filename.as_ref(),
860 OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW,
861 Mode::empty(),
862 ) {
863 Ok(dir) => dir,
864 Err(nix::Error::Sys(Errno::ENOENT)) => {
865 self.report_vanished_file(&self.full_path())?;
866 self.relative_path.pop();
867 continue;
868 }
869 Err(nix::Error::Sys(Errno::EACCES)) => {
870 eprintln!(
871 "skipping {:?}: open dir failed - EACCES",
872 self.full_path(),
873 );
874 self.relative_path.pop();
875 continue;
876 }
877 Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
878 };
879
880 let child_magic = if dir_stat.st_dev != stat.st_dev {
881 detect_fs_type(dir.as_raw_fd())?
882 } else {
883 magic
884 };
885
886 self.write_filename(&filename)?;
887 if let Some(ref mut catalog) = self.catalog {
888 catalog.start_directory(&filename)?;
889 }
890 self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
891 if let Some(ref mut catalog) = self.catalog {
892 catalog.end_directory()?;
893 }
894 } else if is_reg_file(&stat) {
895 let mut hardlink_target = None;
896
897 if stat.st_nlink > 1 {
898 let link_info = HardLinkInfo {
899 st_dev: stat.st_dev,
900 st_ino: stat.st_ino,
901 };
902 hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
903 let mut target = v.clone().into_os_string();
904 target.push("\0"); // add Nul byte
905 (target, (start_pos as u64) - offset)
906 });
907 if hardlink_target == None {
908 self.hardlinks
909 .insert(link_info, (self.relative_path.clone(), start_pos as u64));
910 }
911 }
912
913 if let Some((target, offset)) = hardlink_target {
914 if let Some(ref mut catalog) = self.catalog {
915 catalog.add_hardlink(&filename)?;
916 }
917 self.write_filename(&filename)?;
918 self.encode_hardlink(target.as_bytes(), offset)?;
919 } else {
920 let filefd = match nix::fcntl::openat(
921 rawfd,
922 filename.as_ref(),
923 OFlag::O_NOFOLLOW,
924 Mode::empty(),
925 ) {
926 Ok(filefd) => filefd,
927 Err(nix::Error::Sys(Errno::ENOENT)) => {
928 self.report_vanished_file(&self.full_path())?;
929 self.relative_path.pop();
930 continue;
931 }
932 Err(nix::Error::Sys(Errno::EACCES)) => {
933 eprintln!(
934 "skipping {:?}: open file failed - EACCES",
935 self.full_path(),
936 );
937 self.relative_path.pop();
938 continue;
939 }
940 Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
941 };
942
943 if let Some(ref mut catalog) = self.catalog {
944 catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
945 }
946 let child_magic = if dir_stat.st_dev != stat.st_dev {
947 detect_fs_type(filefd)?
948 } else {
949 magic
950 };
951
952 self.write_filename(&filename)?;
953 let res = self.encode_file(filefd, &stat, child_magic);
954 let _ = nix::unistd::close(filefd); // ignore close errors
955 res?;
956 }
957 } else if is_symlink(&stat) {
958 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
959
960 let res = filename.with_nix_path(|cstr| unsafe {
961 libc::readlinkat(
962 rawfd,
963 cstr.as_ptr(),
964 buffer.as_mut_ptr() as *mut libc::c_char,
965 buffer.len() - 1,
966 )
967 })?;
968
969 match Errno::result(res) {
970 Ok(len) => {
971 if let Some(ref mut catalog) = self.catalog {
972 catalog.add_symlink(&filename)?;
973 }
974 buffer[len as usize] = 0u8; // add Nul byte
975 self.write_filename(&filename)?;
976 self.encode_symlink(&buffer[..((len + 1) as usize)], &stat)?
977 }
978 Err(nix::Error::Sys(Errno::ENOENT)) => {
979 self.report_vanished_file(&self.full_path())?;
980 self.relative_path.pop();
981 continue;
982 }
983 Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
984 }
985 } else if is_block_dev(&stat) || is_char_dev(&stat) {
986 if self.has_features(flags::WITH_DEVICE_NODES) {
987 if let Some(ref mut catalog) = self.catalog {
988 if is_block_dev(&stat) {
989 catalog.add_block_device(&filename)?;
990 } else {
991 catalog.add_char_device(&filename)?;
992 }
993 }
994 self.write_filename(&filename)?;
995 self.encode_device(&stat)?;
996 } else {
997 eprintln!("skip device node: {:?}", self.full_path());
998 self.relative_path.pop();
999 continue;
1000 }
1001 } else if is_fifo(&stat) {
1002 if self.has_features(flags::WITH_FIFOS) {
1003 if let Some(ref mut catalog) = self.catalog {
1004 catalog.add_fifo(&filename)?;
1005 }
1006 self.write_filename(&filename)?;
1007 self.encode_special(&stat)?;
1008 } else {
1009 eprintln!("skip fifo: {:?}", self.full_path());
1010 self.relative_path.pop();
1011 continue;
1012 }
1013 } else if is_socket(&stat) {
1014 if self.has_features(flags::WITH_SOCKETS) {
1015 if let Some(ref mut catalog) = self.catalog {
1016 catalog.add_socket(&filename)?;
1017 }
1018 self.write_filename(&filename)?;
1019 self.encode_special(&stat)?;
1020 } else {
1021 eprintln!("skip socket: {:?}", self.full_path());
1022 self.relative_path.pop();
1023 continue;
1024 }
1025 } else {
1026 bail!(
1027 "unsupported file type (mode {:o} {:?})",
1028 stat.st_mode,
1029 self.full_path()
1030 );
1031 }
1032
1033 let end_pos = self.writer_pos;
1034
1035 goodbye_items.push(PxarGoodbyeItem {
1036 offset: start_pos as u64,
1037 size: (end_pos - start_pos) as u64,
1038 hash: compute_goodbye_hash(filename.to_bytes()),
1039 });
1040
1041 self.relative_path.pop();
1042 }
1043
1044 //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
1045
1046 // fixup goodby item offsets
1047 let goodbye_start = self.writer_pos as u64;
1048 for item in &mut goodbye_items {
1049 item.offset = goodbye_start - item.offset;
1050 }
1051
1052 let goodbye_offset = self.writer_pos - dir_start_pos;
1053
1054 self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
1055 self.entry_counter -= num_entries;
1056
1057 //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
1058 Ok(())
1059 }
1060
1061 fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64) -> Result<(), Error> {
1062 //println!("encode_file: {:?}", self.full_path());
1063
1064 let mut entry = self.create_entry(&stat)?;
1065
1066 self.read_chattr(filefd, &mut entry)?;
1067 self.read_fat_attr(filefd, magic, &mut entry)?;
1068 let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1069 let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1070 let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1071
1072 self.write_entry(entry)?;
1073 for xattr in xattrs {
1074 self.write_xattr(xattr)?;
1075 }
1076 self.write_fcaps(fcaps)?;
1077 for user in acl_access.users {
1078 self.write_acl_user(user)?;
1079 }
1080 for group in acl_access.groups {
1081 self.write_acl_group(group)?;
1082 }
1083 if let Some(group_obj) = acl_access.group_obj {
1084 self.write_acl_group_obj(group_obj)?;
1085 }
1086 if let Some(projid) = projid {
1087 self.write_quota_project_id(projid)?;
1088 }
1089
1090 let include_payload;
1091 if is_virtual_file_system(magic) {
1092 include_payload = false;
1093 } else if let Some(ref set) = &self.device_set {
1094 include_payload = set.contains(&stat.st_dev);
1095 } else {
1096 include_payload = true;
1097 }
1098
1099 if !include_payload {
1100 eprintln!("skip content: {:?}", self.full_path());
1101 self.write_header(PXAR_PAYLOAD, 0)?;
1102 return Ok(());
1103 }
1104
1105 let size = stat.st_size as u64;
1106
1107 self.write_header(PXAR_PAYLOAD, size)?;
1108
1109 let mut pos: u64 = 0;
1110 loop {
1111 let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
1112 Ok(n) => n,
1113 Err(nix::Error::Sys(Errno::EINTR)) => continue, /* try again */
1114 Err(err) => bail!("read {:?} failed - {}", self.full_path(), err),
1115 };
1116 if n == 0 { // EOF
1117 if pos != size {
1118 // Note:: casync format cannot handle that
1119 bail!(
1120 "detected shrunk file {:?} ({} < {})",
1121 self.full_path(),
1122 pos,
1123 size
1124 );
1125 }
1126 break;
1127 }
1128
1129 let mut next = pos + (n as u64);
1130
1131 if next > size {
1132 next = size;
1133 }
1134
1135 let count = (next - pos) as usize;
1136
1137 self.flush_copy_buffer(count)?;
1138
1139 pos = next;
1140
1141 if pos >= size {
1142 break;
1143 }
1144 }
1145
1146 Ok(())
1147 }
1148
1149 fn encode_device(&mut self, stat: &FileStat) -> Result<(), Error> {
1150 let entry = self.create_entry(&stat)?;
1151
1152 self.write_entry(entry)?;
1153
1154 let major = unsafe { libc::major(stat.st_rdev) } as u64;
1155 let minor = unsafe { libc::minor(stat.st_rdev) } as u64;
1156
1157 //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
1158
1159 self.write_header(PXAR_DEVICE, std::mem::size_of::<PxarDevice>() as u64)?;
1160 self.write_item(PxarDevice { major, minor })?;
1161
1162 Ok(())
1163 }
1164
1165 // FIFO or Socket
1166 fn encode_special(&mut self, stat: &FileStat) -> Result<(), Error> {
1167 let entry = self.create_entry(&stat)?;
1168
1169 self.write_entry(entry)?;
1170
1171 Ok(())
1172 }
1173
1174 fn encode_symlink(&mut self, target: &[u8], stat: &FileStat) -> Result<(), Error> {
1175 //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
1176
1177 let entry = self.create_entry(&stat)?;
1178 self.write_entry(entry)?;
1179
1180 self.write_header(PXAR_SYMLINK, target.len() as u64)?;
1181 self.write(target)?;
1182
1183 Ok(())
1184 }
1185
1186 fn encode_hardlink(&mut self, target: &[u8], offset: u64) -> Result<(), Error> {
1187 //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
1188
1189 // Note: HARDLINK replaces an ENTRY.
1190 self.write_header(PXAR_FORMAT_HARDLINK, (target.len() as u64) + 8)?;
1191 self.write_item(offset)?;
1192 self.write(target)?;
1193
1194 Ok(())
1195 }
1196
1197 fn encode_pxar_exclude(
1198 &mut self,
1199 filefd: RawFd,
1200 stat: &FileStat,
1201 magic: i64,
1202 content: &[u8],
1203 ) -> Result<(), Error> {
1204 let mut entry = self.create_entry(&stat)?;
1205
1206 self.read_chattr(filefd, &mut entry)?;
1207 self.read_fat_attr(filefd, magic, &mut entry)?;
1208 let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1209 let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1210 let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1211
1212 self.write_entry(entry)?;
1213 for xattr in xattrs {
1214 self.write_xattr(xattr)?;
1215 }
1216 self.write_fcaps(fcaps)?;
1217 for user in acl_access.users {
1218 self.write_acl_user(user)?;
1219 }
1220 for group in acl_access.groups {
1221 self.write_acl_group(group)?;
1222 }
1223 if let Some(group_obj) = acl_access.group_obj {
1224 self.write_acl_group_obj(group_obj)?;
1225 }
1226 if let Some(projid) = projid {
1227 self.write_quota_project_id(projid)?;
1228 }
1229
1230 let include_payload;
1231 if is_virtual_file_system(magic) {
1232 include_payload = false;
1233 } else if let Some(set) = &self.device_set {
1234 include_payload = set.contains(&stat.st_dev);
1235 } else {
1236 include_payload = true;
1237 }
1238
1239 if !include_payload {
1240 eprintln!("skip content: {:?}", self.full_path());
1241 self.write_header(PXAR_PAYLOAD, 0)?;
1242 return Ok(());
1243 }
1244
1245 let size = content.len();
1246 self.write_header(PXAR_PAYLOAD, size as u64)?;
1247 self.writer.write_all(content)?;
1248 self.writer_pos += size;
1249
1250 Ok(())
1251 }
1252
1253 /// Encodes the excude match patterns passed via cli as file in the archive.
1254 fn encode_pxar_exclude_cli(
1255 &mut self,
1256 uid: u32,
1257 gid: u32,
1258 mtime: u64,
1259 content: &[u8],
1260 ) -> Result<(), Error> {
1261 let entry = PxarEntry {
1262 mode: (libc::S_IFREG | 0o600) as u64,
1263 flags: 0,
1264 uid,
1265 gid,
1266 mtime,
1267 };
1268 self.write_entry(entry)?;
1269 let size = content.len();
1270 self.write_header(PXAR_PAYLOAD, size as u64)?;
1271 self.writer.write_all(content)?;
1272 self.writer_pos += size;
1273
1274 Ok(())
1275 }
1276
1277 // the report_XXX method may raise and error - depending on encoder configuration
1278
1279 fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
1280 eprintln!("WARNING: detected vanished file {:?}", path);
1281
1282 Ok(())
1283 }
1284 }
1285
1286 fn errno_is_unsupported(errno: Errno) -> bool {
1287 match errno {
1288 Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => true,
1289 _ => false,
1290 }
1291 }
1292
1293 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
1294 let mut fs_stat = std::mem::MaybeUninit::uninit();
1295 let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
1296 Errno::result(res)?;
1297 let fs_stat = unsafe { fs_stat.assume_init() };
1298
1299 Ok(fs_stat.f_type)
1300 }
1301
1302 #[inline(always)]
1303 pub fn is_temporary_file_system(magic: i64) -> bool {
1304 use proxmox::sys::linux::magic::*;
1305 magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
1306 }
1307
1308 pub fn is_virtual_file_system(magic: i64) -> bool {
1309 use proxmox::sys::linux::magic::*;
1310
1311 match magic {
1312 BINFMTFS_MAGIC |
1313 CGROUP2_SUPER_MAGIC |
1314 CGROUP_SUPER_MAGIC |
1315 CONFIGFS_MAGIC |
1316 DEBUGFS_MAGIC |
1317 DEVPTS_SUPER_MAGIC |
1318 EFIVARFS_MAGIC |
1319 FUSE_CTL_SUPER_MAGIC |
1320 HUGETLBFS_MAGIC |
1321 MQUEUE_MAGIC |
1322 NFSD_MAGIC |
1323 PROC_SUPER_MAGIC |
1324 PSTOREFS_MAGIC |
1325 RPCAUTH_GSSMAGIC |
1326 SECURITYFS_MAGIC |
1327 SELINUX_MAGIC |
1328 SMACK_MAGIC |
1329 SYSFS_MAGIC => true,
1330 _ => false
1331 }
1332 }