]> git.proxmox.com Git - proxmox-backup.git/blob - src/pxar/encoder.rs
avoid some clippy warnings
[proxmox-backup.git] / src / pxar / encoder.rs
1 //! *pxar* format encoder.
2 //!
3 //! This module contain the code to generate *pxar* archive files.
4 use std::collections::{HashMap, HashSet};
5 use std::ffi::{CStr, CString};
6 use std::io::Write;
7 use std::os::unix::ffi::OsStrExt;
8 use std::os::unix::io::AsRawFd;
9 use std::os::unix::io::RawFd;
10 use std::path::{Path, PathBuf};
11
12 use endian_trait::Endian;
13 use failure::*;
14 use nix::errno::Errno;
15 use nix::fcntl::OFlag;
16 use nix::sys::stat::FileStat;
17 use nix::sys::stat::Mode;
18 use nix::NixPath;
19
20 use proxmox::tools::vec;
21
22 use super::binary_search_tree::*;
23 use super::catalog::BackupCatalogWriter;
24 use super::flags;
25 use super::format_definition::*;
26 use super::helper::*;
27 use super::match_pattern::{MatchPattern, MatchType};
28 use crate::tools::acl;
29 use crate::tools::fs;
30 use crate::tools::xattr;
31
32 /// The format requires to build sorted directory lookup tables in
33 /// memory, so we restrict the number of allowed entries to limit
34 /// maximum memory usage.
35 pub const MAX_DIRECTORY_ENTRIES: usize = 256 * 1024;
36
37 #[derive(Eq, PartialEq, Hash)]
38 struct HardLinkInfo {
39 st_dev: u64,
40 st_ino: u64,
41 }
42
43 pub struct Encoder<'a, W: Write, C: BackupCatalogWriter> {
44 base_path: PathBuf,
45 relative_path: PathBuf,
46 writer: &'a mut W,
47 writer_pos: usize,
48 catalog: Option<&'a mut C>,
49 _size: usize,
50 file_copy_buffer: Vec<u8>,
51 device_set: Option<HashSet<u64>>,
52 verbose: bool,
53 // Flags set by the user
54 feature_flags: u64,
55 // Flags signaling features supported by the filesystem
56 fs_feature_flags: u64,
57 hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
58 }
59
60 impl<'a, W: Write, C: BackupCatalogWriter> Encoder<'a, W, C> {
61 // used for error reporting
62 fn full_path(&self) -> PathBuf {
63 self.base_path.join(&self.relative_path)
64 }
65
66 /// Create archive, write result data to ``writer``.
67 ///
68 /// The ``device_set`` can be use used to limit included mount points.
69 ///
70 /// - ``None``: include all mount points
71 /// - ``Some(set)``: only include devices listed in this set (the
72 /// root path device is automathically added to this list, so
73 /// you can pass an empty set if you want to archive a single
74 /// mount point.)
75 pub fn encode(
76 path: PathBuf,
77 dir: &mut nix::dir::Dir,
78 writer: &'a mut W,
79 catalog: Option<&'a mut C>,
80 device_set: Option<HashSet<u64>>,
81 verbose: bool,
82 skip_lost_and_found: bool, // fixme: should be a feature flag ??
83 feature_flags: u64,
84 mut excludes: Vec<MatchPattern>,
85 ) -> Result<(), Error> {
86 const FILE_COPY_BUFFER_SIZE: usize = 1024 * 1024;
87
88 let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
89 unsafe {
90 file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE);
91 }
92
93 // todo: use scandirat??
94
95 let dir_fd = dir.as_raw_fd();
96 let stat = nix::sys::stat::fstat(dir_fd)
97 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
98
99 if !is_directory(&stat) {
100 bail!("got unexpected file type {:?} (not a directory)", path);
101 }
102
103 let mut device_set = device_set.clone();
104 if let Some(ref mut set) = device_set {
105 set.insert(stat.st_dev);
106 }
107
108 let magic = detect_fs_type(dir_fd)?;
109
110 if is_virtual_file_system(magic) {
111 bail!("backup virtual file systems is disabled!");
112 }
113
114 let fs_feature_flags = flags::feature_flags_from_magic(magic);
115
116 let mut me = Self {
117 base_path: path,
118 relative_path: PathBuf::new(),
119 writer,
120 writer_pos: 0,
121 catalog,
122 _size: 0,
123 file_copy_buffer,
124 device_set,
125 verbose,
126 feature_flags,
127 fs_feature_flags,
128 hardlinks: HashMap::new(),
129 };
130
131 if verbose {
132 println!("{:?}", me.full_path());
133 }
134
135 if skip_lost_and_found {
136 excludes.push(MatchPattern::from_line(b"**/lost+found").unwrap().unwrap());
137 }
138
139 me.encode_dir(dir, &stat, magic, excludes)?;
140
141 Ok(())
142 }
143
144 fn write(&mut self, buf: &[u8]) -> Result<(), Error> {
145 self.writer.write_all(buf)?;
146 self.writer_pos += buf.len();
147 Ok(())
148 }
149
150 fn write_item<T: Endian>(&mut self, item: T) -> Result<(), Error> {
151 let data = item.to_le();
152
153 let buffer = unsafe {
154 std::slice::from_raw_parts(&data as *const T as *const u8, std::mem::size_of::<T>())
155 };
156
157 self.write(buffer)?;
158
159 Ok(())
160 }
161
162 fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
163 self.writer.write_all(&self.file_copy_buffer[..size])?;
164 self.writer_pos += size;
165 Ok(())
166 }
167
168 fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
169 let size = size + (std::mem::size_of::<PxarHeader>() as u64);
170 self.write_item(PxarHeader { size, htype })?;
171
172 Ok(())
173 }
174
175 fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
176 let buffer = name.to_bytes_with_nul();
177 self.write_header(PXAR_FILENAME, buffer.len() as u64)?;
178 self.write(buffer)?;
179
180 Ok(())
181 }
182
183 fn create_entry(&self, stat: &FileStat) -> Result<PxarEntry, Error> {
184 let mode = if is_symlink(&stat) {
185 (libc::S_IFLNK | 0o777) as u64
186 } else {
187 (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
188 };
189
190 let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
191 if mtime < 0 {
192 bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
193 }
194
195 let entry = PxarEntry {
196 mode,
197 flags: 0,
198 uid: stat.st_uid,
199 gid: stat.st_gid,
200 mtime: mtime as u64,
201 };
202
203 Ok(entry)
204 }
205
206 fn read_chattr(&self, fd: RawFd, entry: &mut PxarEntry) -> Result<(), Error> {
207 let mut attr: usize = 0;
208
209 let res = unsafe { fs::read_attr_fd(fd, &mut attr) };
210 if let Err(err) = res {
211 if let nix::Error::Sys(errno) = err {
212 if errno_is_unsupported(errno) {
213 return Ok(());
214 };
215 }
216 bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
217 }
218
219 let flags = flags::feature_flags_from_chattr(attr as u32);
220 entry.flags |= flags;
221
222 Ok(())
223 }
224
225 fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut PxarEntry) -> Result<(), Error> {
226 use proxmox::sys::linux::magic::*;
227
228 if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC {
229 return Ok(());
230 }
231
232 let mut attr: u32 = 0;
233
234 let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr) };
235 if let Err(err) = res {
236 if let nix::Error::Sys(errno) = err {
237 if errno_is_unsupported(errno) {
238 return Ok(());
239 };
240 }
241 bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
242 }
243
244 let flags = flags::feature_flags_from_fat_attr(attr);
245 entry.flags |= flags;
246
247 Ok(())
248 }
249
250 /// True if all of the given feature flags are set in the Encoder, false otherwise
251 fn has_features(&self, feature_flags: u64) -> bool {
252 (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
253 }
254
255 /// True if at least one of the given feature flags is set in the Encoder, false otherwise
256 fn has_some_features(&self, feature_flags: u64) -> bool {
257 (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
258 }
259
260 fn read_xattrs(
261 &self,
262 fd: RawFd,
263 stat: &FileStat,
264 ) -> Result<(Vec<PxarXAttr>, Option<PxarFCaps>), Error> {
265 let mut xattrs = Vec::new();
266 let mut fcaps = None;
267
268 let flags = flags::WITH_XATTRS | flags::WITH_FCAPS;
269 if !self.has_some_features(flags) {
270 return Ok((xattrs, fcaps));
271 }
272 // Should never be called on symlinks, just in case check anyway
273 if is_symlink(&stat) {
274 return Ok((xattrs, fcaps));
275 }
276
277 let xattr_names = match xattr::flistxattr(fd) {
278 Ok(names) => names,
279 // Do not bail if the underlying endpoint does not supports xattrs
280 Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
281 // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
282 Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
283 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
284 };
285
286 for name in xattr_names.split(|c| *c == b'\0') {
287 // Only extract the relevant extended attributes
288 if !xattr::is_valid_xattr_name(&name) {
289 continue;
290 }
291
292 let value = match xattr::fgetxattr(fd, name) {
293 Ok(value) => value,
294 // Vanished between flistattr and getxattr, this is ok, silently ignore
295 Err(Errno::ENODATA) => continue,
296 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
297 };
298
299 if xattr::is_security_capability(&name) {
300 if self.has_features(flags::WITH_FCAPS) {
301 // fcaps are stored in own format within the archive
302 fcaps = Some(PxarFCaps { data: value });
303 }
304 } else if self.has_features(flags::WITH_XATTRS) {
305 xattrs.push(PxarXAttr {
306 name: name.to_vec(),
307 value,
308 });
309 }
310 }
311 xattrs.sort();
312
313 Ok((xattrs, fcaps))
314 }
315
316 fn read_acl(
317 &self,
318 fd: RawFd,
319 stat: &FileStat,
320 acl_type: acl::ACLType,
321 ) -> Result<PxarACL, Error> {
322 let ret = PxarACL {
323 users: Vec::new(),
324 groups: Vec::new(),
325 group_obj: None,
326 default: None,
327 };
328
329 if !self.has_features(flags::WITH_ACL) {
330 return Ok(ret);
331 }
332 if is_symlink(&stat) {
333 return Ok(ret);
334 }
335 if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
336 bail!("ACL_TYPE_DEFAULT only defined for directories.");
337 }
338
339 // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
340 // to create a path for acl_get_file(). acl_get_fd() only allows to get
341 // ACL_TYPE_ACCESS attributes.
342 let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
343 let acl = match acl::ACL::get_file(&proc_path, acl_type) {
344 Ok(acl) => acl,
345 // Don't bail if underlying endpoint does not support acls
346 Err(Errno::EOPNOTSUPP) => return Ok(ret),
347 // Don't bail if the endpoint cannot carry acls
348 Err(Errno::EBADF) => return Ok(ret),
349 // Don't bail if there is no data
350 Err(Errno::ENODATA) => return Ok(ret),
351 Err(err) => bail!("error while reading ACL - {}", err),
352 };
353
354 self.process_acl(acl, acl_type)
355 }
356
357 fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
358 let mut acl_user = Vec::new();
359 let mut acl_group = Vec::new();
360 let mut acl_group_obj = None;
361 let mut acl_default = None;
362 let mut user_obj_permissions = None;
363 let mut group_obj_permissions = None;
364 let mut other_permissions = None;
365 let mut mask_permissions = None;
366
367 for entry in &mut acl.entries() {
368 let tag = entry.get_tag_type()?;
369 let permissions = entry.get_permissions()?;
370 match tag {
371 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
372 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
373 acl::ACL_OTHER => other_permissions = Some(permissions),
374 acl::ACL_MASK => mask_permissions = Some(permissions),
375 acl::ACL_USER => {
376 acl_user.push(PxarACLUser {
377 uid: entry.get_qualifier()?,
378 permissions,
379 });
380 }
381 acl::ACL_GROUP => {
382 acl_group.push(PxarACLGroup {
383 gid: entry.get_qualifier()?,
384 permissions,
385 });
386 }
387 _ => bail!("Unexpected ACL tag encountered!"),
388 }
389 }
390
391 acl_user.sort();
392 acl_group.sort();
393
394 match acl_type {
395 acl::ACL_TYPE_ACCESS => {
396 // The mask permissions are mapped to the stat group permissions
397 // in case that the ACL group permissions were set.
398 // Only in that case we need to store the group permissions,
399 // in the other cases they are identical to the stat group permissions.
400 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
401 acl_group_obj = Some(PxarACLGroupObj { permissions: gop });
402 }
403 }
404 acl::ACL_TYPE_DEFAULT => {
405 if user_obj_permissions != None
406 || group_obj_permissions != None
407 || other_permissions != None
408 || mask_permissions != None
409 {
410 acl_default = Some(PxarACLDefault {
411 // The value is set to UINT64_MAX as placeholder if one
412 // of the permissions is not set
413 user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
414 group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
415 other_permissions: other_permissions.unwrap_or(std::u64::MAX),
416 mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
417 });
418 }
419 }
420 _ => bail!("Unexpected ACL type encountered"),
421 }
422
423 Ok(PxarACL {
424 users: acl_user,
425 groups: acl_group,
426 group_obj: acl_group_obj,
427 default: acl_default,
428 })
429 }
430
431 /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
432 fn read_quota_project_id(
433 &self,
434 fd: RawFd,
435 magic: i64,
436 stat: &FileStat,
437 ) -> Result<Option<PxarQuotaProjID>, Error> {
438 if !(is_directory(&stat) || is_reg_file(&stat)) {
439 return Ok(None);
440 }
441 if !self.has_features(flags::WITH_QUOTA_PROJID) {
442 return Ok(None);
443 }
444
445 use proxmox::sys::linux::magic::*;
446
447 match magic {
448 EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC | ZFS_SUPER_MAGIC => {
449 let mut fsxattr = fs::FSXAttr::default();
450 let res = unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) };
451
452 // On some FUSE filesystems it can happen that ioctl is not supported.
453 // For these cases projid is set to 0 while the error is ignored.
454 if let Err(err) = res {
455 let errno = err.as_errno().ok_or_else(|| {
456 format_err!(
457 "error while reading quota project id for {:#?}",
458 self.full_path()
459 )
460 })?;
461 if errno_is_unsupported(errno) {
462 return Ok(None);
463 } else {
464 bail!(
465 "error while reading quota project id for {:#?} - {}",
466 self.full_path(),
467 errno
468 );
469 }
470 }
471
472 let projid = fsxattr.fsx_projid as u64;
473 if projid == 0 {
474 Ok(None)
475 } else {
476 Ok(Some(PxarQuotaProjID { projid }))
477 }
478 }
479 _ => Ok(None),
480 }
481 }
482
483 fn write_entry(&mut self, entry: PxarEntry) -> Result<(), Error> {
484 self.write_header(PXAR_ENTRY, std::mem::size_of::<PxarEntry>() as u64)?;
485 self.write_item(entry)?;
486
487 Ok(())
488 }
489
490 fn write_xattr(&mut self, xattr: PxarXAttr) -> Result<(), Error> {
491 let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
492 self.write_header(PXAR_XATTR, size as u64)?;
493 self.write(xattr.name.as_slice())?;
494 self.write(&[0])?;
495 self.write(xattr.value.as_slice())?;
496
497 Ok(())
498 }
499
500 fn write_fcaps(&mut self, fcaps: Option<PxarFCaps>) -> Result<(), Error> {
501 if let Some(fcaps) = fcaps {
502 let size = fcaps.data.len();
503 self.write_header(PXAR_FCAPS, size as u64)?;
504 self.write(fcaps.data.as_slice())?;
505 }
506
507 Ok(())
508 }
509
510 fn write_acl_user(&mut self, acl_user: PxarACLUser) -> Result<(), Error> {
511 self.write_header(PXAR_ACL_USER, std::mem::size_of::<PxarACLUser>() as u64)?;
512 self.write_item(acl_user)?;
513
514 Ok(())
515 }
516
517 fn write_acl_group(&mut self, acl_group: PxarACLGroup) -> Result<(), Error> {
518 self.write_header(PXAR_ACL_GROUP, std::mem::size_of::<PxarACLGroup>() as u64)?;
519 self.write_item(acl_group)?;
520
521 Ok(())
522 }
523
524 fn write_acl_group_obj(&mut self, acl_group_obj: PxarACLGroupObj) -> Result<(), Error> {
525 self.write_header(
526 PXAR_ACL_GROUP_OBJ,
527 std::mem::size_of::<PxarACLGroupObj>() as u64,
528 )?;
529 self.write_item(acl_group_obj)?;
530
531 Ok(())
532 }
533
534 fn write_acl_default(&mut self, acl_default: PxarACLDefault) -> Result<(), Error> {
535 self.write_header(
536 PXAR_ACL_DEFAULT,
537 std::mem::size_of::<PxarACLDefault>() as u64,
538 )?;
539 self.write_item(acl_default)?;
540
541 Ok(())
542 }
543
544 fn write_acl_default_user(&mut self, acl_default_user: PxarACLUser) -> Result<(), Error> {
545 self.write_header(
546 PXAR_ACL_DEFAULT_USER,
547 std::mem::size_of::<PxarACLUser>() as u64,
548 )?;
549 self.write_item(acl_default_user)?;
550
551 Ok(())
552 }
553
554 fn write_acl_default_group(&mut self, acl_default_group: PxarACLGroup) -> Result<(), Error> {
555 self.write_header(
556 PXAR_ACL_DEFAULT_GROUP,
557 std::mem::size_of::<PxarACLGroup>() as u64,
558 )?;
559 self.write_item(acl_default_group)?;
560
561 Ok(())
562 }
563
564 fn write_quota_project_id(&mut self, projid: PxarQuotaProjID) -> Result<(), Error> {
565 self.write_header(
566 PXAR_QUOTA_PROJID,
567 std::mem::size_of::<PxarQuotaProjID>() as u64,
568 )?;
569 self.write_item(projid)?;
570
571 Ok(())
572 }
573
574 fn write_goodbye_table(
575 &mut self,
576 goodbye_offset: usize,
577 goodbye_items: &mut [PxarGoodbyeItem],
578 ) -> Result<(), Error> {
579 goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
580
581 let item_count = goodbye_items.len();
582
583 let goodbye_table_size = (item_count + 1) * std::mem::size_of::<PxarGoodbyeItem>();
584
585 self.write_header(PXAR_GOODBYE, goodbye_table_size as u64)?;
586
587 if self.file_copy_buffer.len() < goodbye_table_size {
588 let need = goodbye_table_size - self.file_copy_buffer.len();
589 self.file_copy_buffer.reserve(need);
590 unsafe {
591 self.file_copy_buffer
592 .set_len(self.file_copy_buffer.capacity());
593 }
594 }
595
596 let buffer = &mut self.file_copy_buffer;
597
598 copy_binary_search_tree(item_count, |s, d| {
599 let item = &goodbye_items[s];
600 let offset = d * std::mem::size_of::<PxarGoodbyeItem>();
601 let dest =
602 crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
603 dest.offset = u64::to_le(item.offset);
604 dest.size = u64::to_le(item.size);
605 dest.hash = u64::to_le(item.hash);
606 });
607
608 // append PxarGoodbyeTail as last item
609 let offset = item_count * std::mem::size_of::<PxarGoodbyeItem>();
610 let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
611 dest.offset = u64::to_le(goodbye_offset as u64);
612 dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<PxarHeader>()) as u64);
613 dest.hash = u64::to_le(PXAR_GOODBYE_TAIL_MARKER);
614
615 self.flush_copy_buffer(goodbye_table_size)?;
616
617 Ok(())
618 }
619
620 fn encode_dir(
621 &mut self,
622 dir: &mut nix::dir::Dir,
623 dir_stat: &FileStat,
624 magic: i64,
625 match_pattern: Vec<MatchPattern>,
626 ) -> Result<(), Error> {
627 //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
628
629 let mut name_list = vec![];
630
631 let rawfd = dir.as_raw_fd();
632
633 let dir_start_pos = self.writer_pos;
634
635 let is_root = dir_start_pos == 0;
636
637 let mut dir_entry = self.create_entry(&dir_stat)?;
638
639 self.read_chattr(rawfd, &mut dir_entry)?;
640 self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
641
642 // for each node in the directory tree, the filesystem features are
643 // checked based on the fs magic number.
644 self.fs_feature_flags = flags::feature_flags_from_magic(magic);
645
646 let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
647 let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
648 let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
649 let projid = self.read_quota_project_id(rawfd, magic, &dir_stat)?;
650
651 self.write_entry(dir_entry)?;
652 for xattr in xattrs {
653 self.write_xattr(xattr)?;
654 }
655 self.write_fcaps(fcaps)?;
656
657 for user in acl_access.users {
658 self.write_acl_user(user)?;
659 }
660 for group in acl_access.groups {
661 self.write_acl_group(group)?;
662 }
663 if let Some(group_obj) = acl_access.group_obj {
664 self.write_acl_group_obj(group_obj)?;
665 }
666
667 for default_user in acl_default.users {
668 self.write_acl_default_user(default_user)?;
669 }
670 for default_group in acl_default.groups {
671 self.write_acl_default_group(default_group)?;
672 }
673 if let Some(default) = acl_default.default {
674 self.write_acl_default(default)?;
675 }
676 if let Some(projid) = projid {
677 self.write_quota_project_id(projid)?;
678 }
679
680 let include_children;
681 if is_virtual_file_system(magic) {
682 include_children = false;
683 } else if let Some(set) = &self.device_set {
684 include_children = set.contains(&dir_stat.st_dev);
685 } else {
686 include_children = true;
687 }
688
689 // Expand the exclude match pattern inherited from the parent by local entries, if present
690 let mut local_match_pattern = match_pattern.clone();
691 let pxar_exclude = match MatchPattern::from_file(rawfd, ".pxarexclude") {
692 Ok(Some((mut excludes, buffer, stat))) => {
693 local_match_pattern.append(&mut excludes);
694 Some((buffer, stat))
695 }
696 Ok(None) => None,
697 Err(err) => bail!("error while reading exclude file - {}", err),
698 };
699
700 if include_children {
701 // Exclude patterns passed via the CLI are stored as '.pxarexclude-cli'
702 // in the root directory of the archive.
703 if is_root && !match_pattern.is_empty() {
704 let filename = CString::new(".pxarexclude-cli")?;
705 name_list.push((filename, *dir_stat, match_pattern.clone()));
706 }
707
708 for entry in dir.iter() {
709 let entry = entry
710 .map_err(|err| format_err!("readir {:?} failed - {}", self.full_path(), err))?;
711 let filename = entry.file_name().to_owned();
712
713 let name = filename.to_bytes_with_nul();
714 if name == b".\0" || name == b"..\0" {
715 continue;
716 }
717 // Do not store a ".pxarexclude-cli" file found in the archive root,
718 // as this would confilict with new cli passed exclude patterns,
719 // if present.
720 if is_root && name == b".pxarexclude-cli\0" {
721 eprintln!("skip existing '.pxarexclude-cli' in archive root.");
722 continue;
723 }
724
725 let stat = match nix::sys::stat::fstatat(
726 rawfd,
727 filename.as_ref(),
728 nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW,
729 ) {
730 Ok(stat) => stat,
731 Err(nix::Error::Sys(Errno::ENOENT)) => {
732 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
733 self.report_vanished_file(&self.full_path().join(filename_osstr))?;
734 continue;
735 }
736 Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
737 };
738
739 match match_filename(&filename, &stat, &local_match_pattern)? {
740 (MatchType::Positive, _) => {
741 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
742 eprintln!(
743 "matched by .pxarexclude entry - skipping: {:?}",
744 self.full_path().join(filename_osstr)
745 );
746 }
747 (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
748 }
749
750 if name_list.len() > MAX_DIRECTORY_ENTRIES {
751 bail!(
752 "too many directory items in {:?} (> {})",
753 self.full_path(),
754 MAX_DIRECTORY_ENTRIES
755 );
756 }
757 }
758 } else {
759 eprintln!("skip mount point: {:?}", self.full_path());
760 }
761
762 name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
763
764 let mut goodbye_items = vec![];
765
766 for (filename, stat, exclude_list) in name_list {
767 let start_pos = self.writer_pos;
768
769 if filename.as_bytes() == b".pxarexclude" {
770 if let Some((ref content, ref stat)) = pxar_exclude {
771 let filefd = match nix::fcntl::openat(
772 rawfd,
773 filename.as_ref(),
774 OFlag::O_NOFOLLOW,
775 Mode::empty(),
776 ) {
777 Ok(filefd) => filefd,
778 Err(nix::Error::Sys(Errno::ENOENT)) => {
779 self.report_vanished_file(&self.full_path())?;
780 continue;
781 }
782 Err(err) => {
783 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
784 bail!(
785 "open file {:?} failed - {}",
786 self.full_path().join(filename_osstr),
787 err
788 );
789 }
790 };
791
792 let child_magic = if dir_stat.st_dev != stat.st_dev {
793 detect_fs_type(filefd)?
794 } else {
795 magic
796 };
797
798 self.write_filename(&filename)?;
799 if let Some(ref mut catalog) = self.catalog {
800 catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
801 }
802 self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
803 continue;
804 }
805 }
806
807 if is_root && filename.as_bytes() == b".pxarexclude-cli" {
808 // '.pxarexclude-cli' is used to store the exclude MatchPatterns
809 // passed via the cli in the root directory of the archive.
810 self.write_filename(&filename)?;
811 let content = MatchPattern::to_bytes(&exclude_list);
812 if let Some(ref mut catalog) = self.catalog {
813 catalog.add_file(&filename, content.len() as u64, 0)?;
814 }
815 self.encode_pxar_exclude_cli(stat.st_uid, stat.st_gid, 0, &content)?;
816 continue;
817 }
818
819 self.relative_path
820 .push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
821
822 if self.verbose {
823 println!("{:?}", self.full_path());
824 }
825
826 if is_directory(&stat) {
827 let mut dir = match nix::dir::Dir::openat(
828 rawfd,
829 filename.as_ref(),
830 OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW,
831 Mode::empty(),
832 ) {
833 Ok(dir) => dir,
834 Err(nix::Error::Sys(Errno::ENOENT)) => {
835 self.report_vanished_file(&self.full_path())?;
836 continue; // fixme!!
837 }
838 Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
839 };
840
841 let child_magic = if dir_stat.st_dev != stat.st_dev {
842 detect_fs_type(dir.as_raw_fd())?
843 } else {
844 magic
845 };
846
847 self.write_filename(&filename)?;
848 if let Some(ref mut catalog) = self.catalog {
849 catalog.start_directory(&filename)?;
850 }
851 self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
852 if let Some(ref mut catalog) = self.catalog {
853 catalog.end_directory()?;
854 }
855 } else if is_reg_file(&stat) {
856 let mut hardlink_target = None;
857
858 if stat.st_nlink > 1 {
859 let link_info = HardLinkInfo {
860 st_dev: stat.st_dev,
861 st_ino: stat.st_ino,
862 };
863 hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
864 let mut target = v.clone().into_os_string();
865 target.push("\0"); // add Nul byte
866 (target, (start_pos as u64) - offset)
867 });
868 if hardlink_target == None {
869 self.hardlinks
870 .insert(link_info, (self.relative_path.clone(), start_pos as u64));
871 }
872 }
873
874 if let Some((target, offset)) = hardlink_target {
875 if let Some(ref mut catalog) = self.catalog {
876 catalog.add_hardlink(&filename)?;
877 }
878 self.write_filename(&filename)?;
879 self.encode_hardlink(target.as_bytes(), offset)?;
880 } else {
881 let filefd = match nix::fcntl::openat(
882 rawfd,
883 filename.as_ref(),
884 OFlag::O_NOFOLLOW,
885 Mode::empty(),
886 ) {
887 Ok(filefd) => filefd,
888 Err(nix::Error::Sys(Errno::ENOENT)) => {
889 self.report_vanished_file(&self.full_path())?;
890 continue;
891 }
892 Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
893 };
894
895 if let Some(ref mut catalog) = self.catalog {
896 catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
897 }
898 let child_magic = if dir_stat.st_dev != stat.st_dev {
899 detect_fs_type(filefd)?
900 } else {
901 magic
902 };
903
904 self.write_filename(&filename)?;
905 let res = self.encode_file(filefd, &stat, child_magic);
906 let _ = nix::unistd::close(filefd); // ignore close errors
907 res?;
908 }
909 } else if is_symlink(&stat) {
910 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
911
912 let res = filename.with_nix_path(|cstr| unsafe {
913 libc::readlinkat(
914 rawfd,
915 cstr.as_ptr(),
916 buffer.as_mut_ptr() as *mut libc::c_char,
917 buffer.len() - 1,
918 )
919 })?;
920
921 match Errno::result(res) {
922 Ok(len) => {
923 if let Some(ref mut catalog) = self.catalog {
924 catalog.add_symlink(&filename)?;
925 }
926 buffer[len as usize] = 0u8; // add Nul byte
927 self.write_filename(&filename)?;
928 self.encode_symlink(&buffer[..((len + 1) as usize)], &stat)?
929 }
930 Err(nix::Error::Sys(Errno::ENOENT)) => {
931 self.report_vanished_file(&self.full_path())?;
932 continue;
933 }
934 Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
935 }
936 } else if is_block_dev(&stat) || is_char_dev(&stat) {
937 if self.has_features(flags::WITH_DEVICE_NODES) {
938 if let Some(ref mut catalog) = self.catalog {
939 if is_block_dev(&stat) {
940 catalog.add_block_device(&filename)?;
941 } else {
942 catalog.add_char_device(&filename)?;
943 }
944 }
945 self.write_filename(&filename)?;
946 self.encode_device(&stat)?;
947 } else {
948 eprintln!("skip device node: {:?}", self.full_path());
949 }
950 } else if is_fifo(&stat) {
951 if self.has_features(flags::WITH_FIFOS) {
952 if let Some(ref mut catalog) = self.catalog {
953 catalog.add_fifo(&filename)?;
954 }
955 self.write_filename(&filename)?;
956 self.encode_special(&stat)?;
957 } else {
958 eprintln!("skip fifo: {:?}", self.full_path());
959 }
960 } else if is_socket(&stat) {
961 if self.has_features(flags::WITH_SOCKETS) {
962 if let Some(ref mut catalog) = self.catalog {
963 catalog.add_socket(&filename)?;
964 }
965 self.write_filename(&filename)?;
966 self.encode_special(&stat)?;
967 } else {
968 eprintln!("skip socket: {:?}", self.full_path());
969 }
970 } else {
971 bail!(
972 "unsupported file type (mode {:o} {:?})",
973 stat.st_mode,
974 self.full_path()
975 );
976 }
977
978 let end_pos = self.writer_pos;
979
980 goodbye_items.push(PxarGoodbyeItem {
981 offset: start_pos as u64,
982 size: (end_pos - start_pos) as u64,
983 hash: compute_goodbye_hash(filename.to_bytes()),
984 });
985
986 self.relative_path.pop();
987 }
988
989 //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
990
991 // fixup goodby item offsets
992 let goodbye_start = self.writer_pos as u64;
993 for item in &mut goodbye_items {
994 item.offset = goodbye_start - item.offset;
995 }
996
997 let goodbye_offset = self.writer_pos - dir_start_pos;
998
999 self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
1000
1001 //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
1002 Ok(())
1003 }
1004
1005 fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64) -> Result<(), Error> {
1006 //println!("encode_file: {:?}", self.full_path());
1007
1008 let mut entry = self.create_entry(&stat)?;
1009
1010 self.read_chattr(filefd, &mut entry)?;
1011 self.read_fat_attr(filefd, magic, &mut entry)?;
1012 let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1013 let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1014 let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1015
1016 self.write_entry(entry)?;
1017 for xattr in xattrs {
1018 self.write_xattr(xattr)?;
1019 }
1020 self.write_fcaps(fcaps)?;
1021 for user in acl_access.users {
1022 self.write_acl_user(user)?;
1023 }
1024 for group in acl_access.groups {
1025 self.write_acl_group(group)?;
1026 }
1027 if let Some(group_obj) = acl_access.group_obj {
1028 self.write_acl_group_obj(group_obj)?;
1029 }
1030 if let Some(projid) = projid {
1031 self.write_quota_project_id(projid)?;
1032 }
1033
1034 let include_payload;
1035 if is_virtual_file_system(magic) {
1036 include_payload = false;
1037 } else if let Some(ref set) = &self.device_set {
1038 include_payload = set.contains(&stat.st_dev);
1039 } else {
1040 include_payload = true;
1041 }
1042
1043 if !include_payload {
1044 eprintln!("skip content: {:?}", self.full_path());
1045 self.write_header(PXAR_PAYLOAD, 0)?;
1046 return Ok(());
1047 }
1048
1049 let size = stat.st_size as u64;
1050
1051 self.write_header(PXAR_PAYLOAD, size)?;
1052
1053 let mut pos: u64 = 0;
1054 loop {
1055 let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
1056 Ok(n) => n,
1057 Err(nix::Error::Sys(Errno::EINTR)) => continue, /* try again */
1058 Err(err) => bail!("read {:?} failed - {}", self.full_path(), err),
1059 };
1060 if n == 0 { // EOF
1061 if pos != size {
1062 // Note:: casync format cannot handle that
1063 bail!(
1064 "detected shrinked file {:?} ({} < {})",
1065 self.full_path(),
1066 pos,
1067 size
1068 );
1069 }
1070 break;
1071 }
1072
1073 let mut next = pos + (n as u64);
1074
1075 if next > size {
1076 next = size;
1077 }
1078
1079 let count = (next - pos) as usize;
1080
1081 self.flush_copy_buffer(count)?;
1082
1083 pos = next;
1084
1085 if pos >= size {
1086 break;
1087 }
1088 }
1089
1090 Ok(())
1091 }
1092
1093 fn encode_device(&mut self, stat: &FileStat) -> Result<(), Error> {
1094 let entry = self.create_entry(&stat)?;
1095
1096 self.write_entry(entry)?;
1097
1098 let major = unsafe { libc::major(stat.st_rdev) } as u64;
1099 let minor = unsafe { libc::minor(stat.st_rdev) } as u64;
1100
1101 //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
1102
1103 self.write_header(PXAR_DEVICE, std::mem::size_of::<PxarDevice>() as u64)?;
1104 self.write_item(PxarDevice { major, minor })?;
1105
1106 Ok(())
1107 }
1108
1109 // FIFO or Socket
1110 fn encode_special(&mut self, stat: &FileStat) -> Result<(), Error> {
1111 let entry = self.create_entry(&stat)?;
1112
1113 self.write_entry(entry)?;
1114
1115 Ok(())
1116 }
1117
1118 fn encode_symlink(&mut self, target: &[u8], stat: &FileStat) -> Result<(), Error> {
1119 //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
1120
1121 let entry = self.create_entry(&stat)?;
1122 self.write_entry(entry)?;
1123
1124 self.write_header(PXAR_SYMLINK, target.len() as u64)?;
1125 self.write(target)?;
1126
1127 Ok(())
1128 }
1129
1130 fn encode_hardlink(&mut self, target: &[u8], offset: u64) -> Result<(), Error> {
1131 //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
1132
1133 // Note: HARDLINK replaces an ENTRY.
1134 self.write_header(PXAR_FORMAT_HARDLINK, (target.len() as u64) + 8)?;
1135 self.write_item(offset)?;
1136 self.write(target)?;
1137
1138 Ok(())
1139 }
1140
1141 fn encode_pxar_exclude(
1142 &mut self,
1143 filefd: RawFd,
1144 stat: &FileStat,
1145 magic: i64,
1146 content: &[u8],
1147 ) -> Result<(), Error> {
1148 let mut entry = self.create_entry(&stat)?;
1149
1150 self.read_chattr(filefd, &mut entry)?;
1151 self.read_fat_attr(filefd, magic, &mut entry)?;
1152 let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1153 let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1154 let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1155
1156 self.write_entry(entry)?;
1157 for xattr in xattrs {
1158 self.write_xattr(xattr)?;
1159 }
1160 self.write_fcaps(fcaps)?;
1161 for user in acl_access.users {
1162 self.write_acl_user(user)?;
1163 }
1164 for group in acl_access.groups {
1165 self.write_acl_group(group)?;
1166 }
1167 if let Some(group_obj) = acl_access.group_obj {
1168 self.write_acl_group_obj(group_obj)?;
1169 }
1170 if let Some(projid) = projid {
1171 self.write_quota_project_id(projid)?;
1172 }
1173
1174 let include_payload;
1175 if is_virtual_file_system(magic) {
1176 include_payload = false;
1177 } else if let Some(set) = &self.device_set {
1178 include_payload = set.contains(&stat.st_dev);
1179 } else {
1180 include_payload = true;
1181 }
1182
1183 if !include_payload {
1184 eprintln!("skip content: {:?}", self.full_path());
1185 self.write_header(PXAR_PAYLOAD, 0)?;
1186 return Ok(());
1187 }
1188
1189 let size = content.len();
1190 self.write_header(PXAR_PAYLOAD, size as u64)?;
1191 self.writer.write_all(content)?;
1192 self.writer_pos += size;
1193
1194 Ok(())
1195 }
1196
1197 /// Encodes the excude match patterns passed via cli as file in the archive.
1198 fn encode_pxar_exclude_cli(
1199 &mut self,
1200 uid: u32,
1201 gid: u32,
1202 mtime: u64,
1203 content: &[u8],
1204 ) -> Result<(), Error> {
1205 let entry = PxarEntry {
1206 mode: (libc::S_IFREG | 0o600) as u64,
1207 flags: 0,
1208 uid,
1209 gid,
1210 mtime,
1211 };
1212 self.write_entry(entry)?;
1213 let size = content.len();
1214 self.write_header(PXAR_PAYLOAD, size as u64)?;
1215 self.writer.write_all(content)?;
1216 self.writer_pos += size;
1217
1218 Ok(())
1219 }
1220
1221 // the report_XXX method may raise and error - depending on encoder configuration
1222
1223 fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
1224 eprintln!("WARNING: detected vanished file {:?}", path);
1225
1226 Ok(())
1227 }
1228 }
1229
1230 // If there is a match, an updated MatchPattern list to pass to the matched child is returned.
1231 fn match_filename(
1232 filename: &CStr,
1233 stat: &FileStat,
1234 match_pattern: &[MatchPattern],
1235 ) -> Result<(MatchType, Vec<MatchPattern>), Error> {
1236 let mut child_pattern = Vec::new();
1237 let mut match_state = MatchType::None;
1238
1239 for pattern in match_pattern {
1240 match pattern.matches_filename(filename, is_directory(&stat))? {
1241 MatchType::None => {}
1242 MatchType::Positive => match_state = MatchType::Positive,
1243 MatchType::Negative => match_state = MatchType::Negative,
1244 match_type => {
1245 if match_state != MatchType::Positive && match_state != MatchType::Negative {
1246 match_state = match_type;
1247 }
1248 child_pattern.push(pattern.get_rest_pattern());
1249 }
1250 }
1251 }
1252
1253 Ok((match_state, child_pattern))
1254 }
1255
1256 fn errno_is_unsupported(errno: Errno) -> bool {
1257 match errno {
1258 Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => true,
1259 _ => false,
1260 }
1261 }
1262
1263 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
1264 let mut fs_stat = std::mem::MaybeUninit::uninit();
1265 let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
1266 Errno::result(res)?;
1267 let fs_stat = unsafe { fs_stat.assume_init() };
1268
1269 Ok(fs_stat.f_type)
1270 }
1271
1272 #[inline(always)]
1273 pub fn is_temporary_file_system(magic: i64) -> bool {
1274 use proxmox::sys::linux::magic::*;
1275 magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
1276 }
1277
1278 pub fn is_virtual_file_system(magic: i64) -> bool {
1279 use proxmox::sys::linux::magic::*;
1280
1281 match magic {
1282 BINFMTFS_MAGIC |
1283 CGROUP2_SUPER_MAGIC |
1284 CGROUP_SUPER_MAGIC |
1285 CONFIGFS_MAGIC |
1286 DEBUGFS_MAGIC |
1287 DEVPTS_SUPER_MAGIC |
1288 EFIVARFS_MAGIC |
1289 FUSE_CTL_SUPER_MAGIC |
1290 HUGETLBFS_MAGIC |
1291 MQUEUE_MAGIC |
1292 NFSD_MAGIC |
1293 PROC_SUPER_MAGIC |
1294 PSTOREFS_MAGIC |
1295 RPCAUTH_GSSMAGIC |
1296 SECURITYFS_MAGIC |
1297 SELINUX_MAGIC |
1298 SMACK_MAGIC |
1299 SYSFS_MAGIC => true,
1300 _ => false
1301 }
1302 }