]> git.proxmox.com Git - proxmox-backup.git/blob - src/pxar/encoder.rs
src/pxar/encoder.rs: avoid duplicate check for MAX_DIRECTORY_ENTRIES
[proxmox-backup.git] / src / pxar / encoder.rs
1 //! *pxar* format encoder.
2 //!
3 //! This module contain the code to generate *pxar* archive files.
4 use std::collections::{HashMap, HashSet};
5 use std::ffi::{CStr, CString};
6 use std::io::Write;
7 use std::os::unix::ffi::OsStrExt;
8 use std::os::unix::io::AsRawFd;
9 use std::os::unix::io::RawFd;
10 use std::path::{Path, PathBuf};
11
12 use endian_trait::Endian;
13 use failure::*;
14 use nix::errno::Errno;
15 use nix::fcntl::OFlag;
16 use nix::sys::stat::FileStat;
17 use nix::sys::stat::Mode;
18 use nix::NixPath;
19
20 use proxmox::tools::vec;
21
22 use super::binary_search_tree::*;
23 use super::catalog::BackupCatalogWriter;
24 use super::flags;
25 use super::format_definition::*;
26 use super::helper::*;
27 use super::match_pattern::{MatchPattern, MatchType};
28 use crate::tools::acl;
29 use crate::tools::fs;
30 use crate::tools::xattr;
31
32 /// The format requires to build sorted directory lookup tables in
33 /// memory, so we restrict the number of allowed entries to limit
34 /// maximum memory usage.
35 pub const MAX_DIRECTORY_ENTRIES: usize = 256 * 1024;
36
37 #[derive(Eq, PartialEq, Hash)]
38 struct HardLinkInfo {
39 st_dev: u64,
40 st_ino: u64,
41 }
42
43 pub struct Encoder<'a, W: Write, C: BackupCatalogWriter> {
44 base_path: PathBuf,
45 relative_path: PathBuf,
46 writer: &'a mut W,
47 writer_pos: usize,
48 catalog: Option<&'a mut C>,
49 _size: usize,
50 file_copy_buffer: Vec<u8>,
51 device_set: Option<HashSet<u64>>,
52 verbose: bool,
53 // Flags set by the user
54 feature_flags: u64,
55 // Flags signaling features supported by the filesystem
56 fs_feature_flags: u64,
57 hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
58 }
59
60 impl<'a, W: Write, C: BackupCatalogWriter> Encoder<'a, W, C> {
61 // used for error reporting
62 fn full_path(&self) -> PathBuf {
63 self.base_path.join(&self.relative_path)
64 }
65
66 /// Create archive, write result data to ``writer``.
67 ///
68 /// The ``device_set`` can be use used to limit included mount points.
69 ///
70 /// - ``None``: include all mount points
71 /// - ``Some(set)``: only include devices listed in this set (the
72 /// root path device is automathically added to this list, so
73 /// you can pass an empty set if you want to archive a single
74 /// mount point.)
75 pub fn encode(
76 path: PathBuf,
77 dir: &mut nix::dir::Dir,
78 writer: &'a mut W,
79 catalog: Option<&'a mut C>,
80 device_set: Option<HashSet<u64>>,
81 verbose: bool,
82 skip_lost_and_found: bool, // fixme: should be a feature flag ??
83 feature_flags: u64,
84 mut excludes: Vec<MatchPattern>,
85 ) -> Result<(), Error> {
86 const FILE_COPY_BUFFER_SIZE: usize = 1024 * 1024;
87
88 let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
89 unsafe {
90 file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE);
91 }
92
93 // todo: use scandirat??
94
95 let dir_fd = dir.as_raw_fd();
96 let stat = nix::sys::stat::fstat(dir_fd)
97 .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
98
99 if !is_directory(&stat) {
100 bail!("got unexpected file type {:?} (not a directory)", path);
101 }
102
103 let mut device_set = device_set.clone();
104 if let Some(ref mut set) = device_set {
105 set.insert(stat.st_dev);
106 }
107
108 let magic = detect_fs_type(dir_fd)?;
109
110 if is_virtual_file_system(magic) {
111 bail!("backup virtual file systems is disabled!");
112 }
113
114 let fs_feature_flags = flags::feature_flags_from_magic(magic);
115
116 let mut me = Self {
117 base_path: path,
118 relative_path: PathBuf::new(),
119 writer,
120 writer_pos: 0,
121 catalog,
122 _size: 0,
123 file_copy_buffer,
124 device_set,
125 verbose,
126 feature_flags,
127 fs_feature_flags,
128 hardlinks: HashMap::new(),
129 };
130
131 if verbose {
132 println!("{:?}", me.full_path());
133 }
134
135 if skip_lost_and_found {
136 excludes.push(MatchPattern::from_line(b"**/lost+found").unwrap().unwrap());
137 }
138
139 me.encode_dir(dir, &stat, magic, excludes)?;
140
141 Ok(())
142 }
143
144 fn write(&mut self, buf: &[u8]) -> Result<(), Error> {
145 self.writer.write_all(buf)?;
146 self.writer_pos += buf.len();
147 Ok(())
148 }
149
150 fn write_item<T: Endian>(&mut self, item: T) -> Result<(), Error> {
151 let data = item.to_le();
152
153 let buffer = unsafe {
154 std::slice::from_raw_parts(&data as *const T as *const u8, std::mem::size_of::<T>())
155 };
156
157 self.write(buffer)?;
158
159 Ok(())
160 }
161
162 fn flush_copy_buffer(&mut self, size: usize) -> Result<(), Error> {
163 self.writer.write_all(&self.file_copy_buffer[..size])?;
164 self.writer_pos += size;
165 Ok(())
166 }
167
168 fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
169 let size = size + (std::mem::size_of::<PxarHeader>() as u64);
170 self.write_item(PxarHeader { size, htype })?;
171
172 Ok(())
173 }
174
175 fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
176 let buffer = name.to_bytes_with_nul();
177 self.write_header(PXAR_FILENAME, buffer.len() as u64)?;
178 self.write(buffer)?;
179
180 Ok(())
181 }
182
183 fn create_entry(&self, stat: &FileStat) -> Result<PxarEntry, Error> {
184 let mode = if is_symlink(&stat) {
185 (libc::S_IFLNK | 0o777) as u64
186 } else {
187 (stat.st_mode & (libc::S_IFMT | 0o7777)) as u64
188 };
189
190 let mtime = stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec;
191 if mtime < 0 {
192 bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
193 }
194
195 let entry = PxarEntry {
196 mode,
197 flags: 0,
198 uid: stat.st_uid,
199 gid: stat.st_gid,
200 mtime: mtime as u64,
201 };
202
203 Ok(entry)
204 }
205
206 fn read_chattr(&self, fd: RawFd, entry: &mut PxarEntry) -> Result<(), Error> {
207 let mut attr: usize = 0;
208
209 let res = unsafe { fs::read_attr_fd(fd, &mut attr) };
210 if let Err(err) = res {
211 if let nix::Error::Sys(errno) = err {
212 if errno_is_unsupported(errno) {
213 return Ok(());
214 };
215 }
216 bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
217 }
218
219 let flags = flags::feature_flags_from_chattr(attr as u32);
220 entry.flags = entry.flags | flags;
221
222 Ok(())
223 }
224
225 fn read_fat_attr(&self, fd: RawFd, magic: i64, entry: &mut PxarEntry) -> Result<(), Error> {
226 use proxmox::sys::linux::magic::*;
227
228 if magic != MSDOS_SUPER_MAGIC && magic != FUSE_SUPER_MAGIC {
229 return Ok(());
230 }
231
232 let mut attr: u32 = 0;
233
234 let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr) };
235 if let Err(err) = res {
236 if let nix::Error::Sys(errno) = err {
237 if errno_is_unsupported(errno) {
238 return Ok(());
239 };
240 }
241 bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
242 }
243
244 let flags = flags::feature_flags_from_fat_attr(attr);
245 entry.flags = entry.flags | flags;
246
247 Ok(())
248 }
249
250 /// True if all of the given feature flags are set in the Encoder, false otherwise
251 fn has_features(&self, feature_flags: u64) -> bool {
252 (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
253 }
254
255 /// True if at least one of the given feature flags is set in the Encoder, false otherwise
256 fn has_some_features(&self, feature_flags: u64) -> bool {
257 (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
258 }
259
260 fn read_xattrs(
261 &self,
262 fd: RawFd,
263 stat: &FileStat,
264 ) -> Result<(Vec<PxarXAttr>, Option<PxarFCaps>), Error> {
265 let mut xattrs = Vec::new();
266 let mut fcaps = None;
267
268 let flags = flags::WITH_XATTRS | flags::WITH_FCAPS;
269 if !self.has_some_features(flags) {
270 return Ok((xattrs, fcaps));
271 }
272 // Should never be called on symlinks, just in case check anyway
273 if is_symlink(&stat) {
274 return Ok((xattrs, fcaps));
275 }
276
277 let xattr_names = match xattr::flistxattr(fd) {
278 Ok(names) => names,
279 // Do not bail if the underlying endpoint does not supports xattrs
280 Err(Errno::EOPNOTSUPP) => return Ok((xattrs, fcaps)),
281 // Do not bail if the endpoint cannot carry xattrs (such as symlinks)
282 Err(Errno::EBADF) => return Ok((xattrs, fcaps)),
283 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
284 };
285
286 for name in xattr_names.split(|c| *c == b'\0') {
287 // Only extract the relevant extended attributes
288 if !xattr::is_valid_xattr_name(&name) {
289 continue;
290 }
291
292 let value = match xattr::fgetxattr(fd, name) {
293 Ok(value) => value,
294 // Vanished between flistattr and getxattr, this is ok, silently ignore
295 Err(Errno::ENODATA) => continue,
296 Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
297 };
298
299 if xattr::is_security_capability(&name) {
300 if self.has_features(flags::WITH_FCAPS) {
301 // fcaps are stored in own format within the archive
302 fcaps = Some(PxarFCaps { data: value });
303 }
304 } else if self.has_features(flags::WITH_XATTRS) {
305 xattrs.push(PxarXAttr {
306 name: name.to_vec(),
307 value,
308 });
309 }
310 }
311 xattrs.sort();
312
313 Ok((xattrs, fcaps))
314 }
315
316 fn read_acl(
317 &self,
318 fd: RawFd,
319 stat: &FileStat,
320 acl_type: acl::ACLType,
321 ) -> Result<PxarACL, Error> {
322 let ret = PxarACL {
323 users: Vec::new(),
324 groups: Vec::new(),
325 group_obj: None,
326 default: None,
327 };
328
329 if !self.has_features(flags::WITH_ACL) {
330 return Ok(ret);
331 }
332 if is_symlink(&stat) {
333 return Ok(ret);
334 }
335 if acl_type == acl::ACL_TYPE_DEFAULT && !is_directory(&stat) {
336 bail!("ACL_TYPE_DEFAULT only defined for directories.");
337 }
338
339 // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
340 // to create a path for acl_get_file(). acl_get_fd() only allows to get
341 // ACL_TYPE_ACCESS attributes.
342 let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
343 let acl = match acl::ACL::get_file(&proc_path, acl_type) {
344 Ok(acl) => acl,
345 // Don't bail if underlying endpoint does not support acls
346 Err(Errno::EOPNOTSUPP) => return Ok(ret),
347 // Don't bail if the endpoint cannot carry acls
348 Err(Errno::EBADF) => return Ok(ret),
349 // Don't bail if there is no data
350 Err(Errno::ENODATA) => return Ok(ret),
351 Err(err) => bail!("error while reading ACL - {}", err),
352 };
353
354 self.process_acl(acl, acl_type)
355 }
356
357 fn process_acl(&self, acl: acl::ACL, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
358 let mut acl_user = Vec::new();
359 let mut acl_group = Vec::new();
360 let mut acl_group_obj = None;
361 let mut acl_default = None;
362 let mut user_obj_permissions = None;
363 let mut group_obj_permissions = None;
364 let mut other_permissions = None;
365 let mut mask_permissions = None;
366
367 for entry in &mut acl.entries() {
368 let tag = entry.get_tag_type()?;
369 let permissions = entry.get_permissions()?;
370 match tag {
371 acl::ACL_USER_OBJ => user_obj_permissions = Some(permissions),
372 acl::ACL_GROUP_OBJ => group_obj_permissions = Some(permissions),
373 acl::ACL_OTHER => other_permissions = Some(permissions),
374 acl::ACL_MASK => mask_permissions = Some(permissions),
375 acl::ACL_USER => {
376 acl_user.push(PxarACLUser {
377 uid: entry.get_qualifier()?,
378 permissions,
379 });
380 }
381 acl::ACL_GROUP => {
382 acl_group.push(PxarACLGroup {
383 gid: entry.get_qualifier()?,
384 permissions,
385 });
386 }
387 _ => bail!("Unexpected ACL tag encountered!"),
388 }
389 }
390
391 acl_user.sort();
392 acl_group.sort();
393
394 match acl_type {
395 acl::ACL_TYPE_ACCESS => {
396 // The mask permissions are mapped to the stat group permissions
397 // in case that the ACL group permissions were set.
398 // Only in that case we need to store the group permissions,
399 // in the other cases they are identical to the stat group permissions.
400 if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
401 acl_group_obj = Some(PxarACLGroupObj { permissions: gop });
402 }
403 }
404 acl::ACL_TYPE_DEFAULT => {
405 if user_obj_permissions != None
406 || group_obj_permissions != None
407 || other_permissions != None
408 || mask_permissions != None
409 {
410 acl_default = Some(PxarACLDefault {
411 // The value is set to UINT64_MAX as placeholder if one
412 // of the permissions is not set
413 user_obj_permissions: user_obj_permissions.unwrap_or(std::u64::MAX),
414 group_obj_permissions: group_obj_permissions.unwrap_or(std::u64::MAX),
415 other_permissions: other_permissions.unwrap_or(std::u64::MAX),
416 mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
417 });
418 }
419 }
420 _ => bail!("Unexpected ACL type encountered"),
421 }
422
423 Ok(PxarACL {
424 users: acl_user,
425 groups: acl_group,
426 group_obj: acl_group_obj,
427 default: acl_default,
428 })
429 }
430
431 /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
432 fn read_quota_project_id(
433 &self,
434 fd: RawFd,
435 magic: i64,
436 stat: &FileStat,
437 ) -> Result<Option<PxarQuotaProjID>, Error> {
438 if !(is_directory(&stat) || is_reg_file(&stat)) {
439 return Ok(None);
440 }
441 if !self.has_features(flags::WITH_QUOTA_PROJID) {
442 return Ok(None);
443 }
444
445 use proxmox::sys::linux::magic::*;
446
447 match magic {
448 EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC | ZFS_SUPER_MAGIC => {
449 let mut fsxattr = fs::FSXAttr::default();
450 let res = unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) };
451
452 // On some FUSE filesystems it can happen that ioctl is not supported.
453 // For these cases projid is set to 0 while the error is ignored.
454 if let Err(err) = res {
455 let errno = err.as_errno().ok_or_else(|| {
456 format_err!(
457 "error while reading quota project id for {:#?}",
458 self.full_path()
459 )
460 })?;
461 if errno_is_unsupported(errno) {
462 return Ok(None);
463 } else {
464 bail!(
465 "error while reading quota project id for {:#?} - {}",
466 self.full_path(),
467 errno
468 );
469 }
470 }
471
472 let projid = fsxattr.fsx_projid as u64;
473 if projid == 0 {
474 return Ok(None);
475 } else {
476 return Ok(Some(PxarQuotaProjID { projid }));
477 }
478 }
479 _ => return Ok(None),
480 }
481 }
482
483 fn write_entry(&mut self, entry: PxarEntry) -> Result<(), Error> {
484 self.write_header(PXAR_ENTRY, std::mem::size_of::<PxarEntry>() as u64)?;
485 self.write_item(entry)?;
486
487 Ok(())
488 }
489
490 fn write_xattr(&mut self, xattr: PxarXAttr) -> Result<(), Error> {
491 let size = xattr.name.len() + xattr.value.len() + 1; // +1 for '\0' separating name and value
492 self.write_header(PXAR_XATTR, size as u64)?;
493 self.write(xattr.name.as_slice())?;
494 self.write(&[0])?;
495 self.write(xattr.value.as_slice())?;
496
497 Ok(())
498 }
499
500 fn write_fcaps(&mut self, fcaps: Option<PxarFCaps>) -> Result<(), Error> {
501 if let Some(fcaps) = fcaps {
502 let size = fcaps.data.len();
503 self.write_header(PXAR_FCAPS, size as u64)?;
504 self.write(fcaps.data.as_slice())?;
505 }
506
507 Ok(())
508 }
509
510 fn write_acl_user(&mut self, acl_user: PxarACLUser) -> Result<(), Error> {
511 self.write_header(PXAR_ACL_USER, std::mem::size_of::<PxarACLUser>() as u64)?;
512 self.write_item(acl_user)?;
513
514 Ok(())
515 }
516
517 fn write_acl_group(&mut self, acl_group: PxarACLGroup) -> Result<(), Error> {
518 self.write_header(PXAR_ACL_GROUP, std::mem::size_of::<PxarACLGroup>() as u64)?;
519 self.write_item(acl_group)?;
520
521 Ok(())
522 }
523
524 fn write_acl_group_obj(&mut self, acl_group_obj: PxarACLGroupObj) -> Result<(), Error> {
525 self.write_header(
526 PXAR_ACL_GROUP_OBJ,
527 std::mem::size_of::<PxarACLGroupObj>() as u64,
528 )?;
529 self.write_item(acl_group_obj)?;
530
531 Ok(())
532 }
533
534 fn write_acl_default(&mut self, acl_default: PxarACLDefault) -> Result<(), Error> {
535 self.write_header(
536 PXAR_ACL_DEFAULT,
537 std::mem::size_of::<PxarACLDefault>() as u64,
538 )?;
539 self.write_item(acl_default)?;
540
541 Ok(())
542 }
543
544 fn write_acl_default_user(&mut self, acl_default_user: PxarACLUser) -> Result<(), Error> {
545 self.write_header(
546 PXAR_ACL_DEFAULT_USER,
547 std::mem::size_of::<PxarACLUser>() as u64,
548 )?;
549 self.write_item(acl_default_user)?;
550
551 Ok(())
552 }
553
554 fn write_acl_default_group(&mut self, acl_default_group: PxarACLGroup) -> Result<(), Error> {
555 self.write_header(
556 PXAR_ACL_DEFAULT_GROUP,
557 std::mem::size_of::<PxarACLGroup>() as u64,
558 )?;
559 self.write_item(acl_default_group)?;
560
561 Ok(())
562 }
563
564 fn write_quota_project_id(&mut self, projid: PxarQuotaProjID) -> Result<(), Error> {
565 self.write_header(
566 PXAR_QUOTA_PROJID,
567 std::mem::size_of::<PxarQuotaProjID>() as u64,
568 )?;
569 self.write_item(projid)?;
570
571 Ok(())
572 }
573
574 fn write_goodbye_table(
575 &mut self,
576 goodbye_offset: usize,
577 goodbye_items: &mut [PxarGoodbyeItem],
578 ) -> Result<(), Error> {
579 goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
580
581 let item_count = goodbye_items.len();
582
583 let goodbye_table_size = (item_count + 1) * std::mem::size_of::<PxarGoodbyeItem>();
584
585 self.write_header(PXAR_GOODBYE, goodbye_table_size as u64)?;
586
587 if self.file_copy_buffer.len() < goodbye_table_size {
588 let need = goodbye_table_size - self.file_copy_buffer.len();
589 self.file_copy_buffer.reserve(need);
590 unsafe {
591 self.file_copy_buffer
592 .set_len(self.file_copy_buffer.capacity());
593 }
594 }
595
596 let buffer = &mut self.file_copy_buffer;
597
598 copy_binary_search_tree(item_count, |s, d| {
599 let item = &goodbye_items[s];
600 let offset = d * std::mem::size_of::<PxarGoodbyeItem>();
601 let dest =
602 crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
603 dest.offset = u64::to_le(item.offset);
604 dest.size = u64::to_le(item.size);
605 dest.hash = u64::to_le(item.hash);
606 });
607
608 // append PxarGoodbyeTail as last item
609 let offset = item_count * std::mem::size_of::<PxarGoodbyeItem>();
610 let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
611 dest.offset = u64::to_le(goodbye_offset as u64);
612 dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<PxarHeader>()) as u64);
613 dest.hash = u64::to_le(PXAR_GOODBYE_TAIL_MARKER);
614
615 self.flush_copy_buffer(goodbye_table_size)?;
616
617 Ok(())
618 }
619
620 fn encode_dir(
621 &mut self,
622 dir: &mut nix::dir::Dir,
623 dir_stat: &FileStat,
624 magic: i64,
625 match_pattern: Vec<MatchPattern>,
626 ) -> Result<(), Error> {
627 //println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
628
629 let mut name_list = vec![];
630
631 let rawfd = dir.as_raw_fd();
632
633 let dir_start_pos = self.writer_pos;
634
635 let is_root = dir_start_pos == 0;
636
637 let mut dir_entry = self.create_entry(&dir_stat)?;
638
639 self.read_chattr(rawfd, &mut dir_entry)?;
640 self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
641
642 // for each node in the directory tree, the filesystem features are
643 // checked based on the fs magic number.
644 self.fs_feature_flags = flags::feature_flags_from_magic(magic);
645
646 let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
647 let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
648 let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
649 let projid = self.read_quota_project_id(rawfd, magic, &dir_stat)?;
650
651 self.write_entry(dir_entry)?;
652 for xattr in xattrs {
653 self.write_xattr(xattr)?;
654 }
655 self.write_fcaps(fcaps)?;
656
657 for user in acl_access.users {
658 self.write_acl_user(user)?;
659 }
660 for group in acl_access.groups {
661 self.write_acl_group(group)?;
662 }
663 if let Some(group_obj) = acl_access.group_obj {
664 self.write_acl_group_obj(group_obj)?;
665 }
666
667 for default_user in acl_default.users {
668 self.write_acl_default_user(default_user)?;
669 }
670 for default_group in acl_default.groups {
671 self.write_acl_default_group(default_group)?;
672 }
673 if let Some(default) = acl_default.default {
674 self.write_acl_default(default)?;
675 }
676 if let Some(projid) = projid {
677 self.write_quota_project_id(projid)?;
678 }
679
680 let include_children;
681 if is_virtual_file_system(magic) {
682 include_children = false;
683 } else {
684 if let Some(set) = &self.device_set {
685 include_children = set.contains(&dir_stat.st_dev);
686 } else {
687 include_children = true;
688 }
689 }
690
691 // Expand the exclude match pattern inherited from the parent by local entries, if present
692 let mut local_match_pattern = match_pattern.clone();
693 let pxar_exclude = match MatchPattern::from_file(rawfd, ".pxarexclude") {
694 Ok(Some((mut excludes, buffer, stat))) => {
695 local_match_pattern.append(&mut excludes);
696 Some((buffer, stat))
697 }
698 Ok(None) => None,
699 Err(err) => bail!("error while reading exclude file - {}", err),
700 };
701
702 if include_children {
703 // Exclude patterns passed via the CLI are stored as '.pxarexclude-cli'
704 // in the root directory of the archive.
705 if is_root && match_pattern.len() > 0 {
706 let filename = CString::new(".pxarexclude-cli")?;
707 name_list.push((filename, dir_stat.clone(), match_pattern.clone()));
708 }
709
710 for entry in dir.iter() {
711 let entry = entry
712 .map_err(|err| format_err!("readir {:?} failed - {}", self.full_path(), err))?;
713 let filename = entry.file_name().to_owned();
714
715 let name = filename.to_bytes_with_nul();
716 if name == b".\0" || name == b"..\0" {
717 continue;
718 }
719 // Do not store a ".pxarexclude-cli" file found in the archive root,
720 // as this would confilict with new cli passed exclude patterns,
721 // if present.
722 if is_root && name == b".pxarexclude-cli\0" {
723 eprintln!("skip existing '.pxarexclude-cli' in archive root.");
724 continue;
725 }
726
727 let stat = match nix::sys::stat::fstatat(
728 rawfd,
729 filename.as_ref(),
730 nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW,
731 ) {
732 Ok(stat) => stat,
733 Err(nix::Error::Sys(Errno::ENOENT)) => {
734 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
735 self.report_vanished_file(&self.full_path().join(filename_osstr))?;
736 continue;
737 }
738 Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
739 };
740
741 match match_filename(&filename, &stat, &local_match_pattern)? {
742 (MatchType::Positive, _) => {
743 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
744 eprintln!(
745 "matched by .pxarexclude entry - skipping: {:?}",
746 self.full_path().join(filename_osstr)
747 );
748 }
749 (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
750 }
751
752 if name_list.len() > MAX_DIRECTORY_ENTRIES {
753 bail!(
754 "too many directory items in {:?} (> {})",
755 self.full_path(),
756 MAX_DIRECTORY_ENTRIES
757 );
758 }
759 }
760 } else {
761 eprintln!("skip mount point: {:?}", self.full_path());
762 }
763
764 name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
765
766 let mut goodbye_items = vec![];
767
768 for (filename, stat, exclude_list) in name_list {
769 let start_pos = self.writer_pos;
770
771 if filename.as_bytes() == b".pxarexclude" {
772 if let Some((ref content, ref stat)) = pxar_exclude {
773 let filefd = match nix::fcntl::openat(
774 rawfd,
775 filename.as_ref(),
776 OFlag::O_NOFOLLOW,
777 Mode::empty(),
778 ) {
779 Ok(filefd) => filefd,
780 Err(nix::Error::Sys(Errno::ENOENT)) => {
781 self.report_vanished_file(&self.full_path())?;
782 continue;
783 }
784 Err(err) => {
785 let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
786 bail!(
787 "open file {:?} failed - {}",
788 self.full_path().join(filename_osstr),
789 err
790 );
791 }
792 };
793
794 let child_magic = if dir_stat.st_dev != stat.st_dev {
795 detect_fs_type(filefd)?
796 } else {
797 magic
798 };
799
800 self.write_filename(&filename)?;
801 if let Some(ref mut catalog) = self.catalog {
802 catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
803 }
804 self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
805 continue;
806 }
807 }
808
809 if is_root && filename.as_bytes() == b".pxarexclude-cli" {
810 // '.pxarexclude-cli' is used to store the exclude MatchPatterns
811 // passed via the cli in the root directory of the archive.
812 self.write_filename(&filename)?;
813 let content = MatchPattern::to_bytes(&exclude_list);
814 if let Some(ref mut catalog) = self.catalog {
815 catalog.add_file(&filename, content.len() as u64, 0)?;
816 }
817 self.encode_pxar_exclude_cli(stat.st_uid, stat.st_gid, 0, &content)?;
818 continue;
819 }
820
821 self.relative_path
822 .push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
823
824 if self.verbose {
825 println!("{:?}", self.full_path());
826 }
827
828 if is_directory(&stat) {
829 let mut dir = match nix::dir::Dir::openat(
830 rawfd,
831 filename.as_ref(),
832 OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW,
833 Mode::empty(),
834 ) {
835 Ok(dir) => dir,
836 Err(nix::Error::Sys(Errno::ENOENT)) => {
837 self.report_vanished_file(&self.full_path())?;
838 continue; // fixme!!
839 }
840 Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
841 };
842
843 let child_magic = if dir_stat.st_dev != stat.st_dev {
844 detect_fs_type(dir.as_raw_fd())?
845 } else {
846 magic
847 };
848
849 self.write_filename(&filename)?;
850 if let Some(ref mut catalog) = self.catalog {
851 catalog.start_directory(&filename)?;
852 }
853 self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
854 if let Some(ref mut catalog) = self.catalog {
855 catalog.end_directory()?;
856 }
857 } else if is_reg_file(&stat) {
858 let mut hardlink_target = None;
859
860 if stat.st_nlink > 1 {
861 let link_info = HardLinkInfo {
862 st_dev: stat.st_dev,
863 st_ino: stat.st_ino,
864 };
865 hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
866 let mut target = v.clone().into_os_string();
867 target.push("\0"); // add Nul byte
868 (target, (start_pos as u64) - offset)
869 });
870 if hardlink_target == None {
871 self.hardlinks
872 .insert(link_info, (self.relative_path.clone(), start_pos as u64));
873 }
874 }
875
876 if let Some((target, offset)) = hardlink_target {
877 if let Some(ref mut catalog) = self.catalog {
878 catalog.add_hardlink(&filename)?;
879 }
880 self.write_filename(&filename)?;
881 self.encode_hardlink(target.as_bytes(), offset)?;
882 } else {
883 let filefd = match nix::fcntl::openat(
884 rawfd,
885 filename.as_ref(),
886 OFlag::O_NOFOLLOW,
887 Mode::empty(),
888 ) {
889 Ok(filefd) => filefd,
890 Err(nix::Error::Sys(Errno::ENOENT)) => {
891 self.report_vanished_file(&self.full_path())?;
892 continue;
893 }
894 Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
895 };
896
897 if let Some(ref mut catalog) = self.catalog {
898 catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
899 }
900 let child_magic = if dir_stat.st_dev != stat.st_dev {
901 detect_fs_type(filefd)?
902 } else {
903 magic
904 };
905
906 self.write_filename(&filename)?;
907 let res = self.encode_file(filefd, &stat, child_magic);
908 let _ = nix::unistd::close(filefd); // ignore close errors
909 res?;
910 }
911 } else if is_symlink(&stat) {
912 let mut buffer = vec::undefined(libc::PATH_MAX as usize);
913
914 let res = filename.with_nix_path(|cstr| unsafe {
915 libc::readlinkat(
916 rawfd,
917 cstr.as_ptr(),
918 buffer.as_mut_ptr() as *mut libc::c_char,
919 buffer.len() - 1,
920 )
921 })?;
922
923 match Errno::result(res) {
924 Ok(len) => {
925 if let Some(ref mut catalog) = self.catalog {
926 catalog.add_symlink(&filename)?;
927 }
928 buffer[len as usize] = 0u8; // add Nul byte
929 self.write_filename(&filename)?;
930 self.encode_symlink(&buffer[..((len + 1) as usize)], &stat)?
931 }
932 Err(nix::Error::Sys(Errno::ENOENT)) => {
933 self.report_vanished_file(&self.full_path())?;
934 continue;
935 }
936 Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
937 }
938 } else if is_block_dev(&stat) || is_char_dev(&stat) {
939 if self.has_features(flags::WITH_DEVICE_NODES) {
940 if let Some(ref mut catalog) = self.catalog {
941 if is_block_dev(&stat) {
942 catalog.add_block_device(&filename)?;
943 } else {
944 catalog.add_char_device(&filename)?;
945 }
946 }
947 self.write_filename(&filename)?;
948 self.encode_device(&stat)?;
949 } else {
950 eprintln!("skip device node: {:?}", self.full_path());
951 }
952 } else if is_fifo(&stat) {
953 if self.has_features(flags::WITH_FIFOS) {
954 if let Some(ref mut catalog) = self.catalog {
955 catalog.add_fifo(&filename)?;
956 }
957 self.write_filename(&filename)?;
958 self.encode_special(&stat)?;
959 } else {
960 eprintln!("skip fifo: {:?}", self.full_path());
961 }
962 } else if is_socket(&stat) {
963 if self.has_features(flags::WITH_SOCKETS) {
964 if let Some(ref mut catalog) = self.catalog {
965 catalog.add_socket(&filename)?;
966 }
967 self.write_filename(&filename)?;
968 self.encode_special(&stat)?;
969 } else {
970 eprintln!("skip socket: {:?}", self.full_path());
971 }
972 } else {
973 bail!(
974 "unsupported file type (mode {:o} {:?})",
975 stat.st_mode,
976 self.full_path()
977 );
978 }
979
980 let end_pos = self.writer_pos;
981
982 goodbye_items.push(PxarGoodbyeItem {
983 offset: start_pos as u64,
984 size: (end_pos - start_pos) as u64,
985 hash: compute_goodbye_hash(filename.to_bytes()),
986 });
987
988 self.relative_path.pop();
989 }
990
991 //println!("encode_dir: {:?} end {}", self.full_path(), self.writer_pos);
992
993 // fixup goodby item offsets
994 let goodbye_start = self.writer_pos as u64;
995 for item in &mut goodbye_items {
996 item.offset = goodbye_start - item.offset;
997 }
998
999 let goodbye_offset = self.writer_pos - dir_start_pos;
1000
1001 self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
1002
1003 //println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
1004 Ok(())
1005 }
1006
1007 fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64) -> Result<(), Error> {
1008 //println!("encode_file: {:?}", self.full_path());
1009
1010 let mut entry = self.create_entry(&stat)?;
1011
1012 self.read_chattr(filefd, &mut entry)?;
1013 self.read_fat_attr(filefd, magic, &mut entry)?;
1014 let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1015 let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1016 let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1017
1018 self.write_entry(entry)?;
1019 for xattr in xattrs {
1020 self.write_xattr(xattr)?;
1021 }
1022 self.write_fcaps(fcaps)?;
1023 for user in acl_access.users {
1024 self.write_acl_user(user)?;
1025 }
1026 for group in acl_access.groups {
1027 self.write_acl_group(group)?;
1028 }
1029 if let Some(group_obj) = acl_access.group_obj {
1030 self.write_acl_group_obj(group_obj)?;
1031 }
1032 if let Some(projid) = projid {
1033 self.write_quota_project_id(projid)?;
1034 }
1035
1036 let include_payload;
1037 if is_virtual_file_system(magic) {
1038 include_payload = false;
1039 } else {
1040 if let Some(ref set) = &self.device_set {
1041 include_payload = set.contains(&stat.st_dev);
1042 } else {
1043 include_payload = true;
1044 }
1045 }
1046
1047 if !include_payload {
1048 eprintln!("skip content: {:?}", self.full_path());
1049 self.write_header(PXAR_PAYLOAD, 0)?;
1050 return Ok(());
1051 }
1052
1053 let size = stat.st_size as u64;
1054
1055 self.write_header(PXAR_PAYLOAD, size)?;
1056
1057 let mut pos: u64 = 0;
1058 loop {
1059 let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
1060 Ok(n) => n,
1061 Err(nix::Error::Sys(Errno::EINTR)) => continue, /* try again */
1062 Err(err) => bail!("read {:?} failed - {}", self.full_path(), err),
1063 };
1064 if n == 0 { // EOF
1065 if pos != size {
1066 // Note:: casync format cannot handle that
1067 bail!(
1068 "detected shrinked file {:?} ({} < {})",
1069 self.full_path(),
1070 pos,
1071 size
1072 );
1073 }
1074 break;
1075 }
1076
1077 let mut next = pos + (n as u64);
1078
1079 if next > size {
1080 next = size;
1081 }
1082
1083 let count = (next - pos) as usize;
1084
1085 self.flush_copy_buffer(count)?;
1086
1087 pos = next;
1088
1089 if pos >= size {
1090 break;
1091 }
1092 }
1093
1094 Ok(())
1095 }
1096
1097 fn encode_device(&mut self, stat: &FileStat) -> Result<(), Error> {
1098 let entry = self.create_entry(&stat)?;
1099
1100 self.write_entry(entry)?;
1101
1102 let major = unsafe { libc::major(stat.st_rdev) } as u64;
1103 let minor = unsafe { libc::minor(stat.st_rdev) } as u64;
1104
1105 //println!("encode_device: {:?} {} {} {}", self.full_path(), stat.st_rdev, major, minor);
1106
1107 self.write_header(PXAR_DEVICE, std::mem::size_of::<PxarDevice>() as u64)?;
1108 self.write_item(PxarDevice { major, minor })?;
1109
1110 Ok(())
1111 }
1112
1113 // FIFO or Socket
1114 fn encode_special(&mut self, stat: &FileStat) -> Result<(), Error> {
1115 let entry = self.create_entry(&stat)?;
1116
1117 self.write_entry(entry)?;
1118
1119 Ok(())
1120 }
1121
1122 fn encode_symlink(&mut self, target: &[u8], stat: &FileStat) -> Result<(), Error> {
1123 //println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
1124
1125 let entry = self.create_entry(&stat)?;
1126 self.write_entry(entry)?;
1127
1128 self.write_header(PXAR_SYMLINK, target.len() as u64)?;
1129 self.write(target)?;
1130
1131 Ok(())
1132 }
1133
1134 fn encode_hardlink(&mut self, target: &[u8], offset: u64) -> Result<(), Error> {
1135 //println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
1136
1137 // Note: HARDLINK replaces an ENTRY.
1138 self.write_header(PXAR_FORMAT_HARDLINK, (target.len() as u64) + 8)?;
1139 self.write_item(offset)?;
1140 self.write(target)?;
1141
1142 Ok(())
1143 }
1144
1145 fn encode_pxar_exclude(
1146 &mut self,
1147 filefd: RawFd,
1148 stat: &FileStat,
1149 magic: i64,
1150 content: &[u8],
1151 ) -> Result<(), Error> {
1152 let mut entry = self.create_entry(&stat)?;
1153
1154 self.read_chattr(filefd, &mut entry)?;
1155 self.read_fat_attr(filefd, magic, &mut entry)?;
1156 let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
1157 let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
1158 let projid = self.read_quota_project_id(filefd, magic, &stat)?;
1159
1160 self.write_entry(entry)?;
1161 for xattr in xattrs {
1162 self.write_xattr(xattr)?;
1163 }
1164 self.write_fcaps(fcaps)?;
1165 for user in acl_access.users {
1166 self.write_acl_user(user)?;
1167 }
1168 for group in acl_access.groups {
1169 self.write_acl_group(group)?;
1170 }
1171 if let Some(group_obj) = acl_access.group_obj {
1172 self.write_acl_group_obj(group_obj)?;
1173 }
1174 if let Some(projid) = projid {
1175 self.write_quota_project_id(projid)?;
1176 }
1177
1178 let include_payload;
1179 if is_virtual_file_system(magic) {
1180 include_payload = false;
1181 } else {
1182 if let Some(set) = &self.device_set {
1183 include_payload = set.contains(&stat.st_dev);
1184 } else {
1185 include_payload = true;
1186 }
1187 }
1188
1189 if !include_payload {
1190 eprintln!("skip content: {:?}", self.full_path());
1191 self.write_header(PXAR_PAYLOAD, 0)?;
1192 return Ok(());
1193 }
1194
1195 let size = content.len();
1196 self.write_header(PXAR_PAYLOAD, size as u64)?;
1197 self.writer.write_all(content)?;
1198 self.writer_pos += size;
1199
1200 Ok(())
1201 }
1202
1203 /// Encodes the excude match patterns passed via cli as file in the archive.
1204 fn encode_pxar_exclude_cli(
1205 &mut self,
1206 uid: u32,
1207 gid: u32,
1208 mtime: u64,
1209 content: &[u8],
1210 ) -> Result<(), Error> {
1211 let entry = PxarEntry {
1212 mode: (libc::S_IFREG | 0o600) as u64,
1213 flags: 0,
1214 uid,
1215 gid,
1216 mtime,
1217 };
1218 self.write_entry(entry)?;
1219 let size = content.len();
1220 self.write_header(PXAR_PAYLOAD, size as u64)?;
1221 self.writer.write_all(content)?;
1222 self.writer_pos += size;
1223
1224 Ok(())
1225 }
1226
1227 // the report_XXX method may raise and error - depending on encoder configuration
1228
1229 fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
1230 eprintln!("WARNING: detected vanished file {:?}", path);
1231
1232 Ok(())
1233 }
1234 }
1235
1236 // If there is a match, an updated MatchPattern list to pass to the matched child is returned.
1237 fn match_filename(
1238 filename: &CStr,
1239 stat: &FileStat,
1240 match_pattern: &Vec<MatchPattern>,
1241 ) -> Result<(MatchType, Vec<MatchPattern>), Error> {
1242 let mut child_pattern = Vec::new();
1243 let mut match_state = MatchType::None;
1244
1245 for pattern in match_pattern {
1246 match pattern.matches_filename(filename, is_directory(&stat))? {
1247 MatchType::None => {}
1248 MatchType::Positive => match_state = MatchType::Positive,
1249 MatchType::Negative => match_state = MatchType::Negative,
1250 match_type => {
1251 if match_state != MatchType::Positive && match_state != MatchType::Negative {
1252 match_state = match_type;
1253 }
1254 child_pattern.push(pattern.get_rest_pattern());
1255 }
1256 }
1257 }
1258
1259 Ok((match_state, child_pattern))
1260 }
1261
1262 fn errno_is_unsupported(errno: Errno) -> bool {
1263 match errno {
1264 Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => true,
1265 _ => false,
1266 }
1267 }
1268
1269 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
1270 let mut fs_stat = std::mem::MaybeUninit::uninit();
1271 let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
1272 Errno::result(res)?;
1273 let fs_stat = unsafe { fs_stat.assume_init() };
1274
1275 Ok(fs_stat.f_type)
1276 }
1277
1278 #[inline(always)]
1279 pub fn is_temporary_file_system(magic: i64) -> bool {
1280 use proxmox::sys::linux::magic::*;
1281 magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
1282 }
1283
1284 pub fn is_virtual_file_system(magic: i64) -> bool {
1285 use proxmox::sys::linux::magic::*;
1286
1287 match magic {
1288 BINFMTFS_MAGIC |
1289 CGROUP2_SUPER_MAGIC |
1290 CGROUP_SUPER_MAGIC |
1291 CONFIGFS_MAGIC |
1292 DEBUGFS_MAGIC |
1293 DEVPTS_SUPER_MAGIC |
1294 EFIVARFS_MAGIC |
1295 FUSE_CTL_SUPER_MAGIC |
1296 HUGETLBFS_MAGIC |
1297 MQUEUE_MAGIC |
1298 NFSD_MAGIC |
1299 PROC_SUPER_MAGIC |
1300 PSTOREFS_MAGIC |
1301 RPCAUTH_GSSMAGIC |
1302 SECURITYFS_MAGIC |
1303 SELINUX_MAGIC |
1304 SMACK_MAGIC |
1305 SYSFS_MAGIC => true,
1306 _ => false
1307 }
1308 }