src/pxar/create.rs

   1 use std::collections::{HashSet, HashMap};
   2 use std::ffi::{CStr, CString, OsStr};
   3 use std::fmt;
   4 use std::io::{self, Read, Write};
   5 use std::os::unix::ffi::OsStrExt;
   6 use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd, RawFd};
   7 use std::path::{Path, PathBuf};
   8
   9 use anyhow::{bail, format_err, Error};
  10 use nix::dir::Dir;
  11 use nix::errno::Errno;
  12 use nix::fcntl::OFlag;
  13 use nix::sys::stat::{FileStat, Mode};
  14
  15 use pathpatterns::{MatchEntry, MatchFlag, MatchList, MatchType, PatternFlag};
  16 use pxar::Metadata;
  17 use pxar::encoder::LinkOffset;
  18
  19 use proxmox::c_str;
  20 use proxmox::sys::error::SysError;
  21 use proxmox::tools::fd::RawFdNum;
  22 use proxmox::tools::vec;
  23
  24 use crate::pxar::catalog::BackupCatalogWriter;
  25 use crate::pxar::metadata::errno_is_unsupported;
  26 use crate::pxar::Flags;
  27 use crate::pxar::tools::assert_single_path_component;
  28 use crate::tools::{acl, fs, xattr, Fd};
  29
  30 /// Pxar options for creating a pxar archive/stream
  31 #[derive(Default, Clone)]
  32 pub struct PxarCreateOptions {
  33     /// Device/mountpoint st_dev numbers that should be included. None for no limitation.
  34     pub device_set: Option<HashSet<u64>>,
  35     /// Exclusion patterns
  36     pub patterns: Vec<MatchEntry>,
  37     /// Maximum number of entries to hold in memory
  38     pub entries_max: usize,
  39     /// Skip lost+found directory
  40     pub skip_lost_and_found: bool,
  41     /// Verbose output
  42     pub verbose: bool,
  43 }
  44
  45
  46 fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
  47     let mut fs_stat = std::mem::MaybeUninit::uninit();
  48     let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
  49     Errno::result(res)?;
  50     let fs_stat = unsafe { fs_stat.assume_init() };
  51
  52     Ok(fs_stat.f_type)
  53 }
  54
  55 #[rustfmt::skip]
  56 pub fn is_virtual_file_system(magic: i64) -> bool {
  57     use proxmox::sys::linux::magic::*;
  58
  59     matches!(magic, BINFMTFS_MAGIC |
  60         CGROUP2_SUPER_MAGIC |
  61         CGROUP_SUPER_MAGIC |
  62         CONFIGFS_MAGIC |
  63         DEBUGFS_MAGIC |
  64         DEVPTS_SUPER_MAGIC |
  65         EFIVARFS_MAGIC |
  66         FUSE_CTL_SUPER_MAGIC |
  67         HUGETLBFS_MAGIC |
  68         MQUEUE_MAGIC |
  69         NFSD_MAGIC |
  70         PROC_SUPER_MAGIC |
  71         PSTOREFS_MAGIC |
  72         RPCAUTH_GSSMAGIC |
  73         SECURITYFS_MAGIC |
  74         SELINUX_MAGIC |
  75         SMACK_MAGIC |
  76         SYSFS_MAGIC)
  77 }
  78
  79 #[derive(Debug)]
  80 struct ArchiveError {
  81     path: PathBuf,
  82     error: Error,
  83 }
  84
  85 impl ArchiveError {
  86     fn new(path: PathBuf, error: Error) -> Self {
  87         Self { path, error }
  88     }
  89 }
  90
  91 impl std::error::Error for ArchiveError {}
  92
  93 impl fmt::Display for ArchiveError {
  94     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
  95         write!(f, "error at {:?}: {}", self.path, self.error)
  96     }
  97 }
  98
  99 #[derive(Eq, PartialEq, Hash)]
 100 struct HardLinkInfo {
 101     st_dev: u64,
 102     st_ino: u64,
 103 }
 104
 105 /// TODO: make a builder for the create_archive call for fewer parameters and add a method to add a
 106 /// logger which does not write to stderr.
 107 struct Logger;
 108
 109 impl std::io::Write for Logger {
 110     fn write(&mut self, data: &[u8]) -> io::Result<usize> {
 111         std::io::stderr().write(data)
 112     }
 113
 114     fn flush(&mut self) -> io::Result<()> {
 115         std::io::stderr().flush()
 116     }
 117 }
 118
 119 /// And the error case.
 120 struct ErrorReporter;
 121
 122 impl std::io::Write for ErrorReporter {
 123     fn write(&mut self, data: &[u8]) -> io::Result<usize> {
 124         std::io::stderr().write(data)
 125     }
 126
 127     fn flush(&mut self) -> io::Result<()> {
 128         std::io::stderr().flush()
 129     }
 130 }
 131
 132 struct Archiver<'a, 'b> {
 133     feature_flags: Flags,
 134     fs_feature_flags: Flags,
 135     fs_magic: i64,
 136     patterns: Vec<MatchEntry>,
 137     callback: &'a mut dyn FnMut(&Path) -> Result<(), Error>,
 138     catalog: Option<&'b mut dyn BackupCatalogWriter>,
 139     path: PathBuf,
 140     entry_counter: usize,
 141     entry_limit: usize,
 142     current_st_dev: libc::dev_t,
 143     device_set: Option<HashSet<u64>>,
 144     hardlinks: HashMap<HardLinkInfo, (PathBuf, LinkOffset)>,
 145     errors: ErrorReporter,
 146     logger: Logger,
 147     file_copy_buffer: Vec<u8>,
 148 }
 149
 150 type Encoder<'a, 'b> = pxar::encoder::Encoder<'a, &'b mut dyn pxar::encoder::SeqWrite>;
 151
 152 pub fn create_archive<T, F>(
 153     source_dir: Dir,
 154     mut writer: T,
 155     feature_flags: Flags,
 156     mut callback: F,
 157     catalog: Option<&mut dyn BackupCatalogWriter>,
 158     options: PxarCreateOptions,
 159 ) -> Result<(), Error>
 160 where
 161     T: pxar::encoder::SeqWrite,
 162     F: FnMut(&Path) -> Result<(), Error>,
 163 {
 164     let fs_magic = detect_fs_type(source_dir.as_raw_fd())?;
 165     if is_virtual_file_system(fs_magic) {
 166         bail!("refusing to backup a virtual file system");
 167     }
 168
 169     let fs_feature_flags = Flags::from_magic(fs_magic);
 170
 171     let stat = nix::sys::stat::fstat(source_dir.as_raw_fd())?;
 172     let metadata = get_metadata(
 173         source_dir.as_raw_fd(),
 174         &stat,
 175         feature_flags & fs_feature_flags,
 176         fs_magic,
 177     )
 178     .map_err(|err| format_err!("failed to get metadata for source directory: {}", err))?;
 179
 180     let mut device_set = options.device_set.clone();
 181     if let Some(ref mut set) = device_set {
 182         set.insert(stat.st_dev);
 183     }
 184
 185     let writer = &mut writer as &mut dyn pxar::encoder::SeqWrite;
 186     let mut encoder = Encoder::new(writer, &metadata)?;
 187
 188     let mut patterns = options.patterns.clone();
 189
 190     if options.skip_lost_and_found {
 191         patterns.push(MatchEntry::parse_pattern(
 192             "lost+found",
 193             PatternFlag::PATH_NAME,
 194             MatchType::Exclude,
 195         )?);
 196     }
 197
 198     let mut archiver = Archiver {
 199         feature_flags,
 200         fs_feature_flags,
 201         fs_magic,
 202         callback: &mut callback,
 203         patterns,
 204         catalog,
 205         path: PathBuf::new(),
 206         entry_counter: 0,
 207         entry_limit: options.entries_max,
 208         current_st_dev: stat.st_dev,
 209         device_set,
 210         hardlinks: HashMap::new(),
 211         errors: ErrorReporter,
 212         logger: Logger,
 213         file_copy_buffer: vec::undefined(4 * 1024 * 1024),
 214     };
 215
 216     archiver.archive_dir_contents(&mut encoder, source_dir, true)?;
 217     encoder.finish()?;
 218     Ok(())
 219 }
 220
 221 struct FileListEntry {
 222     name: CString,
 223     path: PathBuf,
 224     stat: FileStat,
 225 }
 226
 227 impl<'a, 'b> Archiver<'a, 'b> {
 228     /// Get the currently effective feature flags. (Requested flags masked by the file system
 229     /// feature flags).
 230     fn flags(&self) -> Flags {
 231         self.feature_flags & self.fs_feature_flags
 232     }
 233
 234     fn wrap_err(&self, err: Error) -> Error {
 235         if err.downcast_ref::<ArchiveError>().is_some() {
 236             err
 237         } else {
 238             ArchiveError::new(self.path.clone(), err).into()
 239         }
 240     }
 241
 242     fn archive_dir_contents(
 243         &mut self,
 244         encoder: &mut Encoder,
 245         mut dir: Dir,
 246         is_root: bool,
 247     ) -> Result<(), Error> {
 248         let entry_counter = self.entry_counter;
 249
 250         let old_patterns_count = self.patterns.len();
 251         self.read_pxar_excludes(dir.as_raw_fd())?;
 252
 253         let mut file_list = self.generate_directory_file_list(&mut dir, is_root)?;
 254
 255         if is_root && old_patterns_count > 0 {
 256             file_list.push(FileListEntry {
 257                 name: CString::new(".pxarexclude-cli").unwrap(),
 258                 path: PathBuf::new(),
 259                 stat: unsafe { std::mem::zeroed() },
 260             });
 261         }
 262
 263         let dir_fd = dir.as_raw_fd();
 264
 265         let old_path = std::mem::take(&mut self.path);
 266
 267         for file_entry in file_list {
 268             let file_name = file_entry.name.to_bytes();
 269
 270             if is_root && file_name == b".pxarexclude-cli" {
 271                 self.encode_pxarexclude_cli(encoder, &file_entry.name, old_patterns_count)?;
 272                 continue;
 273             }
 274
 275             (self.callback)(&file_entry.path)?;
 276             self.path = file_entry.path;
 277             self.add_entry(encoder, dir_fd, &file_entry.name, &file_entry.stat)
 278                 .map_err(|err| self.wrap_err(err))?;
 279         }
 280         self.path = old_path;
 281         self.entry_counter = entry_counter;
 282         self.patterns.truncate(old_patterns_count);
 283
 284         Ok(())
 285     }
 286
 287     /// openat() wrapper which allows but logs `EACCES` and turns `ENOENT` into `None`.
 288     ///
 289     /// The `existed` flag is set when iterating through a directory to note that we know the file
 290     /// is supposed to exist and we should warn if it doesnt'.
 291     fn open_file(
 292         &mut self,
 293         parent: RawFd,
 294         file_name: &CStr,
 295         oflags: OFlag,
 296         existed: bool,
 297     ) -> Result<Option<Fd>, Error> {
 298         // common flags we always want to use:
 299         let oflags = oflags | OFlag::O_CLOEXEC | OFlag::O_NOCTTY;
 300
 301         let mut noatime = OFlag::O_NOATIME;
 302         loop {
 303             return match Fd::openat(
 304                 &unsafe { RawFdNum::from_raw_fd(parent) },
 305                 file_name,
 306                 oflags | noatime,
 307                 Mode::empty(),
 308             ) {
 309                 Ok(fd) => Ok(Some(fd)),
 310                 Err(nix::Error::Sys(Errno::ENOENT)) => {
 311                     if existed {
 312                         self.report_vanished_file()?;
 313                     }
 314                     Ok(None)
 315                 }
 316                 Err(nix::Error::Sys(Errno::EACCES)) => {
 317                     writeln!(self.errors, "failed to open file: {:?}: access denied", file_name)?;
 318                     Ok(None)
 319                 }
 320                 Err(nix::Error::Sys(Errno::EPERM)) if !noatime.is_empty() => {
 321                     // Retry without O_NOATIME:
 322                     noatime = OFlag::empty();
 323                     continue;
 324                 }
 325                 Err(other) => Err(Error::from(other)),
 326             }
 327         }
 328     }
 329
 330     fn read_pxar_excludes(&mut self, parent: RawFd) -> Result<(), Error> {
 331         let fd = match self.open_file(parent, c_str!(".pxarexclude"), OFlag::O_RDONLY, false)? {
 332             Some(fd) => fd,
 333             None => return Ok(()),
 334         };
 335
 336         let old_pattern_count = self.patterns.len();
 337
 338         let path_bytes = self.path.as_os_str().as_bytes();
 339
 340         let file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) };
 341
 342         use io::BufRead;
 343         for line in io::BufReader::new(file).split(b'\n') {
 344             let line = match line {
 345                 Ok(line) => line,
 346                 Err(err) => {
 347                     let _ = writeln!(
 348                         self.errors,
 349                         "ignoring .pxarexclude after read error in {:?}: {}",
 350                         self.path,
 351                         err,
 352                     );
 353                     self.patterns.truncate(old_pattern_count);
 354                     return Ok(());
 355                 }
 356             };
 357
 358             let line = crate::tools::strip_ascii_whitespace(&line);
 359
 360             if line.is_empty() || line[0] == b'#' {
 361                 continue;
 362             }
 363
 364             let mut buf;
 365             let (line, mode, anchored) = if line[0] == b'/' {
 366                 buf = Vec::with_capacity(path_bytes.len() + 1 + line.len());
 367                 buf.extend(path_bytes);
 368                 buf.extend(line);
 369                 (&buf[..], MatchType::Exclude, true)
 370             } else if line.starts_with(b"!/") {
 371                 // inverted case with absolute path
 372                 buf = Vec::with_capacity(path_bytes.len() + line.len());
 373                 buf.extend(path_bytes);
 374                 buf.extend(&line[1..]); // without the '!'
 375                 (&buf[..], MatchType::Include, true)
 376             } else if line.starts_with(b"!") {
 377                 (&line[1..], MatchType::Include, false)
 378             } else {
 379                 (line, MatchType::Exclude, false)
 380             };
 381
 382             match MatchEntry::parse_pattern(line, PatternFlag::PATH_NAME, mode) {
 383                 Ok(pattern) => {
 384                     if anchored {
 385                         self.patterns.push(pattern.add_flags(MatchFlag::ANCHORED));
 386                     } else {
 387                         self.patterns.push(pattern);
 388                     }
 389                 }
 390                 Err(err) => {
 391                     let _ = writeln!(self.errors, "bad pattern in {:?}: {}", self.path, err);
 392                 }
 393             }
 394         }
 395
 396         Ok(())
 397     }
 398
 399     fn encode_pxarexclude_cli(
 400         &mut self,
 401         encoder: &mut Encoder,
 402         file_name: &CStr,
 403         patterns_count: usize,
 404     ) -> Result<(), Error> {
 405         let content = generate_pxar_excludes_cli(&self.patterns[..patterns_count]);
 406
 407         if let Some(ref mut catalog) = self.catalog {
 408             catalog.add_file(file_name, content.len() as u64, 0)?;
 409         }
 410
 411         let mut metadata = Metadata::default();
 412         metadata.stat.mode = pxar::format::mode::IFREG | 0o600;
 413
 414         let mut file = encoder.create_file(&metadata, ".pxarexclude-cli", content.len() as u64)?;
 415         file.write_all(&content)?;
 416
 417         Ok(())
 418     }
 419
 420     fn generate_directory_file_list(
 421         &mut self,
 422         dir: &mut Dir,
 423         is_root: bool,
 424     ) -> Result<Vec<FileListEntry>, Error> {
 425         let dir_fd = dir.as_raw_fd();
 426
 427         let mut file_list = Vec::new();
 428
 429         for file in dir.iter() {
 430             let file = file?;
 431
 432             let file_name = file.file_name().to_owned();
 433             let file_name_bytes = file_name.to_bytes();
 434             if file_name_bytes == b"." || file_name_bytes == b".." {
 435                 continue;
 436             }
 437
 438             if is_root && file_name_bytes == b".pxarexclude-cli" {
 439                 continue;
 440             }
 441
 442             let os_file_name = OsStr::from_bytes(file_name_bytes);
 443             assert_single_path_component(os_file_name)?;
 444             let full_path = self.path.join(os_file_name);
 445
 446             let stat = match nix::sys::stat::fstatat(
 447                 dir_fd,
 448                 file_name.as_c_str(),
 449                 nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW,
 450             ) {
 451                 Ok(stat) => stat,
 452                 Err(ref err) if err.not_found() => continue,
 453                 Err(err) => bail!("stat failed on {:?}: {}", full_path, err),
 454             };
 455
 456             let match_path = PathBuf::from("/").join(full_path.clone());
 457             if self
 458                 .patterns
 459                 .matches(match_path.as_os_str().as_bytes(), Some(stat.st_mode as u32))
 460                 == Some(MatchType::Exclude)
 461             {
 462                 continue;
 463             }
 464
 465             self.entry_counter += 1;
 466             if self.entry_counter > self.entry_limit {
 467                 bail!("exceeded allowed number of file entries (> {})",self.entry_limit);
 468             }
 469
 470             file_list.push(FileListEntry {
 471                 name: file_name,
 472                 path: full_path,
 473                 stat
 474             });
 475         }
 476
 477         file_list.sort_unstable_by(|a, b| a.name.cmp(&b.name));
 478
 479         Ok(file_list)
 480     }
 481
 482     fn report_vanished_file(&mut self) -> Result<(), Error> {
 483         writeln!(self.errors, "warning: file vanished while reading: {:?}", self.path)?;
 484         Ok(())
 485     }
 486
 487     fn report_file_shrunk_while_reading(&mut self) -> Result<(), Error> {
 488         writeln!(
 489             self.errors,
 490             "warning: file size shrunk while reading: {:?}, file will be padded with zeros!",
 491             self.path,
 492         )?;
 493         Ok(())
 494     }
 495
 496     fn report_file_grew_while_reading(&mut self) -> Result<(), Error> {
 497         writeln!(
 498             self.errors,
 499             "warning: file size increased while reading: {:?}, file will be truncated!",
 500             self.path,
 501         )?;
 502         Ok(())
 503     }
 504
 505     fn add_entry(
 506         &mut self,
 507         encoder: &mut Encoder,
 508         parent: RawFd,
 509         c_file_name: &CStr,
 510         stat: &FileStat,
 511     ) -> Result<(), Error> {
 512         use pxar::format::mode;
 513
 514         let file_mode = stat.st_mode & libc::S_IFMT;
 515         let open_mode = if file_mode == libc::S_IFREG || file_mode == libc::S_IFDIR {
 516             OFlag::empty()
 517         } else {
 518             OFlag::O_PATH
 519         };
 520
 521         let fd = self.open_file(
 522             parent,
 523             c_file_name,
 524             open_mode | OFlag::O_RDONLY | OFlag::O_NOFOLLOW,
 525             true,
 526         )?;
 527
 528         let fd = match fd {
 529             Some(fd) => fd,
 530             None => return Ok(()),
 531         };
 532
 533         let metadata = get_metadata(fd.as_raw_fd(), &stat, self.flags(), self.fs_magic)?;
 534
 535         if self
 536             .patterns
 537             .matches(self.path.as_os_str().as_bytes(), Some(stat.st_mode as u32))
 538             == Some(MatchType::Exclude)
 539         {
 540             return Ok(());
 541         }
 542
 543         let file_name: &Path = OsStr::from_bytes(c_file_name.to_bytes()).as_ref();
 544         match metadata.file_type() {
 545             mode::IFREG => {
 546                 let link_info = HardLinkInfo {
 547                     st_dev: stat.st_dev,
 548                     st_ino: stat.st_ino,
 549                 };
 550
 551                 if stat.st_nlink > 1 {
 552                     if let Some((path, offset)) = self.hardlinks.get(&link_info) {
 553                         if let Some(ref mut catalog) = self.catalog {
 554                             catalog.add_hardlink(c_file_name)?;
 555                         }
 556
 557                         encoder.add_hardlink(file_name, path, *offset)?;
 558
 559                         return Ok(());
 560                     }
 561                 }
 562
 563                 let file_size = stat.st_size as u64;
 564                 if let Some(ref mut catalog) = self.catalog {
 565                     catalog.add_file(c_file_name, file_size, stat.st_mtime)?;
 566                 }
 567
 568                 let offset: LinkOffset =
 569                     self.add_regular_file(encoder, fd, file_name, &metadata, file_size)?;
 570
 571                 if stat.st_nlink > 1 {
 572                     self.hardlinks.insert(link_info, (self.path.clone(), offset));
 573                 }
 574
 575                 Ok(())
 576             }
 577             mode::IFDIR => {
 578                 let dir = Dir::from_fd(fd.into_raw_fd())?;
 579
 580                 if let Some(ref mut catalog) = self.catalog {
 581                     catalog.start_directory(c_file_name)?;
 582                 }
 583                 let result = self.add_directory(encoder, dir, c_file_name, &metadata, stat);
 584                 if let Some(ref mut catalog) = self.catalog {
 585                     catalog.end_directory()?;
 586                 }
 587                 result
 588             }
 589             mode::IFSOCK => {
 590                 if let Some(ref mut catalog) = self.catalog {
 591                     catalog.add_socket(c_file_name)?;
 592                 }
 593
 594                 Ok(encoder.add_socket(&metadata, file_name)?)
 595             }
 596             mode::IFIFO => {
 597                 if let Some(ref mut catalog) = self.catalog {
 598                     catalog.add_fifo(c_file_name)?;
 599                 }
 600
 601                 Ok(encoder.add_fifo(&metadata, file_name)?)
 602             }
 603             mode::IFLNK => {
 604                 if let Some(ref mut catalog) = self.catalog {
 605                     catalog.add_symlink(c_file_name)?;
 606                 }
 607
 608                 self.add_symlink(encoder, fd, file_name, &metadata)
 609             }
 610             mode::IFBLK => {
 611                 if let Some(ref mut catalog) = self.catalog {
 612                     catalog.add_block_device(c_file_name)?;
 613                 }
 614
 615                 self.add_device(encoder, file_name, &metadata, &stat)
 616             }
 617             mode::IFCHR => {
 618                 if let Some(ref mut catalog) = self.catalog {
 619                     catalog.add_char_device(c_file_name)?;
 620                 }
 621
 622                 self.add_device(encoder, file_name, &metadata, &stat)
 623             }
 624             other => bail!(
 625                 "encountered unknown file type: 0x{:x} (0o{:o})",
 626                 other,
 627                 other
 628             ),
 629         }
 630     }
 631
 632     fn add_directory(
 633         &mut self,
 634         encoder: &mut Encoder,
 635         dir: Dir,
 636         dir_name: &CStr,
 637         metadata: &Metadata,
 638         stat: &FileStat,
 639     ) -> Result<(), Error> {
 640         let dir_name = OsStr::from_bytes(dir_name.to_bytes());
 641
 642         let mut encoder = encoder.create_directory(dir_name, &metadata)?;
 643
 644         let old_fs_magic = self.fs_magic;
 645         let old_fs_feature_flags = self.fs_feature_flags;
 646         let old_st_dev = self.current_st_dev;
 647
 648         let mut skip_contents = false;
 649         if old_st_dev != stat.st_dev {
 650             self.fs_magic = detect_fs_type(dir.as_raw_fd())?;
 651             self.fs_feature_flags = Flags::from_magic(self.fs_magic);
 652             self.current_st_dev = stat.st_dev;
 653
 654             if is_virtual_file_system(self.fs_magic) {
 655                 skip_contents = true;
 656             } else if let Some(set) = &self.device_set {
 657                 skip_contents = !set.contains(&stat.st_dev);
 658             }
 659         }
 660
 661         let result = if skip_contents {
 662             writeln!(self.logger, "skipping mount point: {:?}", self.path)?;
 663             Ok(())
 664         } else {
 665             self.archive_dir_contents(&mut encoder, dir, false)
 666         };
 667
 668         self.fs_magic = old_fs_magic;
 669         self.fs_feature_flags = old_fs_feature_flags;
 670         self.current_st_dev = old_st_dev;
 671
 672         encoder.finish()?;
 673         result
 674     }
 675
 676     fn add_regular_file(
 677         &mut self,
 678         encoder: &mut Encoder,
 679         fd: Fd,
 680         file_name: &Path,
 681         metadata: &Metadata,
 682         file_size: u64,
 683     ) -> Result<LinkOffset, Error> {
 684         let mut file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) };
 685         let mut remaining = file_size;
 686         let mut out = encoder.create_file(metadata, file_name, file_size)?;
 687         while remaining != 0 {
 688             let mut got = match file.read(&mut self.file_copy_buffer[..]) {
 689                 Ok(0) => break,
 690                 Ok(got) => got,
 691                 Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue,
 692                 Err(err) => bail!(err),
 693             };
 694             if got as u64 > remaining {
 695                 self.report_file_grew_while_reading()?;
 696                 got = remaining as usize;
 697             }
 698             out.write_all(&self.file_copy_buffer[..got])?;
 699             remaining -= got as u64;
 700         }
 701         if remaining > 0 {
 702             self.report_file_shrunk_while_reading()?;
 703             let to_zero = remaining.min(self.file_copy_buffer.len() as u64) as usize;
 704             vec::clear(&mut self.file_copy_buffer[..to_zero]);
 705             while remaining != 0 {
 706                 let fill = remaining.min(self.file_copy_buffer.len() as u64) as usize;
 707                 out.write_all(&self.file_copy_buffer[..fill])?;
 708                 remaining -= fill as u64;
 709             }
 710         }
 711
 712         Ok(out.file_offset())
 713     }
 714
 715     fn add_symlink(
 716         &mut self,
 717         encoder: &mut Encoder,
 718         fd: Fd,
 719         file_name: &Path,
 720         metadata: &Metadata,
 721     ) -> Result<(), Error> {
 722         let dest = nix::fcntl::readlinkat(fd.as_raw_fd(), &b""[..])?;
 723         encoder.add_symlink(metadata, file_name, dest)?;
 724         Ok(())
 725     }
 726
 727     fn add_device(
 728         &mut self,
 729         encoder: &mut Encoder,
 730         file_name: &Path,
 731         metadata: &Metadata,
 732         stat: &FileStat,
 733     ) -> Result<(), Error> {
 734         Ok(encoder.add_device(
 735             metadata,
 736             file_name,
 737             pxar::format::Device::from_dev_t(stat.st_rdev),
 738         )?)
 739     }
 740 }
 741
 742 fn get_metadata(fd: RawFd, stat: &FileStat, flags: Flags, fs_magic: i64) -> Result<Metadata, Error> {
 743     // required for some of these
 744     let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
 745
 746     let mut meta = Metadata {
 747         stat: pxar::Stat {
 748             mode: u64::from(stat.st_mode),
 749             flags: 0,
 750             uid: stat.st_uid,
 751             gid: stat.st_gid,
 752             mtime: pxar::format::StatxTimestamp {
 753                 secs: stat.st_mtime,
 754                 nanos: stat.st_mtime_nsec as u32,
 755             },
 756         },
 757         ..Default::default()
 758     };
 759
 760     get_xattr_fcaps_acl(&mut meta, fd, &proc_path, flags)?;
 761     get_chattr(&mut meta, fd)?;
 762     get_fat_attr(&mut meta, fd, fs_magic)?;
 763     get_quota_project_id(&mut meta, fd, flags, fs_magic)?;
 764     Ok(meta)
 765 }
 766
 767 fn get_fcaps(meta: &mut Metadata, fd: RawFd, flags: Flags) -> Result<(), Error> {
 768     if flags.contains(Flags::WITH_FCAPS) {
 769         return Ok(());
 770     }
 771
 772     match xattr::fgetxattr(fd, xattr::xattr_name_fcaps()) {
 773         Ok(data) => {
 774             meta.fcaps = Some(pxar::format::FCaps { data });
 775             Ok(())
 776         }
 777         Err(Errno::ENODATA) => Ok(()),
 778         Err(Errno::EOPNOTSUPP) => Ok(()),
 779         Err(Errno::EBADF) => Ok(()), // symlinks
 780         Err(err) => bail!("failed to read file capabilities: {}", err),
 781     }
 782 }
 783
 784 fn get_xattr_fcaps_acl(
 785     meta: &mut Metadata,
 786     fd: RawFd,
 787     proc_path: &Path,
 788     flags: Flags,
 789 ) -> Result<(), Error> {
 790     if flags.contains(Flags::WITH_XATTRS) {
 791         return Ok(());
 792     }
 793
 794     let xattrs = match xattr::flistxattr(fd) {
 795         Ok(names) => names,
 796         Err(Errno::EOPNOTSUPP) => return Ok(()),
 797         Err(Errno::EBADF) => return Ok(()), // symlinks
 798         Err(err) => bail!("failed to read xattrs: {}", err),
 799     };
 800
 801     for attr in &xattrs {
 802         if xattr::is_security_capability(&attr) {
 803             get_fcaps(meta, fd, flags)?;
 804             continue;
 805         }
 806
 807         if xattr::is_acl(&attr) {
 808             get_acl(meta, proc_path, flags)?;
 809             continue;
 810         }
 811
 812         if !xattr::is_valid_xattr_name(&attr) {
 813             continue;
 814         }
 815
 816         match xattr::fgetxattr(fd, attr) {
 817             Ok(data) => meta
 818                 .xattrs
 819                 .push(pxar::format::XAttr::new(attr.to_bytes(), data)),
 820             Err(Errno::ENODATA) => (), // it got removed while we were iterating...
 821             Err(Errno::EOPNOTSUPP) => (), // shouldn't be possible so just ignore this
 822             Err(Errno::EBADF) => (),   // symlinks, shouldn't be able to reach this either
 823             Err(err) => bail!("error reading extended attribute {:?}: {}", attr, err),
 824         }
 825     }
 826
 827     Ok(())
 828 }
 829
 830 fn get_chattr(metadata: &mut Metadata, fd: RawFd) -> Result<(), Error> {
 831     let mut attr: libc::c_long = 0;
 832
 833     match unsafe { fs::read_attr_fd(fd, &mut attr) } {
 834         Ok(_) => (),
 835         Err(nix::Error::Sys(errno)) if errno_is_unsupported(errno) => {
 836             return Ok(());
 837         }
 838         Err(err) => bail!("failed to read file attributes: {}", err),
 839     }
 840
 841     metadata.stat.flags |= Flags::from_chattr(attr).bits();
 842
 843     Ok(())
 844 }
 845
 846 fn get_fat_attr(metadata: &mut Metadata, fd: RawFd, fs_magic: i64) -> Result<(), Error> {
 847     use proxmox::sys::linux::magic::*;
 848
 849     if fs_magic != MSDOS_SUPER_MAGIC && fs_magic != FUSE_SUPER_MAGIC {
 850         return Ok(());
 851     }
 852
 853     let mut attr: u32 = 0;
 854
 855     match unsafe { fs::read_fat_attr_fd(fd, &mut attr) } {
 856         Ok(_) => (),
 857         Err(nix::Error::Sys(errno)) if errno_is_unsupported(errno) => {
 858             return Ok(());
 859         }
 860         Err(err) => bail!("failed to read fat attributes: {}", err),
 861     }
 862
 863     metadata.stat.flags |= Flags::from_fat_attr(attr).bits();
 864
 865     Ok(())
 866 }
 867
 868 /// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
 869 fn get_quota_project_id(
 870     metadata: &mut Metadata,
 871     fd: RawFd,
 872     flags: Flags,
 873     magic: i64,
 874 ) -> Result<(), Error> {
 875     if !(metadata.is_dir() || metadata.is_regular_file()) {
 876         return Ok(());
 877     }
 878
 879     if flags.contains(Flags::WITH_QUOTA_PROJID) {
 880         return Ok(());
 881     }
 882
 883     use proxmox::sys::linux::magic::*;
 884
 885     match magic {
 886         EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC | ZFS_SUPER_MAGIC => (),
 887         _ => return Ok(()),
 888     }
 889
 890     let mut fsxattr = fs::FSXAttr::default();
 891     let res = unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) };
 892
 893     // On some FUSE filesystems it can happen that ioctl is not supported.
 894     // For these cases projid is set to 0 while the error is ignored.
 895     if let Err(err) = res {
 896         let errno = err
 897             .as_errno()
 898             .ok_or_else(|| format_err!("error while reading quota project id"))?;
 899         if errno_is_unsupported(errno) {
 900             return Ok(());
 901         } else {
 902             bail!("error while reading quota project id ({})", errno);
 903         }
 904     }
 905
 906     let projid = fsxattr.fsx_projid as u64;
 907     if projid != 0 {
 908         metadata.quota_project_id = Some(pxar::format::QuotaProjectId { projid });
 909     }
 910     Ok(())
 911 }
 912
 913 fn get_acl(metadata: &mut Metadata, proc_path: &Path, flags: Flags) -> Result<(), Error> {
 914     if flags.contains(Flags::WITH_ACL) {
 915         return Ok(());
 916     }
 917
 918     if metadata.is_symlink() {
 919         return Ok(());
 920     }
 921
 922     get_acl_do(metadata, proc_path, acl::ACL_TYPE_ACCESS)?;
 923
 924     if metadata.is_dir() {
 925         get_acl_do(metadata, proc_path, acl::ACL_TYPE_DEFAULT)?;
 926     }
 927
 928     Ok(())
 929 }
 930
 931 fn get_acl_do(
 932     metadata: &mut Metadata,
 933     proc_path: &Path,
 934     acl_type: acl::ACLType,
 935 ) -> Result<(), Error> {
 936     // In order to be able to get ACLs with type ACL_TYPE_DEFAULT, we have
 937     // to create a path for acl_get_file(). acl_get_fd() only allows to get
 938     // ACL_TYPE_ACCESS attributes.
 939     let acl = match acl::ACL::get_file(&proc_path, acl_type) {
 940         Ok(acl) => acl,
 941         // Don't bail if underlying endpoint does not support acls
 942         Err(Errno::EOPNOTSUPP) => return Ok(()),
 943         // Don't bail if the endpoint cannot carry acls
 944         Err(Errno::EBADF) => return Ok(()),
 945         // Don't bail if there is no data
 946         Err(Errno::ENODATA) => return Ok(()),
 947         Err(err) => bail!("error while reading ACL - {}", err),
 948     };
 949
 950     process_acl(metadata, acl, acl_type)
 951 }
 952
 953 fn process_acl(
 954     metadata: &mut Metadata,
 955     acl: acl::ACL,
 956     acl_type: acl::ACLType,
 957 ) -> Result<(), Error> {
 958     use pxar::format::acl as pxar_acl;
 959     use pxar::format::acl::{Group, GroupObject, Permissions, User};
 960
 961     let mut acl_user = Vec::new();
 962     let mut acl_group = Vec::new();
 963     let mut acl_group_obj = None;
 964     let mut acl_default = None;
 965     let mut user_obj_permissions = None;
 966     let mut group_obj_permissions = None;
 967     let mut other_permissions = None;
 968     let mut mask_permissions = None;
 969
 970     for entry in &mut acl.entries() {
 971         let tag = entry.get_tag_type()?;
 972         let permissions = entry.get_permissions()?;
 973         match tag {
 974             acl::ACL_USER_OBJ => user_obj_permissions = Some(Permissions(permissions)),
 975             acl::ACL_GROUP_OBJ => group_obj_permissions = Some(Permissions(permissions)),
 976             acl::ACL_OTHER => other_permissions = Some(Permissions(permissions)),
 977             acl::ACL_MASK => mask_permissions = Some(Permissions(permissions)),
 978             acl::ACL_USER => {
 979                 acl_user.push(User {
 980                     uid: entry.get_qualifier()?,
 981                     permissions: Permissions(permissions),
 982                 });
 983             }
 984             acl::ACL_GROUP => {
 985                 acl_group.push(Group {
 986                     gid: entry.get_qualifier()?,
 987                     permissions: Permissions(permissions),
 988                 });
 989             }
 990             _ => bail!("Unexpected ACL tag encountered!"),
 991         }
 992     }
 993
 994     acl_user.sort();
 995     acl_group.sort();
 996
 997     match acl_type {
 998         acl::ACL_TYPE_ACCESS => {
 999             // The mask permissions are mapped to the stat group permissions
1000             // in case that the ACL group permissions were set.
1001             // Only in that case we need to store the group permissions,
1002             // in the other cases they are identical to the stat group permissions.
1003             if let (Some(gop), true) = (group_obj_permissions, mask_permissions.is_some()) {
1004                 acl_group_obj = Some(GroupObject { permissions: gop });
1005             }
1006
1007             metadata.acl.users = acl_user;
1008             metadata.acl.groups = acl_group;
1009         }
1010         acl::ACL_TYPE_DEFAULT => {
1011             if user_obj_permissions != None
1012                 || group_obj_permissions != None
1013                 || other_permissions != None
1014                 || mask_permissions != None
1015             {
1016                 acl_default = Some(pxar_acl::Default {
1017                     // The value is set to UINT64_MAX as placeholder if one
1018                     // of the permissions is not set
1019                     user_obj_permissions: user_obj_permissions.unwrap_or(Permissions::NO_MASK),
1020                     group_obj_permissions: group_obj_permissions.unwrap_or(Permissions::NO_MASK),
1021                     other_permissions: other_permissions.unwrap_or(Permissions::NO_MASK),
1022                     mask_permissions: mask_permissions.unwrap_or(Permissions::NO_MASK),
1023                 });
1024             }
1025
1026             metadata.acl.default_users = acl_user;
1027             metadata.acl.default_groups = acl_group;
1028         }
1029         _ => bail!("Unexpected ACL type encountered"),
1030     }
1031
1032     metadata.acl.group_obj = acl_group_obj;
1033     metadata.acl.default = acl_default;
1034
1035     Ok(())
1036 }
1037
1038 /// Note that our pattern lists are "positive". `MatchType::Include` means the file is included.
1039 /// Since we are generating an *exclude* list, we need to invert this, so includes get a `'!'`
1040 /// prefix.
1041 fn generate_pxar_excludes_cli(patterns: &[MatchEntry]) -> Vec<u8> {
1042     use pathpatterns::MatchPattern;
1043
1044     let mut content = Vec::new();
1045
1046     for pattern in patterns {
1047         match pattern.match_type() {
1048             MatchType::Include => content.push(b'!'),
1049             MatchType::Exclude => (),
1050         }
1051
1052         match pattern.pattern() {
1053             MatchPattern::Literal(lit) => content.extend(lit),
1054             MatchPattern::Pattern(pat) => content.extend(pat.pattern().to_bytes()),
1055         }
1056
1057         if pattern.match_flags() == MatchFlag::MATCH_DIRECTORIES && content.last() != Some(&b'/') {
1058             content.push(b'/');
1059         }
1060
1061         content.push(b'\n');
1062     }
1063
1064     content
1065 }