From 77486a608e08a8acaeadfe28bb8aefca21353d20 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Fabian=20Gr=C3=BCnbichler?= Date: Mon, 25 Jan 2021 14:42:54 +0100 Subject: [PATCH] pxar: factor out PxarCreateOptions MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit containing the CLI parameters that are mostly passed-through from the client to our pxar archive creation wrapper in pxar::create Signed-off-by: Fabian Grünbichler --- src/bin/proxmox-backup-client.rs | 27 ++++++++++++--------------- src/bin/pxar.rs | 22 ++++++++++++++-------- src/client/pxar_backup_stream.rs | 28 ++++++---------------------- src/pxar/create.rs | 28 ++++++++++++++++++++++------ src/pxar/mod.rs | 2 +- tests/catar.rs | 10 ++++++---- 6 files changed, 61 insertions(+), 56 deletions(-) diff --git a/src/bin/proxmox-backup-client.rs b/src/bin/proxmox-backup-client.rs index af3b8464..c3d6a0ed 100644 --- a/src/bin/proxmox-backup-client.rs +++ b/src/bin/proxmox-backup-client.rs @@ -283,23 +283,15 @@ async fn backup_directory>( dir_path: P, archive_name: &str, chunk_size: Option, - device_set: Option>, - verbose: bool, - skip_lost_and_found: bool, catalog: Arc>>, - exclude_pattern: Vec, - entries_max: usize, + pxar_create_options: proxmox_backup::pxar::PxarCreateOptions, upload_options: UploadOptions, ) -> Result { let pxar_stream = PxarBackupStream::open( dir_path.as_ref(), - device_set, - verbose, - skip_lost_and_found, catalog, - exclude_pattern, - entries_max, + pxar_create_options, )?; let mut chunk_stream = ChunkStream::new(pxar_stream, chunk_size); @@ -1039,6 +1031,15 @@ async fn create_backup( println!("Upload directory '{}' to '{}' as {}", filename, repo, target); catalog.lock().unwrap().start_directory(std::ffi::CString::new(target.as_str())?.as_c_str())?; + + let pxar_options = proxmox_backup::pxar::PxarCreateOptions { + device_set: devices.clone(), + patterns: pattern_list.clone(), + entries_max: entries_max as usize, + skip_lost_and_found, + verbose, + }; + let upload_options = UploadOptions { previous_manifest: previous_manifest.clone(), compress: true, @@ -1051,12 +1052,8 @@ async fn create_backup( &filename, &target, chunk_size_opt, - devices.clone(), - verbose, - skip_lost_and_found, catalog.clone(), - pattern_list.clone(), - entries_max as usize, + pxar_options, upload_options, ).await?; manifest.add_file(target, stats.size, stats.csum, crypt_mode)?; diff --git a/src/bin/pxar.rs b/src/bin/pxar.rs index 3d92dbe0..85606c1d 100644 --- a/src/bin/pxar.rs +++ b/src/bin/pxar.rs @@ -311,16 +311,16 @@ fn create_archive( exclude: Option>, entries_max: isize, ) -> Result<(), Error> { - let pattern_list = { + let patterns = { let input = exclude.unwrap_or_else(Vec::new); - let mut pattern_list = Vec::with_capacity(input.len()); + let mut patterns = Vec::with_capacity(input.len()); for entry in input { - pattern_list.push( + patterns.push( MatchEntry::parse_pattern(entry, PatternFlag::PATH_NAME, MatchType::Exclude) .map_err(|err| format_err!("error in exclude pattern: {}", err))?, ); } - pattern_list + patterns }; let device_set = if all_file_systems { @@ -329,6 +329,15 @@ fn create_archive( Some(HashSet::new()) }; + let options = proxmox_backup::pxar::PxarCreateOptions { + entries_max: entries_max as usize, + device_set, + patterns, + verbose, + skip_lost_and_found: false, + }; + + let source = PathBuf::from(source); let dir = nix::dir::Dir::open( @@ -368,18 +377,15 @@ fn create_archive( proxmox_backup::pxar::create_archive( dir, writer, - pattern_list, feature_flags, - device_set, - false, |path| { if verbose { println!("{:?}", path); } Ok(()) }, - entries_max as usize, None, + options, )?; Ok(()) diff --git a/src/client/pxar_backup_stream.rs b/src/client/pxar_backup_stream.rs index aa3355fe..5fb28fd5 100644 --- a/src/client/pxar_backup_stream.rs +++ b/src/client/pxar_backup_stream.rs @@ -1,4 +1,3 @@ -use std::collections::HashSet; use std::io::Write; //use std::os::unix::io::FromRawFd; use std::path::Path; @@ -13,8 +12,6 @@ use nix::dir::Dir; use nix::fcntl::OFlag; use nix::sys::stat::Mode; -use pathpatterns::MatchEntry; - use crate::backup::CatalogWriter; /// Stream implementation to encode and upload .pxar archives. @@ -38,12 +35,8 @@ impl Drop for PxarBackupStream { impl PxarBackupStream { pub fn new( dir: Dir, - device_set: Option>, - verbose: bool, - skip_lost_and_found: bool, catalog: Arc>>, - patterns: Vec, - entries_max: usize, + options: crate::pxar::PxarCreateOptions, ) -> Result { let (tx, rx) = std::sync::mpsc::sync_channel(10); @@ -61,22 +54,21 @@ impl PxarBackupStream { crate::tools::StdChannelWriter::new(tx), ); + let verbose = options.verbose; + let writer = pxar::encoder::sync::StandardWriter::new(writer); if let Err(err) = crate::pxar::create_archive( dir, writer, - patterns, crate::pxar::Flags::DEFAULT, - device_set, - skip_lost_and_found, |path| { if verbose { println!("{:?}", path); } Ok(()) }, - entries_max, Some(&mut *catalog_guard), + options, ) { let mut error = error.lock().unwrap(); *error = Some(err.to_string()); @@ -93,23 +85,15 @@ impl PxarBackupStream { pub fn open( dirname: &Path, - device_set: Option>, - verbose: bool, - skip_lost_and_found: bool, catalog: Arc>>, - patterns: Vec, - entries_max: usize, + options: crate::pxar::PxarCreateOptions, ) -> Result { let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?; Self::new( dir, - device_set, - verbose, - skip_lost_and_found, catalog, - patterns, - entries_max, + options, ) } } diff --git a/src/pxar/create.rs b/src/pxar/create.rs index a21511c3..229e3f59 100644 --- a/src/pxar/create.rs +++ b/src/pxar/create.rs @@ -27,6 +27,22 @@ use crate::pxar::Flags; use crate::pxar::tools::assert_single_path_component; use crate::tools::{acl, fs, xattr, Fd}; +/// Pxar options for creating a pxar archive/stream +#[derive(Default, Clone)] +pub struct PxarCreateOptions { + /// Device/mountpoint st_dev numbers that should be included. None for no limitation. + pub device_set: Option>, + /// Exclusion patterns + pub patterns: Vec, + /// Maximum number of entries to hold in memory + pub entries_max: usize, + /// Skip lost+found directory + pub skip_lost_and_found: bool, + /// Verbose output + pub verbose: bool, +} + + fn detect_fs_type(fd: RawFd) -> Result { let mut fs_stat = std::mem::MaybeUninit::uninit(); let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) }; @@ -136,13 +152,10 @@ type Encoder<'a, 'b> = pxar::encoder::Encoder<'a, &'b mut dyn pxar::encoder::Seq pub fn create_archive( source_dir: Dir, mut writer: T, - mut patterns: Vec, feature_flags: Flags, - mut device_set: Option>, - skip_lost_and_found: bool, mut callback: F, - entry_limit: usize, catalog: Option<&mut dyn BackupCatalogWriter>, + options: PxarCreateOptions, ) -> Result<(), Error> where T: pxar::encoder::SeqWrite, @@ -164,6 +177,7 @@ where ) .map_err(|err| format_err!("failed to get metadata for source directory: {}", err))?; + let mut device_set = options.device_set.clone(); if let Some(ref mut set) = device_set { set.insert(stat.st_dev); } @@ -171,7 +185,9 @@ where let writer = &mut writer as &mut dyn pxar::encoder::SeqWrite; let mut encoder = Encoder::new(writer, &metadata)?; - if skip_lost_and_found { + let mut patterns = options.patterns.clone(); + + if options.skip_lost_and_found { patterns.push(MatchEntry::parse_pattern( "lost+found", PatternFlag::PATH_NAME, @@ -188,7 +204,7 @@ where catalog, path: PathBuf::new(), entry_counter: 0, - entry_limit, + entry_limit: options.entries_max, current_st_dev: stat.st_dev, device_set, hardlinks: HashMap::new(), diff --git a/src/pxar/mod.rs b/src/pxar/mod.rs index 82998cf8..c10bb6fb 100644 --- a/src/pxar/mod.rs +++ b/src/pxar/mod.rs @@ -58,7 +58,7 @@ pub(crate) mod tools; mod flags; pub use flags::Flags; -pub use create::create_archive; +pub use create::{create_archive, PxarCreateOptions}; pub use extract::{extract_archive, ErrorHandler}; /// The format requires to build sorted directory lookup tables in diff --git a/tests/catar.rs b/tests/catar.rs index a6acf9af..2d9dea71 100644 --- a/tests/catar.rs +++ b/tests/catar.rs @@ -25,16 +25,18 @@ fn run_test(dir_name: &str) -> Result<(), Error> { dir_name, nix::fcntl::OFlag::O_NOFOLLOW, nix::sys::stat::Mode::empty())?; + let options = PxarCreateOptions { + entries_max: ENCODER_MAX_ENTRIES, + ..PxarCreateOptions::default() + }; + create_archive( dir, writer, - Vec::new(), Flags::DEFAULT, - None, - false, |_| Ok(()), - ENCODER_MAX_ENTRIES, None, + options, )?; Command::new("cmp") -- 2.39.2