use std::path::{Path, PathBuf};
use std::collections::{HashSet, HashMap};
use std::io::Write;
+use std::os::unix::fs::OpenOptionsExt;
+
use proxmox::tools::fs::{file_get_contents, file_get_json, file_set_contents, image_size};
use proxmox_backup::tools;
use serde_json::{json, Value};
//use hyper::Body;
-use std::sync::Arc;
+use std::sync::{Arc, Mutex};
use regex::Regex;
use xdg::BaseDirectories;
verbose: bool,
skip_lost_and_found: bool,
crypt_config: Option<Arc<CryptConfig>>,
+ catalog: Arc<Mutex<pxar::catalog::SimpleCatalog>>,
) -> Result<BackupStats, Error> {
- let pxar_stream = PxarBackupStream::open(dir_path.as_ref(), device_set, verbose, skip_lost_and_found)?;
+ let pxar_stream = PxarBackupStream::open(dir_path.as_ref(), device_set, verbose, skip_lost_and_found, catalog)?;
let chunk_stream = ChunkStream::new(pxar_stream, chunk_size);
let (tx, rx) = mpsc::channel(10); // allow to buffer 10 chunks
let mut file_list = vec![];
+ let catalog_filename = format!("/tmp/pbs-catalog-{}.cat", std::process::id());
+ let catalog = Arc::new(Mutex::new(pxar::catalog::SimpleCatalog::new(&catalog_filename)?));
+ let mut upload_catalog = false;
+
for (backup_type, filename, target, size) in upload_list {
match backup_type {
BackupType::CONFIG => {
file_list.push((target, stats));
}
BackupType::PXAR => {
+ upload_catalog = true;
println!("Upload directory '{}' to '{:?}' as {}", filename, repo, target);
let stats = backup_directory(
&client,
verbose,
skip_lost_and_found,
crypt_config.clone(),
+ catalog.clone(),
)?;
file_list.push((target, stats));
}
}
}
+ // finalize and upload catalog
+ if upload_catalog {
+ let mutex = Arc::try_unwrap(catalog)
+ .map_err(|_| format_err!("unable to get catalog (still used)"))?;
+ drop(mutex); // close catalog
+
+ let target = "catalog.blob";
+ let stats = client.upload_blob_from_file(&catalog_filename, target, crypt_config.clone(), true).wait()?;
+ file_list.push((target.to_owned(), stats));
+
+ let _ = std::fs::remove_file(&catalog_filename);
+ }
+
if let Some(rsa_encrypted_key) = rsa_encrypted_key {
let target = "rsa-encrypted.key";
println!("Upload RSA encoded key to '{:?}' as {}", repo, target);
let client = client.start_backup_reader(repo.store(), &backup_type, &backup_id, backup_time, true).wait()?;
- use std::os::unix::fs::OpenOptionsExt;
-
let tmpfile = std::fs::OpenOptions::new()
.write(true)
.read(true)
feature_flags ^= pxar::flags::WITH_SOCKETS;
}
- pxar::Encoder::encode(source, &mut dir, &mut writer, devices, verbose, false, feature_flags)?;
+ let catalog = None::<&mut pxar::catalog::SimpleCatalog>;
+ pxar::Encoder::encode(source, &mut dir, &mut writer, catalog, devices, verbose, false, feature_flags)?;
writer.flush()?;
impl PxarBackupStream {
- pub fn new(mut dir: Dir, path: PathBuf, device_set: Option<HashSet<u64>>, verbose: bool, skip_lost_and_found: bool) -> Result<Self, Error> {
+ pub fn new(
+ mut dir: Dir,
+ path: PathBuf,
+ device_set: Option<HashSet<u64>>,
+ verbose: bool,
+ skip_lost_and_found: bool,
+ catalog: Arc<Mutex<crate::pxar::catalog::SimpleCatalog>>,
+ ) -> Result<Self, Error> {
let (rx, tx) = nix::unistd::pipe()?;
let error = Arc::new(Mutex::new(None));
let error2 = error.clone();
- let child = thread::spawn(move|| {
+ let catalog = catalog.clone();
+ let child = thread::spawn(move || {
+ let mut guard = catalog.lock().unwrap();
let mut writer = unsafe { std::fs::File::from_raw_fd(tx) };
- if let Err(err) = pxar::Encoder::encode(path, &mut dir, &mut writer, device_set, verbose, skip_lost_and_found, pxar::flags::DEFAULT) {
+ if let Err(err) = pxar::Encoder::encode(path, &mut dir, &mut writer, Some(&mut *guard), device_set, verbose, skip_lost_and_found, pxar::flags::DEFAULT) {
let mut error = error2.lock().unwrap();
*error = Some(err.to_string());
}
})
}
- pub fn open(dirname: &Path, device_set: Option<HashSet<u64>>, verbose: bool, skip_lost_and_found: bool) -> Result<Self, Error> {
+ pub fn open(
+ dirname: &Path,
+ device_set: Option<HashSet<u64>>,
+ verbose: bool,
+ skip_lost_and_found: bool,
+ catalog: Arc<Mutex<crate::pxar::catalog::SimpleCatalog>>,
+ ) -> Result<Self, Error> {
let dir = nix::dir::Dir::open(dirname, OFlag::O_DIRECTORY, Mode::empty())?;
let path = std::path::PathBuf::from(dirname);
- Self::new(dir, path, device_set, verbose, skip_lost_and_found)
+ Self::new(dir, path, device_set, verbose, skip_lost_and_found, catalog)
}
}
use super::binary_search_tree::*;
use super::helper::*;
use super::match_pattern::*;
+use super::catalog::BackupCatalogWriter;
+
use crate::tools::fs;
use crate::tools::acl;
use crate::tools::xattr;
st_ino: u64,
}
-pub struct Encoder<'a, W: Write> {
+pub struct Encoder<'a, W: Write, C: BackupCatalogWriter> {
base_path: PathBuf,
relative_path: PathBuf,
writer: &'a mut W,
writer_pos: usize,
+ catalog: Option<&'a mut C>,
_size: usize,
file_copy_buffer: Vec<u8>,
device_set: Option<HashSet<u64>>,
hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
}
-impl <'a, W: Write> Encoder<'a, W> {
+impl <'a, W: Write, C: BackupCatalogWriter> Encoder<'a, W, C> {
// used for error reporting
fn full_path(&self) -> PathBuf {
path: PathBuf,
dir: &mut nix::dir::Dir,
writer: &'a mut W,
+ catalog: Option<&'a mut C>,
device_set: Option<HashSet<u64>>,
verbose: bool,
skip_lost_and_found: bool, // fixme: should be a feature flag ??
relative_path: PathBuf::new(),
writer: writer,
writer_pos: 0,
+ catalog,
_size: 0,
file_copy_buffer,
device_set,
};
self.write_filename(&filename)?;
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.start_directory(&filename)?;
+ }
self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.end_directory()?;
+ }
} else if is_reg_file(&stat) {
}
if let Some((target, offset)) = hardlink_target {
-
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_hardlink(&filename)?;
+ }
self.write_filename(&filename)?;
self.encode_hardlink(target.as_bytes(), offset)?;
Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
};
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
+ }
let child_magic = if dir_stat.st_dev != stat.st_dev {
detect_fs_type(filefd)?
} else {
}
} else if is_symlink(&stat) {
+
let mut buffer = vec::undefined(libc::PATH_MAX as usize);
let res = filename.with_nix_path(|cstr| {
match Errno::result(res) {
Ok(len) => {
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_symlink(&filename)?;
+ }
buffer[len as usize] = 0u8; // add Nul byte
self.write_filename(&filename)?;
self.encode_symlink(&buffer[..((len+1) as usize)], &stat)?
}
} else if is_block_dev(&stat) || is_char_dev(&stat) {
if self.has_features(flags::WITH_DEVICE_NODES) {
+ if let Some(ref mut catalog) = self.catalog {
+ if is_block_dev(&stat) {
+ catalog.add_block_device(&filename)?;
+ } else {
+ catalog.add_char_device(&filename)?;
+ }
+ }
self.write_filename(&filename)?;
self.encode_device(&stat)?;
} else {
}
} else if is_fifo(&stat) {
if self.has_features(flags::WITH_FIFOS) {
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_fifo(&filename)?;
+ }
self.write_filename(&filename)?;
self.encode_special(&stat)?;
} else {
}
} else if is_socket(&stat) {
if self.has_features(flags::WITH_SOCKETS) {
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_socket(&filename)?;
+ }
self.write_filename(&filename)?;
self.encode_special(&stat)?;
} else {
let path = std::path::PathBuf::from(dir_name);
- Encoder::encode(path, &mut dir, &mut writer, None, false, false, flags::DEFAULT)?;
+ let catalog = None::<&mut catalog::SimpleCatalog>;
+ Encoder::encode(path, &mut dir, &mut writer, catalog, None, false, false, flags::DEFAULT)?;
Command::new("cmp")
.arg("--verbose")
projects / proxmox-backup.git / commitdiff