//! *pxar* format encoder.
//!
//! This module contain the code to generate *pxar* archive files.
-
-use failure::*;
-use endian_trait::Endian;
-use std::collections::{HashSet, HashMap};
-
-use super::flags;
-use super::format_definition::*;
-use super::binary_search_tree::*;
-use super::helper::*;
-use super::match_pattern::*;
-use crate::tools::fs;
-use crate::tools::acl;
-use crate::tools::xattr;
-
+use std::collections::{HashMap, HashSet};
+use std::ffi::{CStr, CString};
use std::io::Write;
-use std::os::unix::io::AsRawFd;
use std::os::unix::ffi::OsStrExt;
+use std::os::unix::io::AsRawFd;
use std::os::unix::io::RawFd;
use std::path::{Path, PathBuf};
-use std::ffi::CStr;
-
-use nix::NixPath;
-use nix::fcntl::OFlag;
-use nix::sys::stat::Mode;
+use endian_trait::Endian;
+use anyhow::{bail, format_err, Error};
use nix::errno::Errno;
+use nix::fcntl::OFlag;
use nix::sys::stat::FileStat;
+use nix::sys::stat::Mode;
+use nix::NixPath;
use proxmox::tools::vec;
-/// The format requires to build sorted directory lookup tables in
-/// memory, so we restrict the number of allowed entries to limit
-/// maximum memory usage.
-pub const MAX_DIRECTORY_ENTRIES: usize = 256*1024;
+use super::binary_search_tree::*;
+use super::catalog::BackupCatalogWriter;
+use super::flags;
+use super::format_definition::*;
+use super::helper::*;
+use super::match_pattern::{MatchPattern, MatchPatternSlice, MatchType};
+use crate::tools::acl;
+use crate::tools::fs;
+use crate::tools::xattr;
#[derive(Eq, PartialEq, Hash)]
struct HardLinkInfo {
st_ino: u64,
}
-pub struct Encoder<'a, W: Write> {
+pub struct Encoder<'a, W: Write, C: BackupCatalogWriter> {
base_path: PathBuf,
relative_path: PathBuf,
writer: &'a mut W,
writer_pos: usize,
+ catalog: Option<&'a mut C>,
_size: usize,
file_copy_buffer: Vec<u8>,
device_set: Option<HashSet<u64>>,
// Flags signaling features supported by the filesystem
fs_feature_flags: u64,
hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
+ entry_counter: usize,
+ entry_max: usize,
}
-impl <'a, W: Write> Encoder<'a, W> {
-
+impl<'a, W: Write, C: BackupCatalogWriter> Encoder<'a, W, C> {
// used for error reporting
- fn full_path(&self) -> PathBuf {
+ fn full_path(&self) -> PathBuf {
self.base_path.join(&self.relative_path)
}
path: PathBuf,
dir: &mut nix::dir::Dir,
writer: &'a mut W,
+ catalog: Option<&'a mut C>,
device_set: Option<HashSet<u64>>,
verbose: bool,
skip_lost_and_found: bool, // fixme: should be a feature flag ??
feature_flags: u64,
+ mut excludes: Vec<MatchPattern>,
+ entry_max: usize,
) -> Result<(), Error> {
-
- const FILE_COPY_BUFFER_SIZE: usize = 1024*1024;
+ const FILE_COPY_BUFFER_SIZE: usize = 1024 * 1024;
let mut file_copy_buffer = Vec::with_capacity(FILE_COPY_BUFFER_SIZE);
- unsafe { file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE); }
-
+ unsafe {
+ file_copy_buffer.set_len(FILE_COPY_BUFFER_SIZE);
+ }
// todo: use scandirat??
let mut me = Self {
base_path: path,
relative_path: PathBuf::new(),
- writer: writer,
+ writer,
writer_pos: 0,
+ catalog,
_size: 0,
file_copy_buffer,
device_set,
feature_flags,
fs_feature_flags,
hardlinks: HashMap::new(),
+ entry_counter: 0,
+ entry_max,
};
- if verbose { println!("{:?}", me.full_path()); }
+ if verbose {
+ println!("{:?}", me.full_path());
+ }
- let mut excludes = Vec::new();
if skip_lost_and_found {
excludes.push(MatchPattern::from_line(b"**/lost+found").unwrap().unwrap());
}
- me.encode_dir(dir, &stat, magic, excludes)?;
+ let mut exclude_slices = Vec::new();
+ for excl in &excludes {
+ exclude_slices.push(excl.as_slice());
+ }
+
+ me.encode_dir(dir, &stat, magic, exclude_slices)?;
Ok(())
}
- fn write(&mut self, buf: &[u8]) -> Result<(), Error> {
+ fn write(&mut self, buf: &[u8]) -> Result<(), Error> {
self.writer.write_all(buf)?;
self.writer_pos += buf.len();
Ok(())
}
- fn write_item<T: Endian>(&mut self, item: T) -> Result<(), Error> {
-
+ fn write_item<T: Endian>(&mut self, item: T) -> Result<(), Error> {
let data = item.to_le();
- let buffer = unsafe { std::slice::from_raw_parts(
- &data as *const T as *const u8,
- std::mem::size_of::<T>()
- )};
+ let buffer = unsafe {
+ std::slice::from_raw_parts(&data as *const T as *const u8, std::mem::size_of::<T>())
+ };
self.write(buffer)?;
}
fn write_header(&mut self, htype: u64, size: u64) -> Result<(), Error> {
-
let size = size + (std::mem::size_of::<PxarHeader>() as u64);
self.write_item(PxarHeader { size, htype })?;
}
fn write_filename(&mut self, name: &CStr) -> Result<(), Error> {
-
let buffer = name.to_bytes_with_nul();
self.write_header(PXAR_FILENAME, buffer.len() as u64)?;
self.write(buffer)?;
}
fn create_entry(&self, stat: &FileStat) -> Result<PxarEntry, Error> {
-
let mode = if is_symlink(&stat) {
(libc::S_IFLNK | 0o777) as u64
} else {
bail!("got strange mtime ({}) from fstat for {:?}.", mtime, self.full_path());
}
-
let entry = PxarEntry {
- mode: mode,
+ mode,
flags: 0,
uid: stat.st_uid,
gid: stat.st_gid,
}
fn read_chattr(&self, fd: RawFd, entry: &mut PxarEntry) -> Result<(), Error> {
-
let mut attr: usize = 0;
- let res = unsafe { fs::read_attr_fd(fd, &mut attr)};
+ let res = unsafe { fs::read_attr_fd(fd, &mut attr) };
if let Err(err) = res {
if let nix::Error::Sys(errno) = err {
- if errno_is_unsupported(errno) { return Ok(()) };
+ if errno_is_unsupported(errno) {
+ return Ok(());
+ };
}
bail!("read_attr_fd failed for {:?} - {}", self.full_path(), err);
}
let flags = flags::feature_flags_from_chattr(attr as u32);
- entry.flags = entry.flags | flags;
+ entry.flags |= flags;
Ok(())
}
let mut attr: u32 = 0;
- let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr)};
+ let res = unsafe { fs::read_fat_attr_fd(fd, &mut attr) };
if let Err(err) = res {
if let nix::Error::Sys(errno) = err {
- if errno_is_unsupported(errno) { return Ok(()) };
+ if errno_is_unsupported(errno) {
+ return Ok(());
+ };
}
bail!("read_fat_attr_fd failed for {:?} - {}", self.full_path(), err);
}
let flags = flags::feature_flags_from_fat_attr(attr);
- entry.flags = entry.flags | flags;
+ entry.flags |= flags;
Ok(())
}
(self.feature_flags & self.fs_feature_flags & feature_flags) != 0
}
- fn read_xattrs(&self, fd: RawFd, stat: &FileStat) -> Result<(Vec<PxarXAttr>, Option<PxarFCaps>), Error> {
+ fn read_xattrs(
+ &self,
+ fd: RawFd,
+ stat: &FileStat,
+ ) -> Result<(Vec<PxarXAttr>, Option<PxarFCaps>), Error> {
let mut xattrs = Vec::new();
let mut fcaps = None;
Err(err) => bail!("read_xattrs failed for {:?} - {}", self.full_path(), err),
};
- for name in xattr_names.split(|c| *c == b'\0') {
+ for name in &xattr_names {
// Only extract the relevant extended attributes
if !xattr::is_valid_xattr_name(&name) {
continue;
if xattr::is_security_capability(&name) {
if self.has_features(flags::WITH_FCAPS) {
// fcaps are stored in own format within the archive
- fcaps = Some(PxarFCaps {
- data: value,
- });
+ fcaps = Some(PxarFCaps { data: value });
}
} else if self.has_features(flags::WITH_XATTRS) {
xattrs.push(PxarXAttr {
- name: name.to_vec(),
- value: value,
+ name: name.to_bytes().to_vec(),
+ value,
});
}
}
Ok((xattrs, fcaps))
}
- fn read_acl(&self, fd: RawFd, stat: &FileStat, acl_type: acl::ACLType) -> Result<PxarACL, Error> {
+ fn read_acl(
+ &self,
+ fd: RawFd,
+ stat: &FileStat,
+ acl_type: acl::ACLType,
+ ) -> Result<PxarACL, Error> {
let ret = PxarACL {
users: Vec::new(),
groups: Vec::new(),
acl::ACL_USER => {
acl_user.push(PxarACLUser {
uid: entry.get_qualifier()?,
- permissions: permissions,
+ permissions,
});
- },
+ }
acl::ACL_GROUP => {
acl_group.push(PxarACLGroup {
gid: entry.get_qualifier()?,
- permissions: permissions,
+ permissions,
});
- },
+ }
_ => bail!("Unexpected ACL tag encountered!"),
}
}
// Only in that case we need to store the group permissions,
// in the other cases they are identical to the stat group permissions.
if let (Some(gop), Some(_)) = (group_obj_permissions, mask_permissions) {
- acl_group_obj = Some(PxarACLGroupObj {
- permissions: gop,
- });
+ acl_group_obj = Some(PxarACLGroupObj { permissions: gop });
}
- },
+ }
acl::ACL_TYPE_DEFAULT => {
- if user_obj_permissions != None ||
- group_obj_permissions != None ||
- other_permissions != None ||
- mask_permissions != None
+ if user_obj_permissions != None
+ || group_obj_permissions != None
+ || other_permissions != None
+ || mask_permissions != None
{
acl_default = Some(PxarACLDefault {
// The value is set to UINT64_MAX as placeholder if one
mask_permissions: mask_permissions.unwrap_or(std::u64::MAX),
});
}
- },
+ }
_ => bail!("Unexpected ACL type encountered"),
}
}
/// Read the quota project id for an inode, supported on ext4/XFS/FUSE/ZFS filesystems
- fn read_quota_project_id(&self, fd: RawFd, magic: i64, stat: &FileStat) -> Result<Option<PxarQuotaProjID>, Error> {
+ fn read_quota_project_id(
+ &self,
+ fd: RawFd,
+ magic: i64,
+ stat: &FileStat,
+ ) -> Result<Option<PxarQuotaProjID>, Error> {
if !(is_directory(&stat) || is_reg_file(&stat)) {
return Ok(None);
}
use proxmox::sys::linux::magic::*;
match magic {
- EXT4_SUPER_MAGIC |
- XFS_SUPER_MAGIC |
- FUSE_SUPER_MAGIC |
- ZFS_SUPER_MAGIC => {
+ EXT4_SUPER_MAGIC | XFS_SUPER_MAGIC | FUSE_SUPER_MAGIC | ZFS_SUPER_MAGIC => {
let mut fsxattr = fs::FSXAttr::default();
- let res = unsafe {
- fs::fs_ioc_fsgetxattr(fd, &mut fsxattr)
- };
+ let res = unsafe { fs::fs_ioc_fsgetxattr(fd, &mut fsxattr) };
// On some FUSE filesystems it can happen that ioctl is not supported.
// For these cases projid is set to 0 while the error is ignored.
if let Err(err) = res {
let errno = err.as_errno().ok_or_else(|| {
- format_err!("error while reading quota project id for {:#?}", self.full_path())
+ format_err!(
+ "error while reading quota project id for {:#?}",
+ self.full_path()
+ )
})?;
if errno_is_unsupported(errno) {
return Ok(None);
} else {
- bail!("error while reading quota project id for {:#?} - {}", self.full_path(), errno);
+ bail!(
+ "error while reading quota project id for {:#?} - {}",
+ self.full_path(),
+ errno
+ );
}
}
let projid = fsxattr.fsx_projid as u64;
if projid == 0 {
- return Ok(None);
+ Ok(None)
} else {
- return Ok(Some(PxarQuotaProjID { projid }));
+ Ok(Some(PxarQuotaProjID { projid }))
}
- },
- _ => return Ok(None),
+ }
+ _ => Ok(None),
}
}
fn write_entry(&mut self, entry: PxarEntry) -> Result<(), Error> {
-
self.write_header(PXAR_ENTRY, std::mem::size_of::<PxarEntry>() as u64)?;
self.write_item(entry)?;
}
fn write_acl_user(&mut self, acl_user: PxarACLUser) -> Result<(), Error> {
- self.write_header(PXAR_ACL_USER,
std::mem::size_of::<PxarACLUser>() as u64)?;
+ self.write_header(PXAR_ACL_USER, std::mem::size_of::<PxarACLUser>() as u64)?;
self.write_item(acl_user)?;
Ok(())
}
fn write_acl_group(&mut self, acl_group: PxarACLGroup) -> Result<(), Error> {
- self.write_header(PXAR_ACL_GROUP,
std::mem::size_of::<PxarACLGroup>() as u64)?;
+ self.write_header(PXAR_ACL_GROUP, std::mem::size_of::<PxarACLGroup>() as u64)?;
self.write_item(acl_group)?;
Ok(())
}
fn write_acl_group_obj(&mut self, acl_group_obj: PxarACLGroupObj) -> Result<(), Error> {
- self.write_header(PXAR_ACL_GROUP_OBJ, std::mem::size_of::<PxarACLGroupObj>() as u64)?;
+ self.write_header(
+ PXAR_ACL_GROUP_OBJ,
+ std::mem::size_of::<PxarACLGroupObj>() as u64,
+ )?;
self.write_item(acl_group_obj)?;
Ok(())
}
fn write_acl_default(&mut self, acl_default: PxarACLDefault) -> Result<(), Error> {
- self.write_header(PXAR_ACL_DEFAULT, std::mem::size_of::<PxarACLDefault>() as u64)?;
+ self.write_header(
+ PXAR_ACL_DEFAULT,
+ std::mem::size_of::<PxarACLDefault>() as u64,
+ )?;
self.write_item(acl_default)?;
Ok(())
}
fn write_acl_default_user(&mut self, acl_default_user: PxarACLUser) -> Result<(), Error> {
- self.write_header(PXAR_ACL_DEFAULT_USER, std::mem::size_of::<PxarACLUser>() as u64)?;
+ self.write_header(
+ PXAR_ACL_DEFAULT_USER,
+ std::mem::size_of::<PxarACLUser>() as u64,
+ )?;
self.write_item(acl_default_user)?;
Ok(())
}
fn write_acl_default_group(&mut self, acl_default_group: PxarACLGroup) -> Result<(), Error> {
- self.write_header(PXAR_ACL_DEFAULT_GROUP, std::mem::size_of::<PxarACLGroup>() as u64)?;
+ self.write_header(
+ PXAR_ACL_DEFAULT_GROUP,
+ std::mem::size_of::<PxarACLGroup>() as u64,
+ )?;
self.write_item(acl_default_group)?;
Ok(())
}
fn write_quota_project_id(&mut self, projid: PxarQuotaProjID) -> Result<(), Error> {
- self.write_header(PXAR_QUOTA_PROJID, std::mem::size_of::<PxarQuotaProjID>() as u64)?;
+ self.write_header(
+ PXAR_QUOTA_PROJID,
+ std::mem::size_of::<PxarQuotaProjID>() as u64,
+ )?;
self.write_item(projid)?;
Ok(())
}
- fn write_goodbye_table(&mut self, goodbye_offset: usize, goodbye_items: &mut [PxarGoodbyeItem]) -> Result<(), Error> {
-
+ fn write_goodbye_table(
+ &mut self,
+ goodbye_offset: usize,
+ goodbye_items: &mut [PxarGoodbyeItem],
+ ) -> Result<(), Error> {
goodbye_items.sort_unstable_by(|a, b| a.hash.cmp(&b.hash));
let item_count = goodbye_items.len();
- let goodbye_table_size = (item_count + 1)
*std::mem::size_of::<PxarGoodbyeItem>();
+ let goodbye_table_size = (item_count + 1)
* std::mem::size_of::<PxarGoodbyeItem>();
self.write_header(PXAR_GOODBYE, goodbye_table_size as u64)?;
if self.file_copy_buffer.len() < goodbye_table_size {
let need = goodbye_table_size - self.file_copy_buffer.len();
self.file_copy_buffer.reserve(need);
- unsafe { self.file_copy_buffer.set_len(self.file_copy_buffer.capacity()); }
+ unsafe {
+ self.file_copy_buffer
+ .set_len(self.file_copy_buffer.capacity());
+ }
}
let buffer = &mut self.file_copy_buffer;
copy_binary_search_tree(item_count, |s, d| {
let item = &goodbye_items[s];
- let offset = d*std::mem::size_of::<PxarGoodbyeItem>();
- let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
+ let offset = d * std::mem::size_of::<PxarGoodbyeItem>();
+ let dest =
+ crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
dest.offset = u64::to_le(item.offset);
dest.size = u64::to_le(item.size);
dest.hash = u64::to_le(item.hash);
});
// append PxarGoodbyeTail as last item
- let offset = item_count
*std::mem::size_of::<PxarGoodbyeItem>();
+ let offset = item_count
* std::mem::size_of::<PxarGoodbyeItem>();
let dest = crate::tools::map_struct_mut::<PxarGoodbyeItem>(&mut buffer[offset..]).unwrap();
dest.offset = u64::to_le(goodbye_offset as u64);
dest.size = u64::to_le((goodbye_table_size + std::mem::size_of::<PxarHeader>()) as u64);
dir: &mut nix::dir::Dir,
dir_stat: &FileStat,
magic: i64,
- match_pattern: Vec<MatchPattern>,
+ match_pattern: Vec<MatchPatternSlice>,
) -> Result<(), Error> {
-
//println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
- let mut name_list = vec![];
+ let mut name_list = Vec::new();
let rawfd = dir.as_raw_fd();
let dir_start_pos = self.writer_pos;
+ let is_root = dir_start_pos == 0;
+
let mut dir_entry = self.create_entry(&dir_stat)?;
self.read_chattr(rawfd, &mut dir_entry)?;
let include_children;
if is_virtual_file_system(magic) {
include_children = false;
+ } else if let Some(set) = &self.device_set {
+ include_children = set.contains(&dir_stat.st_dev);
} else {
- if let Some(set) = &self.device_set {
- include_children = set.contains(&dir_stat.st_dev);
- } else {
- include_children = true;
- }
+ include_children = true;
}
// Expand the exclude match pattern inherited from the parent by local entries, if present
let mut local_match_pattern = match_pattern.clone();
- let pxar_exclude = match MatchPattern::from_file(rawfd, ".pxarexclude") {
- Ok(Some((mut excludes, buffer, stat))) => {
- local_match_pattern.append(&mut excludes);
- Some((buffer, stat))
- },
- Ok(None) => None,
+ let (pxar_exclude, excludes) = match MatchPattern::from_file(rawfd, ".pxarexclude") {
+ Ok(Some((excludes, buffer, stat))) => {
+ (Some((buffer, stat)), excludes)
+ }
+ Ok(None) => (None, Vec::new()),
+ Err(nix::Error::Sys(Errno::EACCES)) => {
+ // No permission to read .pxarexclude, ignore its contents.
+ eprintln!(
+ "ignoring match patterns in {:?}: open file failed - EACCES",
+ self.full_path().join(".pxarexclude"),
+ );
+ (None, Vec::new())
+ }
Err(err) => bail!("error while reading exclude file - {}", err),
};
+ for excl in &excludes {
+ local_match_pattern.push(excl.as_slice());
+ }
if include_children {
+ // Exclude patterns passed via the CLI are stored as '.pxarexclude-cli'
+ // in the root directory of the archive.
+ if is_root && !match_pattern.is_empty() {
+ let filename = CString::new(".pxarexclude-cli")?;
+ name_list.push((filename, *dir_stat, match_pattern.clone()));
+ }
+
for entry in dir.iter() {
- let entry = entry.map_err(|err| {
- format_err!("readir {:?} failed - {}", self.full_path(), err)
- })?;
+ let entry = entry
+ .map_err(|err| format_err!("readir {:?} failed - {}", self.full_path(), err))?;
let filename = entry.file_name().to_owned();
let name = filename.to_bytes_with_nul();
if name == b".\0" || name == b"..\0" {
continue;
}
+ // Do not store a ".pxarexclude-cli" file found in the archive root,
+ // as this would confilict with new cli passed exclude patterns,
+ // if present.
+ if is_root && name == b".pxarexclude-cli\0" {
+ eprintln!("skip existing '.pxarexclude-cli' in archive root.");
+ continue;
+ }
- let stat = match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
+ let stat = match nix::sys::stat::fstatat(
+ rawfd,
+ filename.as_ref(),
+ nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW,
+ ) {
Ok(stat) => stat,
Err(nix::Error::Sys(Errno::ENOENT)) => {
let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
self.report_vanished_file(&self.full_path().join(filename_osstr))?;
continue;
- },
+ }
Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
};
- match match_filename(&filename, &stat, &local_match_pattern)? {
+ match MatchPatternSlice::match_filename_exclude(
+ &filename,
+ is_directory(&stat),
+ &local_match_pattern,
+ )? {
(MatchType::Positive, _) => {
let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
- eprintln!("matched by .pxarexclude entry - skipping: {:?}", self.full_path().join(filename_osstr));
- },
- (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
+ eprintln!(
+ "matched by exclude pattern - skipping: {:?}",
+ self.full_path().join(filename_osstr)
+ );
+ }
+ (_, child_pattern) => {
+ self.entry_counter += 1;
+ name_list.push((filename, stat, child_pattern));
+ }
}
- if name_list.len() > MAX_DIRECTORY_ENTRIES {
- bail!("too many directory items in {:?} (> {})", self.full_path(), MAX_DIRECTORY_ENTRIES);
+ if self.entry_counter > self.entry_max {
+ bail!(
+ "exceeded max number of entries (> {})",
+ self.entry_max
+ );
}
}
} else {
}
name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
+ let num_entries = name_list.len();
- let mut goodbye_items = vec![];
+ let mut goodbye_items = Vec::with_capacity(num_entries);
for (filename, stat, exclude_list) in name_list {
let start_pos = self.writer_pos;
if filename.as_bytes() == b".pxarexclude" {
+ // pxar_exclude is none in case of error EACCES.
if let Some((ref content, ref stat)) = pxar_exclude {
- let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
+ let filefd = match nix::fcntl::openat(
+ rawfd,
+ filename.as_ref(),
+ OFlag::O_NOFOLLOW,
+ Mode::empty(),
+ ) {
Ok(filefd) => filefd,
Err(nix::Error::Sys(Errno::ENOENT)) => {
self.report_vanished_file(&self.full_path())?;
continue;
- },
+ }
+ Err(nix::Error::Sys(Errno::EACCES)) => {
+ let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
+ eprintln!(
+ "skipping {:?}: open file failed - EACCES",
+ self.full_path().join(filename_osstr),
+ );
+ continue;
+ }
Err(err) => {
let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
- bail!("open file {:?} failed - {}", self.full_path().join(filename_osstr), err);
- },
+ bail!(
+ "open file {:?} failed - {}",
+ self.full_path().join(filename_osstr),
+ err
+ );
+ }
};
let child_magic = if dir_stat.st_dev != stat.st_dev {
};
self.write_filename(&filename)?;
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
+ }
self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
- continue;
}
+ continue;
+ }
+
+ if is_root && filename.as_bytes() == b".pxarexclude-cli" {
+ // '.pxarexclude-cli' is used to store the exclude MatchPatterns
+ // passed via the cli in the root directory of the archive.
+ self.write_filename(&filename)?;
+ let content = MatchPatternSlice::to_bytes(&exclude_list);
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_file(&filename, content.len() as u64, 0)?;
+ }
+ self.encode_pxar_exclude_cli(stat.st_uid, stat.st_gid, 0, &content)?;
+ continue;
}
- self.relative_path.push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
+ self.relative_path
+ .push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
- if self.verbose { println!("{:?}", self.full_path()); }
+ if self.verbose {
+ println!("{:?}", self.full_path());
+ }
if is_directory(&stat) {
-
- let mut dir = match nix::dir::Dir::openat(rawfd, filename.as_ref(), OFlag::O_DIRECTORY|OFlag::O_NOFOLLOW, Mode::empty()) {
+ let mut dir = match nix::dir::Dir::openat(
+ rawfd,
+ filename.as_ref(),
+ OFlag::O_DIRECTORY | OFlag::O_NOFOLLOW,
+ Mode::empty(),
+ ) {
Ok(dir) => dir,
Err(nix::Error::Sys(Errno::ENOENT)) => {
self.report_vanished_file(&self.full_path())?;
- continue; // fixme!!
- },
+ self.relative_path.pop();
+ continue;
+ }
+ Err(nix::Error::Sys(Errno::EACCES)) => {
+ eprintln!(
+ "skipping {:?}: open dir failed - EACCES",
+ self.full_path(),
+ );
+ self.relative_path.pop();
+ continue;
+ }
Err(err) => bail!("open dir {:?} failed - {}", self.full_path(), err),
};
};
self.write_filename(&filename)?;
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.start_directory(&filename)?;
+ }
self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
-
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.end_directory()?;
+ }
} else if is_reg_file(&stat) {
-
let mut hardlink_target = None;
if stat.st_nlink > 1 {
- let link_info = HardLinkInfo { st_dev: stat.st_dev, st_ino: stat.st_ino };
+ let link_info = HardLinkInfo {
+ st_dev: stat.st_dev,
+ st_ino: stat.st_ino,
+ };
hardlink_target = self.hardlinks.get(&link_info).map(|(v, offset)| {
let mut target = v.clone().into_os_string();
target.push("\0"); // add Nul byte
(target, (start_pos as u64) - offset)
});
if hardlink_target == None {
- self.hardlinks.insert(link_info, (self.relative_path.clone(), start_pos as u64));
+ self.hardlinks
+ .insert(link_info, (self.relative_path.clone(), start_pos as u64));
}
}
if let Some((target, offset)) = hardlink_target {
-
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_hardlink(&filename)?;
+ }
self.write_filename(&filename)?;
self.encode_hardlink(target.as_bytes(), offset)?;
-
} else {
-
- let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
+ let filefd = match nix::fcntl::openat(
+ rawfd,
+ filename.as_ref(),
+ OFlag::O_NOFOLLOW,
+ Mode::empty(),
+ ) {
Ok(filefd) => filefd,
Err(nix::Error::Sys(Errno::ENOENT)) => {
self.report_vanished_file(&self.full_path())?;
+ self.relative_path.pop();
continue;
- },
+ }
+ Err(nix::Error::Sys(Errno::EACCES)) => {
+ eprintln!(
+ "skipping {:?}: open file failed - EACCES",
+ self.full_path(),
+ );
+ self.relative_path.pop();
+ continue;
+ }
Err(err) => bail!("open file {:?} failed - {}", self.full_path(), err),
};
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_file(&filename, stat.st_size as u64, stat.st_mtime as u64)?;
+ }
let child_magic = if dir_stat.st_dev != stat.st_dev {
detect_fs_type(filefd)?
} else {
let _ = nix::unistd::close(filefd); // ignore close errors
res?;
}
-
} else if is_symlink(&stat) {
let mut buffer = vec::undefined(libc::PATH_MAX as usize);
- let res = filename.with_nix_path(|cstr| {
- unsafe { libc::readlinkat(rawfd, cstr.as_ptr(), buffer.as_mut_ptr() as *mut libc::c_char, buffer.len()-1) }
+ let res = filename.with_nix_path(|cstr| unsafe {
+ libc::readlinkat(
+ rawfd,
+ cstr.as_ptr(),
+ buffer.as_mut_ptr() as *mut libc::c_char,
+ buffer.len() - 1,
+ )
})?;
match Errno::result(res) {
Ok(len) => {
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_symlink(&filename)?;
+ }
buffer[len as usize] = 0u8; // add Nul byte
self.write_filename(&filename)?;
- self.encode_symlink(&buffer[..((len+1) as usize)], &stat)?
+ self.encode_symlink(&buffer[..((len + 1) as usize)], &stat)?
}
Err(nix::Error::Sys(Errno::ENOENT)) => {
self.report_vanished_file(&self.full_path())?;
+ self.relative_path.pop();
continue;
}
Err(err) => bail!("readlink {:?} failed - {}", self.full_path(), err),
}
} else if is_block_dev(&stat) || is_char_dev(&stat) {
if self.has_features(flags::WITH_DEVICE_NODES) {
+ if let Some(ref mut catalog) = self.catalog {
+ if is_block_dev(&stat) {
+ catalog.add_block_device(&filename)?;
+ } else {
+ catalog.add_char_device(&filename)?;
+ }
+ }
self.write_filename(&filename)?;
self.encode_device(&stat)?;
} else {
eprintln!("skip device node: {:?}", self.full_path());
+ self.relative_path.pop();
+ continue;
}
} else if is_fifo(&stat) {
if self.has_features(flags::WITH_FIFOS) {
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_fifo(&filename)?;
+ }
self.write_filename(&filename)?;
self.encode_special(&stat)?;
} else {
eprintln!("skip fifo: {:?}", self.full_path());
+ self.relative_path.pop();
+ continue;
}
} else if is_socket(&stat) {
if self.has_features(flags::WITH_SOCKETS) {
+ if let Some(ref mut catalog) = self.catalog {
+ catalog.add_socket(&filename)?;
+ }
self.write_filename(&filename)?;
self.encode_special(&stat)?;
} else {
eprintln!("skip socket: {:?}", self.full_path());
+ self.relative_path.pop();
+ continue;
}
} else {
- bail!("unsupported file type (mode {:o} {:?})", stat.st_mode, self.full_path());
+ bail!(
+ "unsupported file type (mode {:o} {:?})",
+ stat.st_mode,
+ self.full_path()
+ );
}
let end_pos = self.writer_pos;
let goodbye_offset = self.writer_pos - dir_start_pos;
self.write_goodbye_table(goodbye_offset, &mut goodbye_items)?;
+ self.entry_counter -= num_entries;
//println!("encode_dir: {:?} end1 {}", self.full_path(), self.writer_pos);
Ok(())
}
- fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64) -> Result<(), Error> {
-
+ fn encode_file(&mut self, filefd: RawFd, stat: &FileStat, magic: i64) -> Result<(), Error> {
//println!("encode_file: {:?}", self.full_path());
let mut entry = self.create_entry(&stat)?;
let include_payload;
if is_virtual_file_system(magic) {
include_payload = false;
+ } else if let Some(ref set) = &self.device_set {
+ include_payload = set.contains(&stat.st_dev);
} else {
- if let Some(ref set) = &self.device_set {
- include_payload = set.contains(&stat.st_dev);
- } else {
- include_payload = true;
- }
+ include_payload = true;
}
if !include_payload {
loop {
let n = match nix::unistd::read(filefd, &mut self.file_copy_buffer) {
Ok(n) => n,
- Err(nix::Error::Sys(Errno::EINTR)) => continue /* try again */,
- Err(err) => bail!("read {:?} failed - {}", self.full_path(), err),
+ Err(nix::Error::Sys(Errno::EINTR)) => continue, /* try again */
+ Err(err) => bail!("read {:?} failed - {}", self.full_path(), err),
};
if n == 0 { // EOF
if pos != size {
// Note:: casync format cannot handle that
- bail!("detected shrinked file {:?} ({} < {})", self.full_path(), pos, size);
+ bail!(
+ "detected shrunk file {:?} ({} < {})",
+ self.full_path(),
+ pos,
+ size
+ );
}
break;
}
let mut next = pos + (n as u64);
- if next > size { next = size; }
+ if next > size {
+ next = size;
+ }
let count = (next - pos) as usize;
pos = next;
- if pos >= size { break; }
+ if pos >= size {
+ break;
+ }
}
Ok(())
}
- fn encode_device(&mut self, stat: &FileStat) -> Result<(), Error> {
-
+ fn encode_device(&mut self, stat: &FileStat) -> Result<(), Error> {
let entry = self.create_entry(&stat)?;
self.write_entry(entry)?;
}
// FIFO or Socket
- fn encode_special(&mut self, stat: &FileStat) -> Result<(), Error> {
-
+ fn encode_special(&mut self, stat: &FileStat) -> Result<(), Error> {
let entry = self.create_entry(&stat)?;
self.write_entry(entry)?;
Ok(())
}
- fn encode_symlink(&mut self, target: &[u8], stat: &FileStat) -> Result<(), Error> {
-
+ fn encode_symlink(&mut self, target: &[u8], stat: &FileStat) -> Result<(), Error> {
//println!("encode_symlink: {:?} -> {:?}", self.full_path(), target);
let entry = self.create_entry(&stat)?;
Ok(())
}
- fn encode_hardlink(&mut self, target: &[u8], offset: u64) -> Result<(), Error> {
-
+ fn encode_hardlink(&mut self, target: &[u8], offset: u64) -> Result<(), Error> {
//println!("encode_hardlink: {:?} -> {:?}", self.full_path(), target);
// Note: HARDLINK replaces an ENTRY.
Ok(())
}
- fn encode_pxar_exclude(&mut self, filefd: RawFd, stat: &FileStat, magic: i64, content: &[u8]) -> Result<(), Error> {
+ fn encode_pxar_exclude(
+ &mut self,
+ filefd: RawFd,
+ stat: &FileStat,
+ magic: i64,
+ content: &[u8],
+ ) -> Result<(), Error> {
let mut entry = self.create_entry(&stat)?;
self.read_chattr(filefd, &mut entry)?;
let include_payload;
if is_virtual_file_system(magic) {
include_payload = false;
+ } else if let Some(set) = &self.device_set {
+ include_payload = set.contains(&stat.st_dev);
} else {
- if let Some(set) = &self.device_set {
- include_payload = set.contains(&stat.st_dev);
- } else {
- include_payload = true;
- }
+ include_payload = true;
}
if !include_payload {
Ok(())
}
+ /// Encodes the excude match patterns passed via cli as file in the archive.
+ fn encode_pxar_exclude_cli(
+ &mut self,
+ uid: u32,
+ gid: u32,
+ mtime: u64,
+ content: &[u8],
+ ) -> Result<(), Error> {
+ let entry = PxarEntry {
+ mode: (libc::S_IFREG | 0o600) as u64,
+ flags: 0,
+ uid,
+ gid,
+ mtime,
+ };
+ self.write_entry(entry)?;
+ let size = content.len();
+ self.write_header(PXAR_PAYLOAD, size as u64)?;
+ self.writer.write_all(content)?;
+ self.writer_pos += size;
+
+ Ok(())
+ }
+
// the report_XXX method may raise and error - depending on encoder configuration
fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
-
eprintln!("WARNING: detected vanished file {:?}", path);
Ok(())
}
}
-// If there is a match, an updated MatchPattern list to pass to the matched child is returned.
-fn match_filename(
- filename: &CStr,
- stat: &FileStat,
- match_pattern: &Vec<MatchPattern>
-) -> Result<(MatchType, Vec<MatchPattern>), Error> {
- let mut child_pattern = Vec::new();
- let mut match_state = MatchType::None;
-
- for pattern in match_pattern {
- match pattern.matches_filename(filename, is_directory(&stat))? {
- MatchType::None => {},
- MatchType::Positive => match_state = MatchType::Positive,
- MatchType::Negative => match_state = MatchType::Negative,
- match_type => {
- if match_state != MatchType::Positive && match_state != MatchType::Negative {
- match_state = match_type;
- }
- child_pattern.push(pattern.get_rest_pattern());
- },
- }
- }
-
- Ok((match_state, child_pattern))
-}
-
fn errno_is_unsupported(errno: Errno) -> bool {
-
match errno {
- Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => {
- true
- }
+ Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => true,
_ => false,
}
}
fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
- let mut fs_stat: libc::statfs = unsafe { std::mem::uninitialized() };
- let res = unsafe { libc::fstatfs(fd, &mut fs_stat) };
+ let mut fs_stat = std::mem::MaybeUninit::uninit();
+ let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
Errno::result(res)?;
+ let fs_stat = unsafe { fs_stat.assume_init() };
Ok(fs_stat.f_type)
}
projects / proxmox-backup.git / blobdiff