use failure::*;
use endian_trait::Endian;
-use std::collections::HashMap;
+use std::collections::{HashSet, HashMap};
use super::format_definition::*;
use super::binary_search_tree::*;
use super::helper::*;
+use super::exclude_pattern::*;
use crate::tools::fs;
use crate::tools::acl;
use crate::tools::xattr;
use nix::errno::Errno;
use nix::sys::stat::FileStat;
-use crate::tools::vec;
+use proxmox::tools::vec;
/// The format requires to build sorted directory lookup tables in
/// memory, so we restrict the number of allowed entries to limit
writer_pos: usize,
_size: usize,
file_copy_buffer: Vec<u8>,
- all_file_systems: bool,
- root_st_dev: u64,
+ device_set: Option<HashSet<u64>>,
verbose: bool,
+ // Flags set by the user
feature_flags: u64,
+ // Flags signaling features supported by the filesystem
+ fs_feature_flags: u64,
hardlinks: HashMap<HardLinkInfo, (PathBuf, u64)>,
}
self.base_path.join(&self.relative_path)
}
+ /// Create archive, write result data to ``writer``.
+ ///
+ /// The ``device_set`` can be use used to limit included mount points.
+ ///
+ /// - ``None``: include all mount points
+ /// - ``Some(set)``: only include devices listed in this set (the
+ /// root path device is automathically added to this list, so
+ /// you can pass an empty set if you want to archive a single
+ /// mount point.)
pub fn encode(
path: PathBuf,
dir: &mut nix::dir::Dir,
writer: &'a mut W,
- all_file_systems: bool,
+ device_set: Option<HashSet<u64>>,
verbose: bool,
feature_flags: u64,
) -> Result<(), Error> {
// todo: use scandirat??
let dir_fd = dir.as_raw_fd();
- let stat = match nix::sys::stat::fstat(dir_fd) {
- Ok(stat) => stat,
- Err(err) => bail!("fstat {:?} failed - {}", path, err),
- };
+ let stat = nix::sys::stat::fstat(dir_fd)
+ .map_err(|err| format_err!("fstat {:?} failed - {}", path, err))?;
if !is_directory(&stat) {
bail!("got unexpected file type {:?} (not a directory)", path);
}
+ let mut device_set = device_set.clone();
+ if let Some(ref mut set) = device_set {
+ set.insert(stat.st_dev);
+ }
+
let magic = detect_fs_type(dir_fd)?;
if is_virtual_file_system(magic) {
bail!("backup virtual file systems is disabled!");
}
+ let fs_feature_flags = feature_flags_from_magic(magic);
+
let mut me = Self {
base_path: path,
relative_path: PathBuf::new(),
writer_pos: 0,
_size: 0,
file_copy_buffer,
- all_file_systems,
- root_st_dev: stat.st_dev,
+ device_set,
verbose,
feature_flags,
+ fs_feature_flags,
hardlinks: HashMap::new(),
};
if verbose { println!("{:?}", me.full_path()); }
- me.encode_dir(dir, &stat, magic)?;
+ me.encode_dir(dir, &stat, magic, Vec::new())?;
Ok(())
}
/// True if all of the given feature flags are set in the Encoder, false otherwise
fn has_features(&self, feature_flags: u64) -> bool {
- (self.feature_flags & feature_flags) == feature_flags
+ (self.feature_flags & self.fs_feature_flags & feature_flags) == feature_flags
}
/// True if at least one of the given feature flags is set in the Encoder, false otherwise
fn has_some_features(&self, feature_flags: u64) -> bool {
- (self.feature_flags & feature_flags) != 0
+ (self.feature_flags & self.fs_feature_flags & feature_flags) != 0
}
fn read_xattrs(&self, fd: RawFd, stat: &FileStat) -> Result<(Vec<CaFormatXAttr>, Option<CaFormatFCaps>), Error> {
Ok(())
}
- fn encode_dir(&mut self, dir: &mut nix::dir::Dir, dir_stat: &FileStat, magic: i64) -> Result<(), Error> {
+ fn encode_dir(&mut self, dir: &mut nix::dir::Dir, dir_stat: &FileStat, magic: i64, match_pattern: Vec<PxarExcludePattern>) -> Result<(), Error> {
//println!("encode_dir: {:?} start {}", self.full_path(), self.writer_pos);
self.read_chattr(rawfd, &mut dir_entry)?;
self.read_fat_attr(rawfd, magic, &mut dir_entry)?;
+
+ // for each node in the directory tree, the filesystem features are
+ // checked based on the fs magic number.
+ self.fs_feature_flags = feature_flags_from_magic(magic);
+
let (xattrs, fcaps) = self.read_xattrs(rawfd, &dir_stat)?;
let acl_access = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_ACCESS)?;
let acl_default = self.read_acl(rawfd, &dir_stat, acl::ACL_TYPE_DEFAULT)?;
self.write_quota_project_id(projid)?;
}
- let mut dir_count = 0;
-
let include_children;
if is_virtual_file_system(magic) {
include_children = false;
} else {
- include_children = (self.root_st_dev == dir_stat.st_dev) || self.all_file_systems;
+ if let Some(set) = &self.device_set {
+ include_children = set.contains(&dir_stat.st_dev);
+ } else {
+ include_children = true;
+ }
}
+ // Expand the exclude match pattern inherited from the parent by local entries, if present
+ let mut local_match_pattern = match_pattern.clone();
+ let pxar_exclude = match PxarExcludePattern::from_file(rawfd, ".pxarexclude") {
+ Ok(Some((mut excludes, buffer, stat))) => {
+ local_match_pattern.append(&mut excludes);
+ Some((buffer, stat))
+ },
+ Ok(None) => None,
+ Err(err) => bail!("error while reading exclude file - {}", err),
+ };
+
if include_children {
for entry in dir.iter() {
- dir_count += 1;
- if dir_count > MAX_DIRECTORY_ENTRIES {
- bail!("too many directory items in {:?} (> {})",
- self.full_path(), MAX_DIRECTORY_ENTRIES);
- }
-
let entry = entry.map_err(|err| {
format_err!("readir {:?} failed - {}", self.full_path(), err)
})?;
continue;
}
- name_list.push(filename);
+ let stat = match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
+ Ok(stat) => stat,
+ Err(nix::Error::Sys(Errno::ENOENT)) => {
+ let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
+ self.report_vanished_file(&self.full_path().join(filename_osstr))?;
+ continue;
+ },
+ Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
+ };
+
+ match match_exclude_pattern(&filename, &stat, &local_match_pattern) {
+ (MatchType::Exclude, _) => {
+ let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
+ eprintln!("matched by .pxarexclude entry - skipping: {:?}", self.full_path().join(filename_osstr));
+ },
+ (_, child_pattern) => name_list.push((filename, stat, child_pattern)),
+ }
+
+ if name_list.len() > MAX_DIRECTORY_ENTRIES {
+ bail!("too many directory items in {:?} (> {})", self.full_path(), MAX_DIRECTORY_ENTRIES);
+ }
}
} else {
eprintln!("skip mount point: {:?}", self.full_path());
}
- name_list.sort_unstable_by(|a, b| a.cmp(&b));
+ name_list.sort_unstable_by(|a, b| a.0.cmp(&b.0));
let mut goodbye_items = vec![];
- for filename in &name_list {
- self.relative_path.push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
+ for (filename, stat, exclude_list) in name_list {
+ let start_pos = self.writer_pos;
- if self.verbose { println!("{:?}", self.full_path()); }
+ if filename.as_bytes() == b".pxarexclude" {
+ if let Some((ref content, ref stat)) = pxar_exclude {
+ let filefd = match nix::fcntl::openat(rawfd, filename.as_ref(), OFlag::O_NOFOLLOW, Mode::empty()) {
+ Ok(filefd) => filefd,
+ Err(nix::Error::Sys(Errno::ENOENT)) => {
+ self.report_vanished_file(&self.full_path())?;
+ continue;
+ },
+ Err(err) => {
+ let filename_osstr = std::ffi::OsStr::from_bytes(filename.to_bytes());
+ bail!("open file {:?} failed - {}", self.full_path().join(filename_osstr), err);
+ },
+ };
+
+ let child_magic = if dir_stat.st_dev != stat.st_dev {
+ detect_fs_type(filefd)?
+ } else {
+ magic
+ };
- let stat = match nix::sys::stat::fstatat(rawfd, filename.as_ref(), nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
- Ok(stat) => stat,
- Err(nix::Error::Sys(Errno::ENOENT)) => {
- self.report_vanished_file(&self.full_path())?;
+ self.write_filename(&filename)?;
+ self.encode_pxar_exclude(filefd, stat, child_magic, content)?;
continue;
}
- Err(err) => bail!("fstat {:?} failed - {}", self.full_path(), err),
- };
+ }
- let start_pos = self.writer_pos;
+ self.relative_path.push(std::ffi::OsStr::from_bytes(filename.as_bytes()));
+
+ if self.verbose { println!("{:?}", self.full_path()); }
if is_directory(&stat) {
};
self.write_filename(&filename)?;
- self.encode_dir(&mut dir, &stat, child_magic)?;
+ self.encode_dir(&mut dir, &stat, child_magic, exclude_list)?;
} else if is_reg_file(&stat) {
if is_virtual_file_system(magic) {
include_payload = false;
} else {
- include_payload = (stat.st_dev == self.root_st_dev) || self.all_file_systems;
+ if let Some(ref set) = &self.device_set {
+ include_payload = set.contains(&stat.st_dev);
+ } else {
+ include_payload = true;
+ }
}
if !include_payload {
Ok(())
}
+ fn encode_pxar_exclude(&mut self, filefd: RawFd, stat: &FileStat, magic: i64, content: &[u8]) -> Result<(), Error> {
+ let mut entry = self.create_entry(&stat)?;
+
+ self.read_chattr(filefd, &mut entry)?;
+ self.read_fat_attr(filefd, magic, &mut entry)?;
+ let (xattrs, fcaps) = self.read_xattrs(filefd, &stat)?;
+ let acl_access = self.read_acl(filefd, &stat, acl::ACL_TYPE_ACCESS)?;
+ let projid = self.read_quota_project_id(filefd, magic, &stat)?;
+
+ self.write_entry(entry)?;
+ for xattr in xattrs {
+ self.write_xattr(xattr)?;
+ }
+ self.write_fcaps(fcaps)?;
+ for user in acl_access.users {
+ self.write_acl_user(user)?;
+ }
+ for group in acl_access.groups {
+ self.write_acl_group(group)?;
+ }
+ if let Some(group_obj) = acl_access.group_obj {
+ self.write_acl_group_obj(group_obj)?;
+ }
+ if let Some(projid) = projid {
+ self.write_quota_project_id(projid)?;
+ }
+
+ let include_payload;
+ if is_virtual_file_system(magic) {
+ include_payload = false;
+ } else {
+ if let Some(set) = &self.device_set {
+ include_payload = set.contains(&stat.st_dev);
+ } else {
+ include_payload = true;
+ }
+ }
+
+ if !include_payload {
+ eprintln!("skip content: {:?}", self.full_path());
+ self.write_header(CA_FORMAT_PAYLOAD, 0)?;
+ return Ok(());
+ }
+
+ let size = content.len();
+ self.write_header(CA_FORMAT_PAYLOAD, size as u64)?;
+ self.writer.write_all(content)?;
+ self.writer_pos += size;
+
+ Ok(())
+ }
+
// the report_XXX method may raise and error - depending on encoder configuration
fn report_vanished_file(&self, path: &Path) -> Result<(), Error> {
}
}
+// If there is a match, an updated PxarExcludePattern list to pass to the matched child is returned.
+fn match_exclude_pattern(
+ filename: &CStr,
+ stat: &FileStat,
+ match_pattern: &Vec<PxarExcludePattern>
+) -> (MatchType, Vec<PxarExcludePattern>) {
+ let mut child_pattern = Vec::new();
+ let mut match_state = MatchType::None;
+
+ for pattern in match_pattern {
+ match pattern.matches_filename(filename, is_directory(&stat)) {
+ MatchType::None => {},
+ MatchType::Exclude => match_state = MatchType::Exclude,
+ MatchType::Include => match_state = MatchType::Include,
+ MatchType::PartialExclude => {
+ if match_state != MatchType::Exclude && match_state != MatchType::Include {
+ match_state = MatchType::PartialExclude;
+ }
+ child_pattern.push(pattern.get_rest_pattern());
+ },
+ MatchType::PartialInclude => {
+ if match_state != MatchType::Exclude && match_state != MatchType::Include {
+ match_state = MatchType::PartialInclude;
+ }
+ child_pattern.push(pattern.get_rest_pattern());
+ },
+ }
+ }
+
+ (match_state, child_pattern)
+}
+
fn errno_is_unsupported(errno: Errno) -> bool {
match errno {
Ok(fs_stat.f_type)
}
-
-// from /usr/include/linux/magic.h
-// and from casync util.h
-pub const BINFMTFS_MAGIC: i64 = 0x42494e4d;
-pub const CGROUP2_SUPER_MAGIC: i64 = 0x63677270;
-pub const CGROUP_SUPER_MAGIC: i64 = 0x0027e0eb;
-pub const CONFIGFS_MAGIC: i64 = 0x62656570;
-pub const DEBUGFS_MAGIC: i64 = 0x64626720;
-pub const DEVPTS_SUPER_MAGIC: i64 = 0x00001cd1;
-pub const EFIVARFS_MAGIC: i64 = 0xde5e81e4;
-pub const FUSE_CTL_SUPER_MAGIC: i64 = 0x65735543;
-pub const HUGETLBFS_MAGIC: i64 = 0x958458f6;
-pub const MQUEUE_MAGIC: i64 = 0x19800202;
-pub const NFSD_MAGIC: i64 = 0x6e667364;
-pub const PROC_SUPER_MAGIC: i64 = 0x00009fa0;
-pub const PSTOREFS_MAGIC: i64 = 0x6165676C;
-pub const RPCAUTH_GSSMAGIC: i64 = 0x67596969;
-pub const SECURITYFS_MAGIC: i64 = 0x73636673;
-pub const SELINUX_MAGIC: i64 = 0xf97cff8c;
-pub const SMACK_MAGIC: i64 = 0x43415d53;
-pub const RAMFS_MAGIC: i64 = 0x858458f6;
-pub const TMPFS_MAGIC: i64 = 0x01021994;
-pub const SYSFS_MAGIC: i64 = 0x62656572;
-pub const MSDOS_SUPER_MAGIC: i64 = 0x00004d44;
-pub const FUSE_SUPER_MAGIC: i64 = 0x65735546;
-pub const EXT4_SUPER_MAGIC: i64 = 0x0000EF53;
-pub const XFS_SUPER_MAGIC: i64 = 0x58465342;
-
-
#[inline(always)]
pub fn is_temporary_file_system(magic: i64) -> bool {
magic == RAMFS_MAGIC || magic == TMPFS_MAGIC
projects / proxmox-backup.git / blobdiff