use std::collections::{HashSet, HashMap};
-use std::convert::TryFrom;
use std::ffi::{CStr, CString, OsStr};
use std::fmt;
use std::io::{self, Read, Write};
use nix::fcntl::OFlag;
use nix::sys::stat::{FileStat, Mode};
-use pathpatterns::{MatchEntry, MatchList, MatchType, PatternFlag};
+use pathpatterns::{MatchEntry, MatchFlag, MatchList, MatchType, PatternFlag};
use pxar::Metadata;
use pxar::encoder::LinkOffset;
use proxmox::tools::vec;
use crate::pxar::catalog::BackupCatalogWriter;
+use crate::pxar::metadata::errno_is_unsupported;
use crate::pxar::Flags;
use crate::pxar::tools::assert_single_path_component;
use crate::tools::{acl, fs, xattr, Fd};
+/// Pxar options for creating a pxar archive/stream
+#[derive(Default, Clone)]
+pub struct PxarCreateOptions {
+ /// Device/mountpoint st_dev numbers that should be included. None for no limitation.
+ pub device_set: Option<HashSet<u64>>,
+ /// Exclusion patterns
+ pub patterns: Vec<MatchEntry>,
+ /// Maximum number of entries to hold in memory
+ pub entries_max: usize,
+ /// Skip lost+found directory
+ pub skip_lost_and_found: bool,
+ /// Verbose output
+ pub verbose: bool,
+}
+
+
fn detect_fs_type(fd: RawFd) -> Result<i64, Error> {
let mut fs_stat = std::mem::MaybeUninit::uninit();
let res = unsafe { libc::fstatfs(fd, fs_stat.as_mut_ptr()) };
pub fn is_virtual_file_system(magic: i64) -> bool {
use proxmox::sys::linux::magic::*;
- match magic {
- BINFMTFS_MAGIC |
+ matches!(magic, BINFMTFS_MAGIC |
CGROUP2_SUPER_MAGIC |
CGROUP_SUPER_MAGIC |
CONFIGFS_MAGIC |
SECURITYFS_MAGIC |
SELINUX_MAGIC |
SMACK_MAGIC |
- SYSFS_MAGIC => true,
- _ => false
- }
+ SYSFS_MAGIC)
}
#[derive(Debug)]
st_ino: u64,
}
-/// In case we want to collect them or redirect them we can just add this here:
+/// TODO: make a builder for the create_archive call for fewer parameters and add a method to add a
+/// logger which does not write to stderr.
+struct Logger;
+
+impl std::io::Write for Logger {
+ fn write(&mut self, data: &[u8]) -> io::Result<usize> {
+ std::io::stderr().write(data)
+ }
+
+ fn flush(&mut self) -> io::Result<()> {
+ std::io::stderr().flush()
+ }
+}
+
+/// And the error case.
struct ErrorReporter;
impl std::io::Write for ErrorReporter {
device_set: Option<HashSet<u64>>,
hardlinks: HashMap<HardLinkInfo, (PathBuf, LinkOffset)>,
errors: ErrorReporter,
+ logger: Logger,
file_copy_buffer: Vec<u8>,
}
pub fn create_archive<T, F>(
source_dir: Dir,
mut writer: T,
- mut patterns: Vec<MatchEntry>,
feature_flags: Flags,
- mut device_set: Option<HashSet<u64>>,
- skip_lost_and_found: bool,
mut callback: F,
- entry_limit: usize,
catalog: Option<&mut dyn BackupCatalogWriter>,
+ options: PxarCreateOptions,
) -> Result<(), Error>
where
T: pxar::encoder::SeqWrite,
)
.map_err(|err| format_err!("failed to get metadata for source directory: {}", err))?;
+ let mut device_set = options.device_set.clone();
if let Some(ref mut set) = device_set {
set.insert(stat.st_dev);
}
let writer = &mut writer as &mut dyn pxar::encoder::SeqWrite;
let mut encoder = Encoder::new(writer, &metadata)?;
- if skip_lost_and_found {
+ let mut patterns = options.patterns.clone();
+
+ if options.skip_lost_and_found {
patterns.push(MatchEntry::parse_pattern(
"lost+found",
PatternFlag::PATH_NAME,
catalog,
path: PathBuf::new(),
entry_counter: 0,
- entry_limit,
+ entry_limit: options.entries_max,
current_st_dev: stat.st_dev,
device_set,
hardlinks: HashMap::new(),
errors: ErrorReporter,
+ logger: Logger,
file_copy_buffer: vec::undefined(4 * 1024 * 1024),
};
let old_patterns_count = self.patterns.len();
self.read_pxar_excludes(dir.as_raw_fd())?;
- let file_list = self.generate_directory_file_list(&mut dir, is_root)?;
+ let mut file_list = self.generate_directory_file_list(&mut dir, is_root)?;
+
+ if is_root && old_patterns_count > 0 {
+ file_list.push(FileListEntry {
+ name: CString::new(".pxarexclude-cli").unwrap(),
+ path: PathBuf::new(),
+ stat: unsafe { std::mem::zeroed() },
+ });
+ }
let dir_fd = dir.as_raw_fd();
let file_name = file_entry.name.to_bytes();
if is_root && file_name == b".pxarexclude-cli" {
- self.encode_pxarexclude_cli(encoder, &file_entry.name)?;
+ self.encode_pxarexclude_cli(encoder, &file_entry.name, old_patterns_count)?;
continue;
}
- (self.callback)(Path::new(OsStr::from_bytes(file_name)))?;
+ (self.callback)(&file_entry.path)?;
self.path = file_entry.path;
self.add_entry(encoder, dir_fd, &file_entry.name, &file_entry.stat)
.map_err(|err| self.wrap_err(err))?;
oflags: OFlag,
existed: bool,
) -> Result<Option<Fd>, Error> {
- match Fd::openat(
- &unsafe { RawFdNum::from_raw_fd(parent) },
- file_name,
- oflags,
- Mode::empty(),
- ) {
- Ok(fd) => Ok(Some(fd)),
- Err(nix::Error::Sys(Errno::ENOENT)) => {
- if existed {
- self.report_vanished_file()?;
+ // common flags we always want to use:
+ let oflags = oflags | OFlag::O_CLOEXEC | OFlag::O_NOCTTY;
+
+ let mut noatime = OFlag::O_NOATIME;
+ loop {
+ return match Fd::openat(
+ &unsafe { RawFdNum::from_raw_fd(parent) },
+ file_name,
+ oflags | noatime,
+ Mode::empty(),
+ ) {
+ Ok(fd) => Ok(Some(fd)),
+ Err(nix::Error::Sys(Errno::ENOENT)) => {
+ if existed {
+ self.report_vanished_file()?;
+ }
+ Ok(None)
}
- Ok(None)
- }
- Err(nix::Error::Sys(Errno::EACCES)) => {
- writeln!(self.errors, "failed to open file: {:?}: access denied", file_name)?;
- Ok(None)
+ Err(nix::Error::Sys(Errno::EACCES)) => {
+ writeln!(self.errors, "failed to open file: {:?}: access denied", file_name)?;
+ Ok(None)
+ }
+ Err(nix::Error::Sys(Errno::EPERM)) if !noatime.is_empty() => {
+ // Retry without O_NOATIME:
+ noatime = OFlag::empty();
+ continue;
+ }
+ Err(other) => Err(Error::from(other)),
}
- Err(other) => Err(Error::from(other)),
}
}
fn read_pxar_excludes(&mut self, parent: RawFd) -> Result<(), Error> {
- let fd = self.open_file(
- parent,
- c_str!(".pxarexclude"),
- OFlag::O_RDONLY | OFlag::O_CLOEXEC | OFlag::O_NOCTTY,
- false,
- )?;
+ let fd = match self.open_file(parent, c_str!(".pxarexclude"), OFlag::O_RDONLY, false)? {
+ Some(fd) => fd,
+ None => return Ok(()),
+ };
let old_pattern_count = self.patterns.len();
- if let Some(fd) = fd {
- let file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) };
-
- use io::BufRead;
- for line in io::BufReader::new(file).lines() {
- let line = match line {
- Ok(line) => line,
- Err(err) => {
- let _ = writeln!(
- self.errors,
- "ignoring .pxarexclude after read error in {:?}: {}",
- self.path,
- err,
- );
- self.patterns.truncate(old_pattern_count);
- return Ok(());
- }
- };
+ let path_bytes = self.path.as_os_str().as_bytes();
+
+ let file = unsafe { std::fs::File::from_raw_fd(fd.into_raw_fd()) };
+
+ use io::BufRead;
+ for line in io::BufReader::new(file).split(b'\n') {
+ let line = match line {
+ Ok(line) => line,
+ Err(err) => {
+ let _ = writeln!(
+ self.errors,
+ "ignoring .pxarexclude after read error in {:?}: {}",
+ self.path,
+ err,
+ );
+ self.patterns.truncate(old_pattern_count);
+ return Ok(());
+ }
+ };
- let line = line.trim();
+ let line = crate::tools::strip_ascii_whitespace(&line);
- if line.is_empty() || line.starts_with('#') {
- continue;
- }
+ if line.is_empty() || line[0] == b'#' {
+ continue;
+ }
+
+ let mut buf;
+ let (line, mode, anchored) = if line[0] == b'/' {
+ buf = Vec::with_capacity(path_bytes.len() + 1 + line.len());
+ buf.extend(path_bytes);
+ buf.extend(line);
+ (&buf[..], MatchType::Exclude, true)
+ } else if line.starts_with(b"!/") {
+ // inverted case with absolute path
+ buf = Vec::with_capacity(path_bytes.len() + line.len());
+ buf.extend(path_bytes);
+ buf.extend(&line[1..]); // without the '!'
+ (&buf[..], MatchType::Include, true)
+ } else if line.starts_with(b"!") {
+ (&line[1..], MatchType::Include, false)
+ } else {
+ (line, MatchType::Exclude, false)
+ };
- match MatchEntry::parse_pattern(line, PatternFlag::PATH_NAME, MatchType::Exclude) {
- Ok(pattern) => self.patterns.push(pattern),
- Err(err) => {
- let _ = writeln!(self.errors, "bad pattern in {:?}: {}", self.path, err);
+ match MatchEntry::parse_pattern(line, PatternFlag::PATH_NAME, mode) {
+ Ok(pattern) => {
+ if anchored {
+ self.patterns.push(pattern.add_flags(MatchFlag::ANCHORED));
+ } else {
+ self.patterns.push(pattern);
}
}
+ Err(err) => {
+ let _ = writeln!(self.errors, "bad pattern in {:?}: {}", self.path, err);
+ }
}
}
&mut self,
encoder: &mut Encoder,
file_name: &CStr,
+ patterns_count: usize,
) -> Result<(), Error> {
- let content = generate_pxar_excludes_cli(&self.patterns);
+ let content = generate_pxar_excludes_cli(&self.patterns[..patterns_count]);
if let Some(ref mut catalog) = self.catalog {
catalog.add_file(file_name, content.len() as u64, 0)?;
let mut file_list = Vec::new();
- if is_root && !self.patterns.is_empty() {
- file_list.push(FileListEntry {
- name: CString::new(".pxarexclude-cli").unwrap(),
- path: PathBuf::new(),
- stat: unsafe { std::mem::zeroed() },
- });
- }
-
for file in dir.iter() {
let file = file?;
continue;
}
- if file_name_bytes == b".pxarexclude" {
- continue;
- }
-
let os_file_name = OsStr::from_bytes(file_name_bytes);
assert_single_path_component(os_file_name)?;
let full_path = self.path.join(os_file_name);
Err(err) => bail!("stat failed on {:?}: {}", full_path, err),
};
+ let match_path = PathBuf::from("/").join(full_path.clone());
if self
.patterns
- .matches(full_path.as_os_str().as_bytes(), Some(stat.st_mode as u32))
+ .matches(match_path.as_os_str().as_bytes(), Some(stat.st_mode as u32))
== Some(MatchType::Exclude)
{
continue;
let fd = self.open_file(
parent,
c_file_name,
- open_mode | OFlag::O_RDONLY | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC | OFlag::O_NOCTTY,
+ open_mode | OFlag::O_RDONLY | OFlag::O_NOFOLLOW,
true,
)?;
let file_size = stat.st_size as u64;
if let Some(ref mut catalog) = self.catalog {
- catalog.add_file(c_file_name, file_size, stat.st_mtime as u64)?;
+ catalog.add_file(c_file_name, file_size, stat.st_mtime)?;
}
let offset: LinkOffset =
}
let result = if skip_contents {
+ writeln!(self.logger, "skipping mount point: {:?}", self.path)?;
Ok(())
} else {
self.archive_dir_contents(&mut encoder, dir, false)
let mut remaining = file_size;
let mut out = encoder.create_file(metadata, file_name, file_size)?;
while remaining != 0 {
- let mut got = file.read(&mut self.file_copy_buffer[..])?;
+ let mut got = match file.read(&mut self.file_copy_buffer[..]) {
+ Ok(0) => break,
+ Ok(got) => got,
+ Err(err) if err.kind() == std::io::ErrorKind::Interrupted => continue,
+ Err(err) => bail!(err),
+ };
if got as u64 > remaining {
self.report_file_grew_while_reading()?;
got = remaining as usize;
// required for some of these
let proc_path = Path::new("/proc/self/fd/").join(fd.to_string());
- let mtime = u64::try_from(stat.st_mtime * 1_000_000_000 + stat.st_mtime_nsec)
- .map_err(|_| format_err!("file with negative mtime"))?;
-
let mut meta = Metadata {
stat: pxar::Stat {
mode: u64::from(stat.st_mode),
flags: 0,
uid: stat.st_uid,
gid: stat.st_gid,
- mtime,
+ mtime: pxar::format::StatxTimestamp {
+ secs: stat.st_mtime,
+ nanos: stat.st_mtime_nsec as u32,
+ },
},
..Default::default()
};
Ok(meta)
}
-fn errno_is_unsupported(errno: Errno) -> bool {
- match errno {
- Errno::ENOTTY | Errno::ENOSYS | Errno::EBADF | Errno::EOPNOTSUPP | Errno::EINVAL => true,
- _ => false,
- }
-}
-
fn get_fcaps(meta: &mut Metadata, fd: RawFd, flags: Flags) -> Result<(), Error> {
if flags.contains(Flags::WITH_FCAPS) {
return Ok(());
}
fn get_chattr(metadata: &mut Metadata, fd: RawFd) -> Result<(), Error> {
- let mut attr: usize = 0;
+ let mut attr: libc::c_long = 0;
match unsafe { fs::read_attr_fd(fd, &mut attr) } {
Ok(_) => (),
Err(err) => bail!("failed to read file attributes: {}", err),
}
- metadata.stat.flags |= Flags::from_chattr(attr as u32).bits();
+ metadata.stat.flags |= Flags::from_chattr(attr).bits();
Ok(())
}
/// Since we are generating an *exclude* list, we need to invert this, so includes get a `'!'`
/// prefix.
fn generate_pxar_excludes_cli(patterns: &[MatchEntry]) -> Vec<u8> {
- use pathpatterns::{MatchFlag, MatchPattern};
+ use pathpatterns::MatchPattern;
let mut content = Vec::new();
projects / proxmox-backup.git / blobdiff