let default_options = CreateOptions::new();
- if let Err(err) = create_path(&base, Some(default_options.clone()), Some(options.clone())) {
- bail!("unable to create chunk store '{}' at {:?} - {}", name, base, err);
+ match create_path(&base, Some(default_options.clone()), Some(options.clone())) {
+ Err(err) => bail!("unable to create chunk store '{}' at {:?} - {}", name, base, err),
+ Ok(res) => if ! res { nix::unistd::chown(&base, Some(uid), Some(gid))? },
}
if let Err(err) = create_dir(&chunk_dir, options.clone()) {
}
let percentage = (i*100)/(64*1024);
if percentage != last_percentage {
- eprintln!("Percentage done: {}", percentage);
+ eprintln!("{}%", percentage);
last_percentage = percentage;
}
}
let (chunk_path, _digest_str) = self.chunk_path(digest);
- const UTIME_NOW: i64 = ((1 << 30) - 1);
- const UTIME_OMIT: i64 = ((1 << 30) - 2);
+ const UTIME_NOW: i64 = (1 << 30) - 1;
+ const UTIME_OMIT: i64 = (1 << 30) - 2;
let times: [libc::timespec; 2] = [
libc::timespec { tv_sec: 0, tv_nsec: UTIME_NOW },
return Ok(false);
}
- bail!("updata atime failed for chunk {:?} - {}", chunk_path, err);
+ bail!("update atime failed for chunk {:?} - {}", chunk_path, err);
}
Ok(true)
}
- pub fn read_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> {
-
- let (chunk_path, digest_str) = self.chunk_path(digest);
- let mut file = std::fs::File::open(&chunk_path)
- .map_err(|err| {
- format_err!(
- "store '{}', unable to read chunk '{}' - {}",
- self.name,
- digest_str,
- err,
- )
- })?;
-
- DataBlob::load(&mut file)
- }
-
pub fn get_chunk_iterator(
&self,
) -> Result<
- impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize)> + std::iter::FusedIterator,
+ impl Iterator<Item = (Result<tools::fs::ReadDirEntry, Error>, usize, bool)> + std::iter::FusedIterator,
Error
> {
use nix::dir::Dir;
Some(Ok(entry)) => {
// skip files if they're not a hash
let bytes = entry.file_name().to_bytes();
- if bytes.len() != 64 {
+ if bytes.len() != 64 && bytes.len() != 64 + ".0.bad".len() {
continue;
}
- if !bytes.iter().all(u8::is_ascii_hexdigit) {
+ if !bytes.iter().take(64).all(u8::is_ascii_hexdigit) {
continue;
}
- return Some((Ok(entry), percentage));
+
+ let bad = bytes.ends_with(".bad".as_bytes());
+ return Some((Ok(entry), percentage, bad));
}
Some(Err(err)) => {
// stop after first error
done = true;
// and pass the error through:
- return Some((Err(err), percentage));
+ return Some((Err(err), percentage, false));
}
None => (), // open next directory
}
// other errors are fatal, so end our iteration
done = true;
// and pass the error through:
- return Some((Err(format_err!("unable to read subdir '{}' - {}", subdir, err)), percentage));
+ return Some((Err(format_err!("unable to read subdir '{}' - {}", subdir, err)), percentage, false));
}
}
}
pub fn sweep_unused_chunks(
&self,
oldest_writer: i64,
+ phase1_start_time: i64,
status: &mut GarbageCollectionStatus,
worker: &WorkerTask,
) -> Result<(), Error> {
use nix::sys::stat::fstatat;
+ use nix::unistd::{unlinkat, UnlinkatFlags};
- let now = unsafe { libc::time(std::ptr::null_mut()) };
-
- let mut min_atime = now - 3600*24; // at least 24h (see mount option relatime)
+ let mut min_atime = phase1_start_time - 3600*24; // at least 24h (see mount option relatime)
if oldest_writer < min_atime {
min_atime = oldest_writer;
let mut last_percentage = 0;
let mut chunk_count = 0;
- for (entry, percentage) in self.get_chunk_iterator()? {
+ for (entry, percentage, bad) in self.get_chunk_iterator()? {
if last_percentage != percentage {
last_percentage = percentage;
- worker.log(format!("percentage done: {}, chunk count: {}", percentage, chunk_count));
+ worker.log(format!("percentage done: phase2 {}% (processed {} chunks)", percentage, chunk_count));
}
worker.fail_on_abort()?;
let lock = self.mutex.lock();
if let Ok(stat) = fstatat(dirfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
- if stat.st_atime < min_atime {
+ if bad {
+ match std::ffi::CString::new(&filename.to_bytes()[..64]) {
+ Ok(orig_filename) => {
+ match fstatat(
+ dirfd,
+ orig_filename.as_c_str(),
+ nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW)
+ {
+ Ok(_) => { /* do nothing */ },
+ Err(nix::Error::Sys(nix::errno::Errno::ENOENT)) => {
+ // chunk hasn't been rewritten yet, keep
+ // .bad file around for manual recovery
+ continue;
+ },
+ Err(err) => {
+ // some other error, warn user and keep
+ // .bad file around too
+ worker.warn(format!(
+ "error during stat on '{:?}' - {}",
+ orig_filename,
+ err,
+ ));
+ continue;
+ }
+ }
+ },
+ Err(err) => {
+ worker.warn(format!(
+ "could not get original filename from .bad file '{:?}' - {}",
+ filename,
+ err,
+ ));
+ continue;
+ }
+ }
+
+ if let Err(err) = unlinkat(Some(dirfd), filename, UnlinkatFlags::NoRemoveDir) {
+ worker.warn(format!(
+ "unlinking corrupt chunk {:?} failed on store '{}' - {}",
+ filename,
+ self.name,
+ err,
+ ));
+ } else {
+ status.removed_bad += 1;
+ status.removed_bytes += stat.st_size as u64;
+ }
+ } else if stat.st_atime < min_atime {
//let age = now - stat.st_atime;
//println!("UNLINK {} {:?}", age/(3600*24), filename);
- let res = unsafe { libc::unlinkat(dirfd, filename.as_ptr(), 0) };
- if res != 0 {
- let err = nix::Error::last();
+ if let Err(err) = unlinkat(Some(dirfd), filename, UnlinkatFlags::NoRemoveDir) {
bail!(
- "unlink chunk {:?} failed on store '{}' - {}",
+ "unlinking chunk {:?} failed on store '{}' - {}",
filename,
self.name,
err,
if let Ok(metadata) = std::fs::metadata(&chunk_path) {
if metadata.is_file() {
+ self.touch_chunk(digest)?;
return Ok((true, metadata.len()));
} else {
bail!("Got unexpected file type on store '{}' for chunk {}", self.name, digest_str);
full_path
}
+ pub fn name(&self) -> &str {
+ &self.name
+ }
+
pub fn base_path(&self) -> PathBuf {
self.base.clone()
}
projects / proxmox-backup.git / blobdiff