src/backup/chunk_store.rs

   1 use failure::*;
   2 use std::path::{Path, PathBuf};
   3 use std::io::Write;
   4 use std::time::Duration;
   5
   6 use crypto::digest::Digest;
   7 use crypto::sha2::Sha512Trunc256;
   8 use std::sync::Mutex;
   9
  10 use std::fs::File;
  11 use std::os::unix::io::AsRawFd;
  12
  13 use crate::tools;
  14
  15 pub struct ChunkStore {
  16     name: String, // used for error reporting
  17     base: PathBuf,
  18     chunk_dir: PathBuf,
  19     hasher: Sha512Trunc256,
  20     mutex: Mutex<bool>,
  21     _lockfile: File,
  22 }
  23
  24 const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef";
  25
  26 pub fn digest_to_hex(digest: &[u8]) -> String {
  27
  28     let mut buf = Vec::<u8>::with_capacity(digest.len()*2);
  29
  30     for i in 0..digest.len() {
  31         buf.push(HEX_CHARS[(digest[i] >> 4) as usize]);
  32         buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]);
  33     }
  34
  35     unsafe { String::from_utf8_unchecked(buf) }
  36 }
  37
  38 fn digest_to_prefix(digest: &[u8]) -> PathBuf {
  39
  40     let mut buf = Vec::<u8>::with_capacity(3+1+2+1);
  41
  42     buf.push(HEX_CHARS[(digest[0] as usize) >> 4]);
  43     buf.push(HEX_CHARS[(digest[0] as usize) &0xf]);
  44     buf.push(HEX_CHARS[(digest[1] as usize) >> 4]);
  45     buf.push('/' as u8);
  46
  47     buf.push(HEX_CHARS[(digest[1] as usize) & 0xf]);
  48     buf.push(HEX_CHARS[(digest[2] as usize) >> 4]);
  49     buf.push('/' as u8);
  50
  51     let path = unsafe { String::from_utf8_unchecked(buf)};
  52
  53     path.into()
  54 }
  55
  56
  57 impl ChunkStore {
  58
  59     fn chunk_dir<P: AsRef<Path>>(path: P) -> PathBuf {
  60
  61         let mut chunk_dir: PathBuf = PathBuf::from(path.as_ref());
  62         chunk_dir.push(".chunks");
  63
  64         chunk_dir
  65     }
  66
  67     pub fn create<P: Into<PathBuf>>(name: &str, path: P) -> Result<Self, Error> {
  68
  69         let base: PathBuf = path.into();
  70         let chunk_dir = Self::chunk_dir(&base);
  71
  72         if let Err(err) = std::fs::create_dir(&base) {
  73             bail!("unable to create chunk store '{}' at {:?} - {}", name, base, err);
  74         }
  75
  76         if let Err(err) = std::fs::create_dir(&chunk_dir) {
  77             bail!("unable to create chunk store '{}' subdir {:?} - {}", name, chunk_dir, err);
  78         }
  79
  80         // create 4096 subdir
  81         for i in 0..4096 {
  82             let mut l1path = chunk_dir.clone();
  83             l1path.push(format!("{:03x}",i));
  84             if let Err(err) = std::fs::create_dir(&l1path) {
  85                 bail!("unable to create chunk store '{}' subdir {:?} - {}", name, l1path, err);
  86             }
  87         }
  88
  89         Self::open(name, base)
  90     }
  91
  92     pub fn open<P: Into<PathBuf>>(name: &str, path: P) -> Result<Self, Error> {
  93
  94         let base: PathBuf = path.into();
  95         let chunk_dir = Self::chunk_dir(&base);
  96
  97         if let Err(err) = std::fs::metadata(&chunk_dir) {
  98             bail!("unable to open chunk store '{}' at {:?} - {}", name, chunk_dir, err);
  99         }
 100
 101         let mut lockfile_path = base.clone();
 102         lockfile_path.push(".lock");
 103
 104         // make sure only one process/thread/task can use it
 105         let lockfile = tools::open_file_locked(
 106             lockfile_path, Duration::from_secs(10))?;
 107
 108         Ok(ChunkStore {
 109             name: name.to_owned(),
 110             base,
 111             chunk_dir,
 112             hasher: Sha512Trunc256::new(),
 113             _lockfile: lockfile,
 114             mutex: Mutex::new(false)
 115         })
 116     }
 117
 118     pub fn touch_chunk(&mut self, digest:&[u8]) ->  Result<(), Error> {
 119
 120         // fixme:  nix::sys::stat::utimensat
 121         let mut chunk_path = self.chunk_dir.clone();
 122         let prefix = digest_to_prefix(&digest);
 123         chunk_path.push(&prefix);
 124         let digest_str = digest_to_hex(&digest);
 125         chunk_path.push(&digest_str);
 126
 127         std::fs::metadata(&chunk_path)?;
 128         Ok(())
 129     }
 130
 131     fn sweep_old_files(&self, handle: &mut nix::dir::Dir) -> Result<(), Error> {
 132
 133         let rawfd = handle.as_raw_fd();
 134
 135         let now = unsafe { libc::time(std::ptr::null_mut()) };
 136
 137         for entry in handle.iter() {
 138             let entry = match entry {
 139                 Ok(entry) => entry,
 140                 Err(_) => continue /* ignore */,
 141             };
 142             let file_type = match entry.file_type() {
 143                 Some(file_type) => file_type,
 144                 None => bail!("unsupported file system type on chunk store '{}'", self.name),
 145             };
 146             if file_type != nix::dir::Type::File { continue; }
 147
 148             let filename = entry.file_name();
 149             if let Ok(stat) = nix::sys::stat::fstatat(rawfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
 150                 let age = now - stat.st_atime;
 151                 println!("FOUND {}  {:?}", age/(3600*24), filename);
 152                 if age/(3600*24) >= 2 {
 153                     println!("UNLINK {}  {:?}", age/(3600*24), filename);
 154                     let res = unsafe { libc::unlinkat(rawfd, filename.as_ptr(), 0) };
 155                     if res != 0 {
 156                         let err = nix::Error::last();
 157                         bail!("unlink chunk {:?} failed on store '{}' - {}", filename, self.name, err);
 158                     }
 159                 }
 160             }
 161         }
 162         Ok(())
 163     }
 164
 165     pub fn sweep_used_chunks(&mut self) -> Result<(), Error> {
 166
 167         use nix::fcntl::OFlag;
 168         use nix::sys::stat::Mode;
 169         use nix::dir::Dir;
 170
 171         let base_handle = match Dir::open(
 172             &self.chunk_dir, OFlag::O_RDONLY, Mode::empty()) {
 173             Ok(h) => h,
 174             Err(err) => bail!("unable to open store '{}' chunk dir {:?} - {}",
 175                               self.name, self.chunk_dir, err),
 176         };
 177
 178         let base_fd = base_handle.as_raw_fd();
 179
 180         for i in 0..4096 {
 181             let l1name = PathBuf::from(format!("{:03x}", i));
 182             let mut l1_handle = match nix::dir::Dir::openat(
 183                 base_fd, &l1name, OFlag::O_RDONLY, Mode::empty()) {
 184                 Ok(h) => h,
 185                 Err(err) => bail!("unable to open store '{}' dir {:?}/{:?} - {}",
 186                                   self.name, self.chunk_dir, l1name, err),
 187             };
 188
 189             let l1_fd = l1_handle.as_raw_fd();
 190
 191             for l1_entry in l1_handle.iter() {
 192                 let l1_entry = match l1_entry {
 193                     Ok(l1_entry) => l1_entry,
 194                     Err(_) => continue /* ignore errors? */,
 195                 };
 196                 let file_type = match l1_entry.file_type() {
 197                     Some(file_type) => file_type,
 198                     None => bail!("unsupported file system type on chunk store '{}'", self.name),
 199                 };
 200                 if file_type != nix::dir::Type::Directory { continue; }
 201
 202                 let l2name = l1_entry.file_name();
 203                 if l2name.to_bytes_with_nul()[0] == b'.' { continue; }
 204
 205                 let mut l2_handle = match Dir::openat(
 206                     l1_fd, l2name, OFlag::O_RDONLY, Mode::empty()) {
 207                     Ok(h) => h,
 208                     Err(err) => bail!(
 209                         "unable to open store '{}' dir {:?}/{:?}/{:?} - {}",
 210                         self.name, self.chunk_dir, l1name, l2name, err),
 211                 };
 212                 self.sweep_old_files(&mut l2_handle)?;
 213             }
 214         }
 215         Ok(())
 216     }
 217
 218     pub fn insert_chunk(&mut self, chunk: &[u8]) -> Result<(bool, [u8; 32]), Error> {
 219
 220         self.hasher.reset();
 221         self.hasher.input(chunk);
 222
 223         let mut digest = [0u8; 32];
 224         self.hasher.result(&mut digest);
 225         //println!("DIGEST {}", digest_to_hex(&digest));
 226
 227         let mut chunk_path = self.chunk_dir.clone();
 228         let prefix = digest_to_prefix(&digest);
 229         chunk_path.push(&prefix);
 230         let digest_str = digest_to_hex(&digest);
 231         chunk_path.push(&digest_str);
 232
 233         let lock = self.mutex.lock();
 234
 235         if let Ok(metadata) = std::fs::metadata(&chunk_path) {
 236             if metadata.is_file() {
 237                  return Ok((true, digest));
 238             } else {
 239                 bail!("Got unexpected file type for chunk {}", digest_str);
 240             }
 241         }
 242
 243         let mut chunk_dir = self.chunk_dir.clone();
 244         chunk_dir.push(&prefix);
 245
 246         if let Err(_) = std::fs::create_dir(&chunk_dir) { /* ignore */ }
 247
 248         let mut tmp_path = chunk_path.clone();
 249         tmp_path.set_extension("tmp");
 250         let mut f = std::fs::File::create(&tmp_path)?;
 251         f.write_all(chunk)?;
 252
 253         if let Err(err) = std::fs::rename(&tmp_path, &chunk_path) {
 254             if let Err(_) = std::fs::remove_file(&tmp_path)  { /* ignore */ }
 255             bail!("Atomic rename failed for chunk {} - {}", digest_str, err);
 256         }
 257
 258         println!("PATH {:?}", chunk_path);
 259
 260         drop(lock);
 261
 262         Ok((false, digest))
 263     }
 264
 265     pub fn relative_path(&self, path: &Path) -> PathBuf {
 266
 267         let mut full_path = self.base.clone();
 268         full_path.push(path);
 269         full_path
 270     }
 271
 272     pub fn base_path(&self) -> PathBuf {
 273         self.base.clone()
 274     }
 275
 276 }
 277
 278
 279 #[test]
 280 fn test_chunk_store1() {
 281
 282     if let Err(_e) = std::fs::remove_dir_all(".testdir") { /* ignore */ }
 283
 284     let chunk_store = ChunkStore::open(".testdir");
 285     assert!(chunk_store.is_err());
 286
 287     let mut chunk_store = ChunkStore::create(".testdir").unwrap();
 288     let (exists, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap();
 289     assert!(!exists);
 290
 291     let (exists, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap();
 292     assert!(exists);
 293
 294
 295     let chunk_store = ChunkStore::create(".testdir");
 296     assert!(chunk_store.is_err());
 297
 298
 299 }