impl sweep_used_chunks, first try

diff --git a/Cargo.toml b/Cargo.toml
index c36583a8c6b7bf146ed30a00f101d5d8867472c4..fbea3dc68498872fab082902d9d9f1aa20477ab3 100644 (file)
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -25,6 +25,7 @@ hyper = "0.12.14"
 lazy_static = "1.1.0"
 regex = "1.0.6"
 rust-crypto = "0.2.36"
+libc = "0.2"
 nix = "0.12.0"
 shellwords = "1.0.0"
 uuid = { version = "0.7", features = ["v4"] }

diff --git a/src/backup/chunk_store.rs b/src/backup/chunk_store.rs
index 2975ee7aa48335f49d96fd9dce1103b5e536b830..f54956daf3fa6af060f5cb4ed90ad9d939d4d243 100644 (file)
--- a/src/backup/chunk_store.rs
+++ b/src/backup/chunk_store.rs
@@ -20,7 +20,7 @@ pub struct ChunkStore {
 
 const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef";
 
-pub fn u256_to_hex(digest: &[u8]) -> String {
+pub fn digest_to_hex(digest: &[u8]) -> String {
 
     let mut buf = Vec::<u8>::with_capacity(digest.len()*2);
 
@@ -32,7 +32,7 @@ pub fn u256_to_hex(digest: &[u8]) -> String {
     unsafe { String::from_utf8_unchecked(buf) }
 }
 
-fn u256_to_prefix(digest: &[u8]) -> PathBuf {
+fn digest_to_prefix(digest: &[u8]) -> PathBuf {
 
     let mut buf = Vec::<u8>::with_capacity(3+1+2+1);
 
@@ -153,18 +153,66 @@ impl ChunkStore {
 
     pub fn touch_chunk(&mut self, digest:&[u8]) ->  Result<(), Error> {
 
+        // fixme:  nix::sys::stat::utimensat
         let mut chunk_path = self.chunk_dir.clone();
-        let prefix = u256_to_prefix(&digest);
+        let prefix = digest_to_prefix(&digest);
         chunk_path.push(&prefix);
-        let digest_str = u256_to_hex(&digest);
+        let digest_str = digest_to_hex(&digest);
         chunk_path.push(&digest_str);
 
         std::fs::metadata(&chunk_path)?;
         Ok(())
     }
 
+    fn sweep_old_files(&self, dir: &Path) {
+
+        let mut handle = match nix::dir::Dir::open(
+            dir, nix::fcntl::OFlag::O_RDONLY, nix::sys::stat::Mode::empty()) {
+            Ok(h) => h,
+            Err(_) => return,
+        };
+
+        let rawfd = handle.as_raw_fd();
+
+        let now = unsafe { libc::time(std::ptr::null_mut()) };
+
+        for entry in handle.iter() {
+             match entry {
+                Ok(entry) => {
+                    if let Some(file_type) = entry.file_type() {
+                        if file_type == nix::dir::Type::File {
+                            let filename = entry.file_name();
+                            if let Ok(stat) = nix::sys::stat::fstatat(rawfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) {
+                                let age = now - stat.st_atime;
+                                println!("FOUND {}  {:?}", age/(3600*24), filename);
+                                if age/(3600*24) >= 2 {
+                                    println!("UNLINK {}  {:?}", age/(3600*24), filename);
+                                    unsafe { libc::unlinkat(rawfd, filename.as_ptr(), 0); }
+                                }
+                            }
+                        }
+                    }
+                }
+                Err(_) => {
+                    // fixme ??
+                }
+             }
+         }
+
+    }
+
     pub fn sweep_used_chunks(&mut self) -> Result<(), Error> {
 
+        for i in 0..4096 {
+            let mut l1path = self.chunk_dir.clone();
+            l1path.push(format!("{:03x}", i));
+            for j in 0..256 {
+                let mut l2path = l1path.clone();
+                l2path.push(format!("{:02x}", j));
+                self.sweep_old_files(&l2path);
+            }
+        }
+
         Ok(())
     }
 
@@ -175,12 +223,12 @@ impl ChunkStore {
 
         let mut digest = [0u8; 32];
         self.hasher.result(&mut digest);
-        //println!("DIGEST {}", u256_to_hex(&digest));
+        //println!("DIGEST {}", digest_to_hex(&digest));
 
         let mut chunk_path = self.chunk_dir.clone();
-        let prefix = u256_to_prefix(&digest);
+        let prefix = digest_to_prefix(&digest);
         chunk_path.push(&prefix);
-        let digest_str = u256_to_hex(&digest);
+        let digest_str = digest_to_hex(&digest);
         chunk_path.push(&digest_str);
 
         let lock = self.mutex.lock();

diff --git a/src/backup/image_index.rs b/src/backup/image_index.rs
index 5b9f6eba695e792ec9fffe2cc3ac9367c2436dc2..cfc056ac44c95f309d7dd282bb865aa10849542c 100644 (file)
--- a/src/backup/image_index.rs
+++ b/src/backup/image_index.rs
@@ -111,7 +111,7 @@ impl <'a> ImageIndexReader<'a> {
             let digest = unsafe { std::slice::from_raw_parts_mut(self.index.add(pos*32), 32) };
             if let Err(err) = self.store.touch_chunk(digest) {
                 bail!("unable to access chunk {}, required by {:?} - {}",
-                      u256_to_hex(digest), self.filename, err);
+                      digest_to_hex(digest), self.filename, err);
             }
         }
 
@@ -262,7 +262,7 @@ impl <'a> ImageIndexWriter<'a> {
 
         let (is_duplicate, digest) = self.store.insert_chunk(chunk)?;
 
-        println!("ADD CHUNK {} {} {} {}", pos, chunk.len(), is_duplicate,  u256_to_hex(&digest));
+        println!("ADD CHUNK {} {} {} {}", pos, chunk.len(), is_duplicate,  digest_to_hex(&digest));
 
         let index_pos = (pos/self.chunk_size)*32;
         unsafe {