]>
Commit | Line | Data |
---|---|---|
35cf5daa DM |
1 | use failure::*; |
2 | use std::path::{Path, PathBuf}; | |
128b37fe | 3 | use std::io::Write; |
1628a4c7 | 4 | use std::time::Duration; |
128b37fe | 5 | |
7b2b40a8 | 6 | use openssl::sha; |
c5d82e5f DM |
7 | use std::sync::Mutex; |
8 | ||
365bb90f | 9 | use std::fs::File; |
45773720 | 10 | use std::os::unix::io::AsRawFd; |
35cf5daa | 11 | |
365bb90f DM |
12 | use crate::tools; |
13 | ||
64e53b28 DM |
14 | pub struct GarbageCollectionStatus { |
15 | pub used_bytes: usize, | |
16 | pub used_chunks: usize, | |
17 | pub disk_bytes: usize, | |
18 | pub disk_chunks: usize, | |
19 | } | |
20 | ||
21 | impl Default for GarbageCollectionStatus { | |
22 | fn default() -> Self { | |
23 | GarbageCollectionStatus { | |
24 | used_bytes: 0, | |
25 | used_chunks: 0, | |
26 | disk_bytes: 0, | |
27 | disk_chunks: 0, | |
28 | } | |
29 | } | |
30 | } | |
31 | ||
35cf5daa | 32 | pub struct ChunkStore { |
277fc5a3 | 33 | name: String, // used for error reporting |
2c32fdde | 34 | pub (crate) base: PathBuf, |
35cf5daa | 35 | chunk_dir: PathBuf, |
c5d82e5f | 36 | mutex: Mutex<bool>, |
eae8aa3a | 37 | _lockfile: File, |
128b37fe DM |
38 | } |
39 | ||
40 | const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef"; | |
41 | ||
176e4af9 DM |
42 | // TODO: what about sysctl setting vm.vfs_cache_pressure (0 - 100) ? |
43 | ||
08481a0b | 44 | pub fn digest_to_hex(digest: &[u8]) -> String { |
128b37fe | 45 | |
3d5c11e5 | 46 | let mut buf = Vec::<u8>::with_capacity(digest.len()*2); |
128b37fe | 47 | |
3d5c11e5 | 48 | for i in 0..digest.len() { |
128b37fe DM |
49 | buf.push(HEX_CHARS[(digest[i] >> 4) as usize]); |
50 | buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]); | |
51 | } | |
52 | ||
53 | unsafe { String::from_utf8_unchecked(buf) } | |
35cf5daa DM |
54 | } |
55 | ||
08481a0b | 56 | fn digest_to_prefix(digest: &[u8]) -> PathBuf { |
128b37fe | 57 | |
e95950e4 | 58 | let mut buf = Vec::<u8>::with_capacity(2+1+2+1); |
128b37fe DM |
59 | |
60 | buf.push(HEX_CHARS[(digest[0] as usize) >> 4]); | |
61 | buf.push(HEX_CHARS[(digest[0] as usize) &0xf]); | |
e95950e4 | 62 | buf.push(HEX_CHARS[(digest[1] as usize) >> 4]); |
128b37fe | 63 | buf.push(HEX_CHARS[(digest[1] as usize) & 0xf]); |
128b37fe DM |
64 | buf.push('/' as u8); |
65 | ||
66 | let path = unsafe { String::from_utf8_unchecked(buf)}; | |
67 | ||
68 | path.into() | |
35cf5daa DM |
69 | } |
70 | ||
12bb93b3 | 71 | |
35cf5daa DM |
72 | impl ChunkStore { |
73 | ||
45773720 DM |
74 | fn chunk_dir<P: AsRef<Path>>(path: P) -> PathBuf { |
75 | ||
76 | let mut chunk_dir: PathBuf = PathBuf::from(path.as_ref()); | |
77 | chunk_dir.push(".chunks"); | |
78 | ||
79 | chunk_dir | |
80 | } | |
81 | ||
277fc5a3 | 82 | pub fn create<P: Into<PathBuf>>(name: &str, path: P) -> Result<Self, Error> { |
35cf5daa | 83 | |
45773720 DM |
84 | let base: PathBuf = path.into(); |
85 | let chunk_dir = Self::chunk_dir(&base); | |
35cf5daa | 86 | |
2989f6bf | 87 | if let Err(err) = std::fs::create_dir(&base) { |
277fc5a3 | 88 | bail!("unable to create chunk store '{}' at {:?} - {}", name, base, err); |
2989f6bf DM |
89 | } |
90 | ||
91 | if let Err(err) = std::fs::create_dir(&chunk_dir) { | |
277fc5a3 | 92 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, chunk_dir, err); |
2989f6bf | 93 | } |
35cf5daa | 94 | |
bc616633 | 95 | // create 64*1024 subdirs |
e95950e4 DM |
96 | let mut last_percentage = 0; |
97 | ||
bc616633 | 98 | for i in 0..64*1024 { |
af3e7d75 | 99 | let mut l1path = chunk_dir.clone(); |
bc616633 | 100 | l1path.push(format!("{:04x}", i)); |
2989f6bf | 101 | if let Err(err) = std::fs::create_dir(&l1path) { |
277fc5a3 | 102 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, l1path, err); |
2989f6bf | 103 | } |
bc616633 DM |
104 | let percentage = (i*100)/(64*1024); |
105 | if percentage != last_percentage { | |
106 | eprintln!("Percentage done: {}", percentage); | |
107 | last_percentage = percentage; | |
e95950e4 | 108 | } |
35cf5daa DM |
109 | } |
110 | ||
277fc5a3 | 111 | Self::open(name, base) |
35cf5daa DM |
112 | } |
113 | ||
277fc5a3 | 114 | pub fn open<P: Into<PathBuf>>(name: &str, path: P) -> Result<Self, Error> { |
35cf5daa | 115 | |
45773720 DM |
116 | let base: PathBuf = path.into(); |
117 | let chunk_dir = Self::chunk_dir(&base); | |
118 | ||
ce55dbbc | 119 | if let Err(err) = std::fs::metadata(&chunk_dir) { |
277fc5a3 | 120 | bail!("unable to open chunk store '{}' at {:?} - {}", name, chunk_dir, err); |
ce55dbbc | 121 | } |
45773720 DM |
122 | |
123 | let mut lockfile_path = base.clone(); | |
124 | lockfile_path.push(".lock"); | |
125 | ||
5ba69689 | 126 | // make sure only one process/thread/task can use it |
1628a4c7 WB |
127 | let lockfile = tools::open_file_locked( |
128 | lockfile_path, Duration::from_secs(10))?; | |
35cf5daa | 129 | |
b8d4766a | 130 | Ok(ChunkStore { |
277fc5a3 | 131 | name: name.to_owned(), |
b8d4766a DM |
132 | base, |
133 | chunk_dir, | |
bc616633 | 134 | _lockfile: lockfile, |
b8d4766a DM |
135 | mutex: Mutex::new(false) |
136 | }) | |
35cf5daa DM |
137 | } |
138 | ||
03e4753d | 139 | pub fn touch_chunk(&self, digest:&[u8]) -> Result<(), Error> { |
3d5c11e5 | 140 | |
a198d74f | 141 | let mut chunk_path = self.chunk_dir.clone(); |
08481a0b | 142 | let prefix = digest_to_prefix(&digest); |
3d5c11e5 | 143 | chunk_path.push(&prefix); |
08481a0b | 144 | let digest_str = digest_to_hex(&digest); |
3d5c11e5 DM |
145 | chunk_path.push(&digest_str); |
146 | ||
7ee2aa1b DM |
147 | const UTIME_NOW: i64 = ((1 << 30) - 1); |
148 | const UTIME_OMIT: i64 = ((1 << 30) - 2); | |
149 | ||
a198d74f | 150 | let times: [libc::timespec; 2] = [ |
7ee2aa1b DM |
151 | libc::timespec { tv_sec: 0, tv_nsec: UTIME_NOW }, |
152 | libc::timespec { tv_sec: 0, tv_nsec: UTIME_OMIT } | |
153 | ]; | |
154 | ||
155 | use nix::NixPath; | |
156 | ||
157 | let res = chunk_path.with_nix_path(|cstr| unsafe { | |
158 | libc::utimensat(-1, cstr.as_ptr(), ×[0], libc::AT_SYMLINK_NOFOLLOW) | |
159 | })?; | |
160 | ||
161 | if let Err(err) = nix::errno::Errno::result(res) { | |
162 | bail!("updata atime failed for chunk {:?} - {}", chunk_path, err); | |
163 | } | |
164 | ||
3d5c11e5 DM |
165 | Ok(()) |
166 | } | |
167 | ||
64e53b28 | 168 | fn sweep_old_files(&self, handle: &mut nix::dir::Dir, status: &mut GarbageCollectionStatus) -> Result<(), Error> { |
08481a0b DM |
169 | |
170 | let rawfd = handle.as_raw_fd(); | |
171 | ||
172 | let now = unsafe { libc::time(std::ptr::null_mut()) }; | |
173 | ||
174 | for entry in handle.iter() { | |
eae8aa3a DM |
175 | let entry = match entry { |
176 | Ok(entry) => entry, | |
177 | Err(_) => continue /* ignore */, | |
178 | }; | |
179 | let file_type = match entry.file_type() { | |
180 | Some(file_type) => file_type, | |
277fc5a3 | 181 | None => bail!("unsupported file system type on chunk store '{}'", self.name), |
eae8aa3a DM |
182 | }; |
183 | if file_type != nix::dir::Type::File { continue; } | |
184 | ||
185 | let filename = entry.file_name(); | |
186 | if let Ok(stat) = nix::sys::stat::fstatat(rawfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) { | |
187 | let age = now - stat.st_atime; | |
e95950e4 | 188 | //println!("FOUND {} {:?}", age/(3600*24), filename); |
eae8aa3a DM |
189 | if age/(3600*24) >= 2 { |
190 | println!("UNLINK {} {:?}", age/(3600*24), filename); | |
277fc5a3 DM |
191 | let res = unsafe { libc::unlinkat(rawfd, filename.as_ptr(), 0) }; |
192 | if res != 0 { | |
193 | let err = nix::Error::last(); | |
194 | bail!("unlink chunk {:?} failed on store '{}' - {}", filename, self.name, err); | |
195 | } | |
64e53b28 DM |
196 | } else { |
197 | status.disk_chunks += 1; | |
198 | status.disk_bytes += stat.st_size as usize; | |
199 | ||
08481a0b | 200 | } |
eae8aa3a DM |
201 | } |
202 | } | |
277fc5a3 | 203 | Ok(()) |
08481a0b DM |
204 | } |
205 | ||
77703d95 | 206 | pub fn sweep_unused_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> { |
6ea3a0b7 | 207 | |
1c43c56b DM |
208 | use nix::fcntl::OFlag; |
209 | use nix::sys::stat::Mode; | |
210 | use nix::dir::Dir; | |
211 | ||
eae8aa3a | 212 | let base_handle = match Dir::open( |
1c43c56b | 213 | &self.chunk_dir, OFlag::O_RDONLY, Mode::empty()) { |
2bf5f6b2 | 214 | Ok(h) => h, |
277fc5a3 DM |
215 | Err(err) => bail!("unable to open store '{}' chunk dir {:?} - {}", |
216 | self.name, self.chunk_dir, err), | |
2bf5f6b2 DM |
217 | }; |
218 | ||
219 | let base_fd = base_handle.as_raw_fd(); | |
220 | ||
176e4af9 DM |
221 | let mut last_percentage = 0; |
222 | ||
bc616633 | 223 | for i in 0..64*1024 { |
2bf5f6b2 | 224 | |
bc616633 DM |
225 | let percentage = (i*100)/(64*1024); |
226 | if percentage != last_percentage { | |
227 | eprintln!("Percentage done: {}", percentage); | |
228 | last_percentage = percentage; | |
229 | } | |
6c20a13d | 230 | |
bc616633 DM |
231 | let l1name = PathBuf::from(format!("{:04x}", i)); |
232 | match nix::dir::Dir::openat(base_fd, &l1name, OFlag::O_RDONLY, Mode::empty()) { | |
233 | Ok(mut h) => { | |
234 | //println!("SCAN {:?} {:?}", l1name); | |
235 | self.sweep_old_files(&mut h, status)?; | |
176e4af9 | 236 | } |
bc616633 DM |
237 | Err(err) => bail!("unable to open store '{}' dir {:?}/{:?} - {}", |
238 | self.name, self.chunk_dir, l1name, err), | |
239 | }; | |
08481a0b | 240 | } |
6ea3a0b7 DM |
241 | Ok(()) |
242 | } | |
243 | ||
03e4753d | 244 | pub fn insert_chunk(&self, chunk: &[u8]) -> Result<(bool, [u8; 32]), Error> { |
128b37fe | 245 | |
7b2b40a8 DM |
246 | // fixme: use Sha512/256 when available |
247 | let mut hasher = sha::Sha256::new(); | |
248 | hasher.update(chunk); | |
249 | ||
250 | let digest = hasher.finish(); | |
251 | ||
08481a0b | 252 | //println!("DIGEST {}", digest_to_hex(&digest)); |
128b37fe | 253 | |
af3e7d75 | 254 | let mut chunk_path = self.chunk_dir.clone(); |
08481a0b | 255 | let prefix = digest_to_prefix(&digest); |
c5d82e5f | 256 | chunk_path.push(&prefix); |
08481a0b | 257 | let digest_str = digest_to_hex(&digest); |
c5d82e5f DM |
258 | chunk_path.push(&digest_str); |
259 | ||
260 | let lock = self.mutex.lock(); | |
261 | ||
262 | if let Ok(metadata) = std::fs::metadata(&chunk_path) { | |
263 | if metadata.is_file() { | |
391a2e43 | 264 | return Ok((true, digest)); |
c5d82e5f | 265 | } else { |
f7dd683b | 266 | bail!("Got unexpected file type on store '{}' for chunk {}", self.name, digest_str); |
c5d82e5f DM |
267 | } |
268 | } | |
128b37fe | 269 | |
c5d82e5f DM |
270 | let mut tmp_path = chunk_path.clone(); |
271 | tmp_path.set_extension("tmp"); | |
272 | let mut f = std::fs::File::create(&tmp_path)?; | |
128b37fe DM |
273 | f.write_all(chunk)?; |
274 | ||
c5d82e5f DM |
275 | if let Err(err) = std::fs::rename(&tmp_path, &chunk_path) { |
276 | if let Err(_) = std::fs::remove_file(&tmp_path) { /* ignore */ } | |
f7dd683b | 277 | bail!("Atomic rename on store '{}' failed for chunk {} - {}", self.name, digest_str, err); |
c5d82e5f DM |
278 | } |
279 | ||
77703d95 | 280 | //println!("PATH {:?}", chunk_path); |
128b37fe | 281 | |
c5d82e5f DM |
282 | drop(lock); |
283 | ||
391a2e43 | 284 | Ok((false, digest)) |
128b37fe DM |
285 | } |
286 | ||
606ce64b DM |
287 | pub fn relative_path(&self, path: &Path) -> PathBuf { |
288 | ||
289 | let mut full_path = self.base.clone(); | |
290 | full_path.push(path); | |
291 | full_path | |
292 | } | |
293 | ||
3d5c11e5 DM |
294 | pub fn base_path(&self) -> PathBuf { |
295 | self.base.clone() | |
296 | } | |
35cf5daa DM |
297 | } |
298 | ||
299 | ||
300 | #[test] | |
301 | fn test_chunk_store1() { | |
302 | ||
303 | if let Err(_e) = std::fs::remove_dir_all(".testdir") { /* ignore */ } | |
304 | ||
f7dd683b | 305 | let chunk_store = ChunkStore::open("test", ".testdir"); |
35cf5daa DM |
306 | assert!(chunk_store.is_err()); |
307 | ||
03e4753d | 308 | let chunk_store = ChunkStore::create("test", ".testdir").unwrap(); |
391a2e43 DM |
309 | let (exists, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap(); |
310 | assert!(!exists); | |
311 | ||
312 | let (exists, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap(); | |
313 | assert!(exists); | |
128b37fe | 314 | |
35cf5daa | 315 | |
f7dd683b | 316 | let chunk_store = ChunkStore::create("test", ".testdir"); |
35cf5daa DM |
317 | assert!(chunk_store.is_err()); |
318 | ||
319 | ||
320 | } |