]>
Commit | Line | Data |
---|---|---|
35cf5daa DM |
1 | use failure::*; |
2 | use std::path::{Path, PathBuf}; | |
128b37fe | 3 | use std::io::Write; |
1628a4c7 | 4 | use std::time::Duration; |
128b37fe | 5 | |
7b2b40a8 | 6 | use openssl::sha; |
c5d82e5f DM |
7 | use std::sync::Mutex; |
8 | ||
365bb90f | 9 | use std::fs::File; |
45773720 | 10 | use std::os::unix::io::AsRawFd; |
35cf5daa | 11 | |
365bb90f DM |
12 | use crate::tools; |
13 | ||
64e53b28 DM |
14 | pub struct GarbageCollectionStatus { |
15 | pub used_bytes: usize, | |
16 | pub used_chunks: usize, | |
17 | pub disk_bytes: usize, | |
18 | pub disk_chunks: usize, | |
19 | } | |
20 | ||
21 | impl Default for GarbageCollectionStatus { | |
22 | fn default() -> Self { | |
23 | GarbageCollectionStatus { | |
24 | used_bytes: 0, | |
25 | used_chunks: 0, | |
26 | disk_bytes: 0, | |
27 | disk_chunks: 0, | |
28 | } | |
29 | } | |
30 | } | |
31 | ||
35cf5daa | 32 | pub struct ChunkStore { |
277fc5a3 | 33 | name: String, // used for error reporting |
2c32fdde | 34 | pub (crate) base: PathBuf, |
35cf5daa | 35 | chunk_dir: PathBuf, |
c5d82e5f | 36 | mutex: Mutex<bool>, |
eae8aa3a | 37 | _lockfile: File, |
128b37fe DM |
38 | } |
39 | ||
40 | const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef"; | |
41 | ||
176e4af9 DM |
42 | // TODO: what about sysctl setting vm.vfs_cache_pressure (0 - 100) ? |
43 | ||
08481a0b | 44 | pub fn digest_to_hex(digest: &[u8]) -> String { |
128b37fe | 45 | |
3d5c11e5 | 46 | let mut buf = Vec::<u8>::with_capacity(digest.len()*2); |
128b37fe | 47 | |
3d5c11e5 | 48 | for i in 0..digest.len() { |
128b37fe DM |
49 | buf.push(HEX_CHARS[(digest[i] >> 4) as usize]); |
50 | buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]); | |
51 | } | |
52 | ||
53 | unsafe { String::from_utf8_unchecked(buf) } | |
35cf5daa DM |
54 | } |
55 | ||
08481a0b | 56 | fn digest_to_prefix(digest: &[u8]) -> PathBuf { |
128b37fe | 57 | |
e95950e4 | 58 | let mut buf = Vec::<u8>::with_capacity(2+1+2+1); |
128b37fe DM |
59 | |
60 | buf.push(HEX_CHARS[(digest[0] as usize) >> 4]); | |
61 | buf.push(HEX_CHARS[(digest[0] as usize) &0xf]); | |
128b37fe | 62 | buf.push('/' as u8); |
35cf5daa | 63 | |
e95950e4 | 64 | buf.push(HEX_CHARS[(digest[1] as usize) >> 4]); |
128b37fe | 65 | buf.push(HEX_CHARS[(digest[1] as usize) & 0xf]); |
128b37fe DM |
66 | buf.push('/' as u8); |
67 | ||
68 | let path = unsafe { String::from_utf8_unchecked(buf)}; | |
69 | ||
70 | path.into() | |
35cf5daa DM |
71 | } |
72 | ||
12bb93b3 | 73 | |
35cf5daa DM |
74 | impl ChunkStore { |
75 | ||
45773720 DM |
76 | fn chunk_dir<P: AsRef<Path>>(path: P) -> PathBuf { |
77 | ||
78 | let mut chunk_dir: PathBuf = PathBuf::from(path.as_ref()); | |
79 | chunk_dir.push(".chunks"); | |
80 | ||
81 | chunk_dir | |
82 | } | |
83 | ||
277fc5a3 | 84 | pub fn create<P: Into<PathBuf>>(name: &str, path: P) -> Result<Self, Error> { |
35cf5daa | 85 | |
45773720 DM |
86 | let base: PathBuf = path.into(); |
87 | let chunk_dir = Self::chunk_dir(&base); | |
35cf5daa | 88 | |
2989f6bf | 89 | if let Err(err) = std::fs::create_dir(&base) { |
277fc5a3 | 90 | bail!("unable to create chunk store '{}' at {:?} - {}", name, base, err); |
2989f6bf DM |
91 | } |
92 | ||
93 | if let Err(err) = std::fs::create_dir(&chunk_dir) { | |
277fc5a3 | 94 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, chunk_dir, err); |
2989f6bf | 95 | } |
35cf5daa | 96 | |
e95950e4 DM |
97 | // create 256*256 subdirs |
98 | let mut last_percentage = 0; | |
99 | ||
100 | for i in 0..256 { | |
af3e7d75 | 101 | let mut l1path = chunk_dir.clone(); |
e95950e4 | 102 | l1path.push(format!("{:02x}",i)); |
2989f6bf | 103 | if let Err(err) = std::fs::create_dir(&l1path) { |
277fc5a3 | 104 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, l1path, err); |
2989f6bf | 105 | } |
e95950e4 DM |
106 | for j in 0..256 { |
107 | let mut l2path = l1path.clone(); | |
108 | l2path.push(format!("{:02x}",j)); | |
109 | if let Err(err) = std::fs::create_dir(&l2path) { | |
110 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, l2path, err); | |
111 | } | |
112 | let percentage = ((i*256+j)*100)/(256*256); | |
113 | if percentage != last_percentage { | |
114 | eprintln!("Percentage done: {}", percentage); | |
115 | last_percentage = percentage; | |
116 | } | |
117 | } | |
35cf5daa DM |
118 | } |
119 | ||
277fc5a3 | 120 | Self::open(name, base) |
35cf5daa DM |
121 | } |
122 | ||
277fc5a3 | 123 | pub fn open<P: Into<PathBuf>>(name: &str, path: P) -> Result<Self, Error> { |
35cf5daa | 124 | |
45773720 DM |
125 | let base: PathBuf = path.into(); |
126 | let chunk_dir = Self::chunk_dir(&base); | |
127 | ||
ce55dbbc | 128 | if let Err(err) = std::fs::metadata(&chunk_dir) { |
277fc5a3 | 129 | bail!("unable to open chunk store '{}' at {:?} - {}", name, chunk_dir, err); |
ce55dbbc | 130 | } |
45773720 DM |
131 | |
132 | let mut lockfile_path = base.clone(); | |
133 | lockfile_path.push(".lock"); | |
134 | ||
5ba69689 | 135 | // make sure only one process/thread/task can use it |
1628a4c7 WB |
136 | let lockfile = tools::open_file_locked( |
137 | lockfile_path, Duration::from_secs(10))?; | |
35cf5daa | 138 | |
b8d4766a | 139 | Ok(ChunkStore { |
277fc5a3 | 140 | name: name.to_owned(), |
b8d4766a DM |
141 | base, |
142 | chunk_dir, | |
03e4753d | 143 | _lockfile: lockfile, |
b8d4766a DM |
144 | mutex: Mutex::new(false) |
145 | }) | |
35cf5daa DM |
146 | } |
147 | ||
03e4753d | 148 | pub fn touch_chunk(&self, digest:&[u8]) -> Result<(), Error> { |
3d5c11e5 | 149 | |
991de600 | 150 | let mut chunk_path = self.chunk_dir.clone(); |
08481a0b | 151 | let prefix = digest_to_prefix(&digest); |
3d5c11e5 | 152 | chunk_path.push(&prefix); |
08481a0b | 153 | let digest_str = digest_to_hex(&digest); |
3d5c11e5 DM |
154 | chunk_path.push(&digest_str); |
155 | ||
7ee2aa1b DM |
156 | const UTIME_NOW: i64 = ((1 << 30) - 1); |
157 | const UTIME_OMIT: i64 = ((1 << 30) - 2); | |
158 | ||
159 | let mut times: [libc::timespec; 2] = [ | |
160 | libc::timespec { tv_sec: 0, tv_nsec: UTIME_NOW }, | |
161 | libc::timespec { tv_sec: 0, tv_nsec: UTIME_OMIT } | |
162 | ]; | |
163 | ||
164 | use nix::NixPath; | |
165 | ||
166 | let res = chunk_path.with_nix_path(|cstr| unsafe { | |
167 | libc::utimensat(-1, cstr.as_ptr(), ×[0], libc::AT_SYMLINK_NOFOLLOW) | |
168 | })?; | |
169 | ||
170 | if let Err(err) = nix::errno::Errno::result(res) { | |
171 | bail!("updata atime failed for chunk {:?} - {}", chunk_path, err); | |
172 | } | |
173 | ||
3d5c11e5 DM |
174 | Ok(()) |
175 | } | |
176 | ||
64e53b28 | 177 | fn sweep_old_files(&self, handle: &mut nix::dir::Dir, status: &mut GarbageCollectionStatus) -> Result<(), Error> { |
08481a0b DM |
178 | |
179 | let rawfd = handle.as_raw_fd(); | |
180 | ||
181 | let now = unsafe { libc::time(std::ptr::null_mut()) }; | |
182 | ||
183 | for entry in handle.iter() { | |
eae8aa3a DM |
184 | let entry = match entry { |
185 | Ok(entry) => entry, | |
186 | Err(_) => continue /* ignore */, | |
187 | }; | |
188 | let file_type = match entry.file_type() { | |
189 | Some(file_type) => file_type, | |
277fc5a3 | 190 | None => bail!("unsupported file system type on chunk store '{}'", self.name), |
eae8aa3a DM |
191 | }; |
192 | if file_type != nix::dir::Type::File { continue; } | |
193 | ||
194 | let filename = entry.file_name(); | |
195 | if let Ok(stat) = nix::sys::stat::fstatat(rawfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) { | |
196 | let age = now - stat.st_atime; | |
e95950e4 | 197 | //println!("FOUND {} {:?}", age/(3600*24), filename); |
eae8aa3a DM |
198 | if age/(3600*24) >= 2 { |
199 | println!("UNLINK {} {:?}", age/(3600*24), filename); | |
277fc5a3 DM |
200 | let res = unsafe { libc::unlinkat(rawfd, filename.as_ptr(), 0) }; |
201 | if res != 0 { | |
202 | let err = nix::Error::last(); | |
203 | bail!("unlink chunk {:?} failed on store '{}' - {}", filename, self.name, err); | |
204 | } | |
64e53b28 DM |
205 | } else { |
206 | status.disk_chunks += 1; | |
207 | status.disk_bytes += stat.st_size as usize; | |
208 | ||
08481a0b | 209 | } |
eae8aa3a DM |
210 | } |
211 | } | |
277fc5a3 | 212 | Ok(()) |
08481a0b DM |
213 | } |
214 | ||
64e53b28 | 215 | pub fn sweep_used_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> { |
6ea3a0b7 | 216 | |
1c43c56b DM |
217 | use nix::fcntl::OFlag; |
218 | use nix::sys::stat::Mode; | |
219 | use nix::dir::Dir; | |
220 | ||
eae8aa3a | 221 | let base_handle = match Dir::open( |
1c43c56b | 222 | &self.chunk_dir, OFlag::O_RDONLY, Mode::empty()) { |
2bf5f6b2 | 223 | Ok(h) => h, |
277fc5a3 DM |
224 | Err(err) => bail!("unable to open store '{}' chunk dir {:?} - {}", |
225 | self.name, self.chunk_dir, err), | |
2bf5f6b2 DM |
226 | }; |
227 | ||
228 | let base_fd = base_handle.as_raw_fd(); | |
229 | ||
176e4af9 DM |
230 | let mut last_percentage = 0; |
231 | ||
e95950e4 DM |
232 | for i in 0..256 { |
233 | let l1name = PathBuf::from(format!("{:02x}", i)); | |
2bf5f6b2 | 234 | let mut l1_handle = match nix::dir::Dir::openat( |
1c43c56b | 235 | base_fd, &l1name, OFlag::O_RDONLY, Mode::empty()) { |
2bf5f6b2 | 236 | Ok(h) => h, |
277fc5a3 DM |
237 | Err(err) => bail!("unable to open store '{}' dir {:?}/{:?} - {}", |
238 | self.name, self.chunk_dir, l1name, err), | |
2bf5f6b2 DM |
239 | }; |
240 | ||
241 | let l1_fd = l1_handle.as_raw_fd(); | |
242 | ||
e95950e4 DM |
243 | for j in 0..256 { |
244 | let l2name = PathBuf::from(format!("{:02x}", j)); | |
6c20a13d | 245 | |
176e4af9 DM |
246 | let percentage = ((i*256+j)*100)/(256*256); |
247 | if percentage != last_percentage { | |
248 | eprintln!("Percentage done: {}", percentage); | |
249 | last_percentage = percentage; | |
250 | } | |
251 | //println!("SCAN {:?} {:?}", l1name, l2name); | |
6c20a13d DM |
252 | |
253 | let mut l2_handle = match Dir::openat( | |
e95950e4 | 254 | l1_fd, &l2name, OFlag::O_RDONLY, Mode::empty()) { |
6c20a13d DM |
255 | Ok(h) => h, |
256 | Err(err) => bail!( | |
277fc5a3 DM |
257 | "unable to open store '{}' dir {:?}/{:?}/{:?} - {}", |
258 | self.name, self.chunk_dir, l1name, l2name, err), | |
6c20a13d | 259 | }; |
64e53b28 | 260 | self.sweep_old_files(&mut l2_handle, status)?; |
08481a0b DM |
261 | } |
262 | } | |
6ea3a0b7 DM |
263 | Ok(()) |
264 | } | |
265 | ||
03e4753d | 266 | pub fn insert_chunk(&self, chunk: &[u8]) -> Result<(bool, [u8; 32]), Error> { |
128b37fe | 267 | |
7b2b40a8 DM |
268 | // fixme: use Sha512/256 when available |
269 | let mut hasher = sha::Sha256::new(); | |
270 | hasher.update(chunk); | |
271 | ||
272 | let digest = hasher.finish(); | |
273 | ||
08481a0b | 274 | //println!("DIGEST {}", digest_to_hex(&digest)); |
128b37fe | 275 | |
af3e7d75 | 276 | let mut chunk_path = self.chunk_dir.clone(); |
08481a0b | 277 | let prefix = digest_to_prefix(&digest); |
c5d82e5f | 278 | chunk_path.push(&prefix); |
08481a0b | 279 | let digest_str = digest_to_hex(&digest); |
c5d82e5f DM |
280 | chunk_path.push(&digest_str); |
281 | ||
282 | let lock = self.mutex.lock(); | |
283 | ||
284 | if let Ok(metadata) = std::fs::metadata(&chunk_path) { | |
285 | if metadata.is_file() { | |
391a2e43 | 286 | return Ok((true, digest)); |
c5d82e5f | 287 | } else { |
f7dd683b | 288 | bail!("Got unexpected file type on store '{}' for chunk {}", self.name, digest_str); |
c5d82e5f DM |
289 | } |
290 | } | |
128b37fe | 291 | |
c5d82e5f DM |
292 | let mut tmp_path = chunk_path.clone(); |
293 | tmp_path.set_extension("tmp"); | |
294 | let mut f = std::fs::File::create(&tmp_path)?; | |
128b37fe DM |
295 | f.write_all(chunk)?; |
296 | ||
c5d82e5f DM |
297 | if let Err(err) = std::fs::rename(&tmp_path, &chunk_path) { |
298 | if let Err(_) = std::fs::remove_file(&tmp_path) { /* ignore */ } | |
f7dd683b | 299 | bail!("Atomic rename on store '{}' failed for chunk {} - {}", self.name, digest_str, err); |
c5d82e5f DM |
300 | } |
301 | ||
128b37fe DM |
302 | println!("PATH {:?}", chunk_path); |
303 | ||
c5d82e5f DM |
304 | drop(lock); |
305 | ||
391a2e43 | 306 | Ok((false, digest)) |
128b37fe DM |
307 | } |
308 | ||
606ce64b DM |
309 | pub fn relative_path(&self, path: &Path) -> PathBuf { |
310 | ||
311 | let mut full_path = self.base.clone(); | |
312 | full_path.push(path); | |
313 | full_path | |
314 | } | |
315 | ||
3d5c11e5 DM |
316 | pub fn base_path(&self) -> PathBuf { |
317 | self.base.clone() | |
318 | } | |
35cf5daa DM |
319 | } |
320 | ||
321 | ||
322 | #[test] | |
323 | fn test_chunk_store1() { | |
324 | ||
325 | if let Err(_e) = std::fs::remove_dir_all(".testdir") { /* ignore */ } | |
326 | ||
f7dd683b | 327 | let chunk_store = ChunkStore::open("test", ".testdir"); |
35cf5daa DM |
328 | assert!(chunk_store.is_err()); |
329 | ||
03e4753d | 330 | let chunk_store = ChunkStore::create("test", ".testdir").unwrap(); |
391a2e43 DM |
331 | let (exists, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap(); |
332 | assert!(!exists); | |
333 | ||
334 | let (exists, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap(); | |
335 | assert!(exists); | |
128b37fe | 336 | |
35cf5daa | 337 | |
f7dd683b | 338 | let chunk_store = ChunkStore::create("test", ".testdir"); |
35cf5daa DM |
339 | assert!(chunk_store.is_err()); |
340 | ||
341 | ||
342 | } |