]>
Commit | Line | Data |
---|---|---|
35cf5daa DM |
1 | use failure::*; |
2 | use std::path::{Path, PathBuf}; | |
128b37fe | 3 | use std::io::Write; |
1628a4c7 | 4 | use std::time::Duration; |
128b37fe | 5 | |
7b2b40a8 | 6 | use openssl::sha; |
c5d82e5f DM |
7 | use std::sync::Mutex; |
8 | ||
365bb90f | 9 | use std::fs::File; |
45773720 | 10 | use std::os::unix::io::AsRawFd; |
35cf5daa | 11 | |
365bb90f DM |
12 | use crate::tools; |
13 | ||
35cf5daa | 14 | pub struct ChunkStore { |
277fc5a3 | 15 | name: String, // used for error reporting |
35cf5daa DM |
16 | base: PathBuf, |
17 | chunk_dir: PathBuf, | |
c5d82e5f | 18 | mutex: Mutex<bool>, |
eae8aa3a | 19 | _lockfile: File, |
128b37fe DM |
20 | } |
21 | ||
22 | const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef"; | |
23 | ||
176e4af9 DM |
24 | // TODO: what about sysctl setting vm.vfs_cache_pressure (0 - 100) ? |
25 | ||
08481a0b | 26 | pub fn digest_to_hex(digest: &[u8]) -> String { |
128b37fe | 27 | |
3d5c11e5 | 28 | let mut buf = Vec::<u8>::with_capacity(digest.len()*2); |
128b37fe | 29 | |
3d5c11e5 | 30 | for i in 0..digest.len() { |
128b37fe DM |
31 | buf.push(HEX_CHARS[(digest[i] >> 4) as usize]); |
32 | buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]); | |
33 | } | |
34 | ||
35 | unsafe { String::from_utf8_unchecked(buf) } | |
35cf5daa DM |
36 | } |
37 | ||
08481a0b | 38 | fn digest_to_prefix(digest: &[u8]) -> PathBuf { |
128b37fe | 39 | |
e95950e4 | 40 | let mut buf = Vec::<u8>::with_capacity(2+1+2+1); |
128b37fe DM |
41 | |
42 | buf.push(HEX_CHARS[(digest[0] as usize) >> 4]); | |
43 | buf.push(HEX_CHARS[(digest[0] as usize) &0xf]); | |
128b37fe | 44 | buf.push('/' as u8); |
35cf5daa | 45 | |
e95950e4 | 46 | buf.push(HEX_CHARS[(digest[1] as usize) >> 4]); |
128b37fe | 47 | buf.push(HEX_CHARS[(digest[1] as usize) & 0xf]); |
128b37fe DM |
48 | buf.push('/' as u8); |
49 | ||
50 | let path = unsafe { String::from_utf8_unchecked(buf)}; | |
51 | ||
52 | path.into() | |
35cf5daa DM |
53 | } |
54 | ||
12bb93b3 | 55 | |
35cf5daa DM |
56 | impl ChunkStore { |
57 | ||
45773720 DM |
58 | fn chunk_dir<P: AsRef<Path>>(path: P) -> PathBuf { |
59 | ||
60 | let mut chunk_dir: PathBuf = PathBuf::from(path.as_ref()); | |
61 | chunk_dir.push(".chunks"); | |
62 | ||
63 | chunk_dir | |
64 | } | |
65 | ||
277fc5a3 | 66 | pub fn create<P: Into<PathBuf>>(name: &str, path: P) -> Result<Self, Error> { |
35cf5daa | 67 | |
45773720 DM |
68 | let base: PathBuf = path.into(); |
69 | let chunk_dir = Self::chunk_dir(&base); | |
35cf5daa | 70 | |
2989f6bf | 71 | if let Err(err) = std::fs::create_dir(&base) { |
277fc5a3 | 72 | bail!("unable to create chunk store '{}' at {:?} - {}", name, base, err); |
2989f6bf DM |
73 | } |
74 | ||
75 | if let Err(err) = std::fs::create_dir(&chunk_dir) { | |
277fc5a3 | 76 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, chunk_dir, err); |
2989f6bf | 77 | } |
35cf5daa | 78 | |
e95950e4 DM |
79 | // create 256*256 subdirs |
80 | let mut last_percentage = 0; | |
81 | ||
82 | for i in 0..256 { | |
af3e7d75 | 83 | let mut l1path = chunk_dir.clone(); |
e95950e4 | 84 | l1path.push(format!("{:02x}",i)); |
2989f6bf | 85 | if let Err(err) = std::fs::create_dir(&l1path) { |
277fc5a3 | 86 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, l1path, err); |
2989f6bf | 87 | } |
e95950e4 DM |
88 | for j in 0..256 { |
89 | let mut l2path = l1path.clone(); | |
90 | l2path.push(format!("{:02x}",j)); | |
91 | if let Err(err) = std::fs::create_dir(&l2path) { | |
92 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, l2path, err); | |
93 | } | |
94 | let percentage = ((i*256+j)*100)/(256*256); | |
95 | if percentage != last_percentage { | |
96 | eprintln!("Percentage done: {}", percentage); | |
97 | last_percentage = percentage; | |
98 | } | |
99 | } | |
35cf5daa DM |
100 | } |
101 | ||
277fc5a3 | 102 | Self::open(name, base) |
35cf5daa DM |
103 | } |
104 | ||
277fc5a3 | 105 | pub fn open<P: Into<PathBuf>>(name: &str, path: P) -> Result<Self, Error> { |
35cf5daa | 106 | |
45773720 DM |
107 | let base: PathBuf = path.into(); |
108 | let chunk_dir = Self::chunk_dir(&base); | |
109 | ||
ce55dbbc | 110 | if let Err(err) = std::fs::metadata(&chunk_dir) { |
277fc5a3 | 111 | bail!("unable to open chunk store '{}' at {:?} - {}", name, chunk_dir, err); |
ce55dbbc | 112 | } |
45773720 DM |
113 | |
114 | let mut lockfile_path = base.clone(); | |
115 | lockfile_path.push(".lock"); | |
116 | ||
5ba69689 | 117 | // make sure only one process/thread/task can use it |
1628a4c7 WB |
118 | let lockfile = tools::open_file_locked( |
119 | lockfile_path, Duration::from_secs(10))?; | |
35cf5daa | 120 | |
b8d4766a | 121 | Ok(ChunkStore { |
277fc5a3 | 122 | name: name.to_owned(), |
b8d4766a DM |
123 | base, |
124 | chunk_dir, | |
03e4753d | 125 | _lockfile: lockfile, |
b8d4766a DM |
126 | mutex: Mutex::new(false) |
127 | }) | |
35cf5daa DM |
128 | } |
129 | ||
03e4753d | 130 | pub fn touch_chunk(&self, digest:&[u8]) -> Result<(), Error> { |
3d5c11e5 | 131 | |
08481a0b | 132 | // fixme: nix::sys::stat::utimensat |
3d5c11e5 | 133 | let mut chunk_path = self.chunk_dir.clone(); |
08481a0b | 134 | let prefix = digest_to_prefix(&digest); |
3d5c11e5 | 135 | chunk_path.push(&prefix); |
08481a0b | 136 | let digest_str = digest_to_hex(&digest); |
3d5c11e5 DM |
137 | chunk_path.push(&digest_str); |
138 | ||
139 | std::fs::metadata(&chunk_path)?; | |
140 | Ok(()) | |
141 | } | |
142 | ||
277fc5a3 | 143 | fn sweep_old_files(&self, handle: &mut nix::dir::Dir) -> Result<(), Error> { |
08481a0b DM |
144 | |
145 | let rawfd = handle.as_raw_fd(); | |
146 | ||
147 | let now = unsafe { libc::time(std::ptr::null_mut()) }; | |
148 | ||
149 | for entry in handle.iter() { | |
eae8aa3a DM |
150 | let entry = match entry { |
151 | Ok(entry) => entry, | |
152 | Err(_) => continue /* ignore */, | |
153 | }; | |
154 | let file_type = match entry.file_type() { | |
155 | Some(file_type) => file_type, | |
277fc5a3 | 156 | None => bail!("unsupported file system type on chunk store '{}'", self.name), |
eae8aa3a DM |
157 | }; |
158 | if file_type != nix::dir::Type::File { continue; } | |
159 | ||
160 | let filename = entry.file_name(); | |
161 | if let Ok(stat) = nix::sys::stat::fstatat(rawfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) { | |
162 | let age = now - stat.st_atime; | |
e95950e4 | 163 | //println!("FOUND {} {:?}", age/(3600*24), filename); |
eae8aa3a DM |
164 | if age/(3600*24) >= 2 { |
165 | println!("UNLINK {} {:?}", age/(3600*24), filename); | |
277fc5a3 DM |
166 | let res = unsafe { libc::unlinkat(rawfd, filename.as_ptr(), 0) }; |
167 | if res != 0 { | |
168 | let err = nix::Error::last(); | |
169 | bail!("unlink chunk {:?} failed on store '{}' - {}", filename, self.name, err); | |
170 | } | |
08481a0b | 171 | } |
eae8aa3a DM |
172 | } |
173 | } | |
277fc5a3 | 174 | Ok(()) |
08481a0b DM |
175 | } |
176 | ||
03e4753d | 177 | pub fn sweep_used_chunks(&self) -> Result<(), Error> { |
6ea3a0b7 | 178 | |
1c43c56b DM |
179 | use nix::fcntl::OFlag; |
180 | use nix::sys::stat::Mode; | |
181 | use nix::dir::Dir; | |
182 | ||
eae8aa3a | 183 | let base_handle = match Dir::open( |
1c43c56b | 184 | &self.chunk_dir, OFlag::O_RDONLY, Mode::empty()) { |
2bf5f6b2 | 185 | Ok(h) => h, |
277fc5a3 DM |
186 | Err(err) => bail!("unable to open store '{}' chunk dir {:?} - {}", |
187 | self.name, self.chunk_dir, err), | |
2bf5f6b2 DM |
188 | }; |
189 | ||
190 | let base_fd = base_handle.as_raw_fd(); | |
191 | ||
176e4af9 DM |
192 | let mut last_percentage = 0; |
193 | ||
e95950e4 DM |
194 | for i in 0..256 { |
195 | let l1name = PathBuf::from(format!("{:02x}", i)); | |
2bf5f6b2 | 196 | let mut l1_handle = match nix::dir::Dir::openat( |
1c43c56b | 197 | base_fd, &l1name, OFlag::O_RDONLY, Mode::empty()) { |
2bf5f6b2 | 198 | Ok(h) => h, |
277fc5a3 DM |
199 | Err(err) => bail!("unable to open store '{}' dir {:?}/{:?} - {}", |
200 | self.name, self.chunk_dir, l1name, err), | |
2bf5f6b2 DM |
201 | }; |
202 | ||
203 | let l1_fd = l1_handle.as_raw_fd(); | |
204 | ||
e95950e4 DM |
205 | for j in 0..256 { |
206 | let l2name = PathBuf::from(format!("{:02x}", j)); | |
6c20a13d | 207 | |
176e4af9 DM |
208 | let percentage = ((i*256+j)*100)/(256*256); |
209 | if percentage != last_percentage { | |
210 | eprintln!("Percentage done: {}", percentage); | |
211 | last_percentage = percentage; | |
212 | } | |
213 | //println!("SCAN {:?} {:?}", l1name, l2name); | |
6c20a13d DM |
214 | |
215 | let mut l2_handle = match Dir::openat( | |
e95950e4 | 216 | l1_fd, &l2name, OFlag::O_RDONLY, Mode::empty()) { |
6c20a13d DM |
217 | Ok(h) => h, |
218 | Err(err) => bail!( | |
277fc5a3 DM |
219 | "unable to open store '{}' dir {:?}/{:?}/{:?} - {}", |
220 | self.name, self.chunk_dir, l1name, l2name, err), | |
6c20a13d | 221 | }; |
277fc5a3 | 222 | self.sweep_old_files(&mut l2_handle)?; |
08481a0b DM |
223 | } |
224 | } | |
6ea3a0b7 DM |
225 | Ok(()) |
226 | } | |
227 | ||
03e4753d | 228 | pub fn insert_chunk(&self, chunk: &[u8]) -> Result<(bool, [u8; 32]), Error> { |
128b37fe | 229 | |
7b2b40a8 DM |
230 | // fixme: use Sha512/256 when available |
231 | let mut hasher = sha::Sha256::new(); | |
232 | hasher.update(chunk); | |
233 | ||
234 | let digest = hasher.finish(); | |
235 | ||
08481a0b | 236 | //println!("DIGEST {}", digest_to_hex(&digest)); |
128b37fe | 237 | |
af3e7d75 | 238 | let mut chunk_path = self.chunk_dir.clone(); |
08481a0b | 239 | let prefix = digest_to_prefix(&digest); |
c5d82e5f | 240 | chunk_path.push(&prefix); |
08481a0b | 241 | let digest_str = digest_to_hex(&digest); |
c5d82e5f DM |
242 | chunk_path.push(&digest_str); |
243 | ||
244 | let lock = self.mutex.lock(); | |
245 | ||
246 | if let Ok(metadata) = std::fs::metadata(&chunk_path) { | |
247 | if metadata.is_file() { | |
391a2e43 | 248 | return Ok((true, digest)); |
c5d82e5f | 249 | } else { |
f7dd683b | 250 | bail!("Got unexpected file type on store '{}' for chunk {}", self.name, digest_str); |
c5d82e5f DM |
251 | } |
252 | } | |
128b37fe | 253 | |
c5d82e5f DM |
254 | let mut tmp_path = chunk_path.clone(); |
255 | tmp_path.set_extension("tmp"); | |
256 | let mut f = std::fs::File::create(&tmp_path)?; | |
128b37fe DM |
257 | f.write_all(chunk)?; |
258 | ||
c5d82e5f DM |
259 | if let Err(err) = std::fs::rename(&tmp_path, &chunk_path) { |
260 | if let Err(_) = std::fs::remove_file(&tmp_path) { /* ignore */ } | |
f7dd683b | 261 | bail!("Atomic rename on store '{}' failed for chunk {} - {}", self.name, digest_str, err); |
c5d82e5f DM |
262 | } |
263 | ||
128b37fe DM |
264 | println!("PATH {:?}", chunk_path); |
265 | ||
c5d82e5f DM |
266 | drop(lock); |
267 | ||
391a2e43 | 268 | Ok((false, digest)) |
128b37fe DM |
269 | } |
270 | ||
606ce64b DM |
271 | pub fn relative_path(&self, path: &Path) -> PathBuf { |
272 | ||
273 | let mut full_path = self.base.clone(); | |
274 | full_path.push(path); | |
275 | full_path | |
276 | } | |
277 | ||
3d5c11e5 DM |
278 | pub fn base_path(&self) -> PathBuf { |
279 | self.base.clone() | |
280 | } | |
35cf5daa DM |
281 | } |
282 | ||
283 | ||
284 | #[test] | |
285 | fn test_chunk_store1() { | |
286 | ||
287 | if let Err(_e) = std::fs::remove_dir_all(".testdir") { /* ignore */ } | |
288 | ||
f7dd683b | 289 | let chunk_store = ChunkStore::open("test", ".testdir"); |
35cf5daa DM |
290 | assert!(chunk_store.is_err()); |
291 | ||
03e4753d | 292 | let chunk_store = ChunkStore::create("test", ".testdir").unwrap(); |
391a2e43 DM |
293 | let (exists, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap(); |
294 | assert!(!exists); | |
295 | ||
296 | let (exists, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap(); | |
297 | assert!(exists); | |
128b37fe | 298 | |
35cf5daa | 299 | |
f7dd683b | 300 | let chunk_store = ChunkStore::create("test", ".testdir"); |
35cf5daa DM |
301 | assert!(chunk_store.is_err()); |
302 | ||
303 | ||
304 | } |