]>
Commit | Line | Data |
---|---|---|
35cf5daa DM |
1 | use failure::*; |
2 | use std::path::{Path, PathBuf}; | |
128b37fe | 3 | use std::io::Write; |
1628a4c7 | 4 | use std::time::Duration; |
128b37fe | 5 | |
7b2b40a8 | 6 | use openssl::sha; |
c5d82e5f DM |
7 | use std::sync::Mutex; |
8 | ||
365bb90f | 9 | use std::fs::File; |
45773720 | 10 | use std::os::unix::io::AsRawFd; |
35cf5daa | 11 | |
365bb90f DM |
12 | use crate::tools; |
13 | ||
35cf5daa | 14 | pub struct ChunkStore { |
277fc5a3 | 15 | name: String, // used for error reporting |
35cf5daa DM |
16 | base: PathBuf, |
17 | chunk_dir: PathBuf, | |
c5d82e5f | 18 | mutex: Mutex<bool>, |
eae8aa3a | 19 | _lockfile: File, |
128b37fe DM |
20 | } |
21 | ||
22 | const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef"; | |
23 | ||
08481a0b | 24 | pub fn digest_to_hex(digest: &[u8]) -> String { |
128b37fe | 25 | |
3d5c11e5 | 26 | let mut buf = Vec::<u8>::with_capacity(digest.len()*2); |
128b37fe | 27 | |
3d5c11e5 | 28 | for i in 0..digest.len() { |
128b37fe DM |
29 | buf.push(HEX_CHARS[(digest[i] >> 4) as usize]); |
30 | buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]); | |
31 | } | |
32 | ||
33 | unsafe { String::from_utf8_unchecked(buf) } | |
35cf5daa DM |
34 | } |
35 | ||
08481a0b | 36 | fn digest_to_prefix(digest: &[u8]) -> PathBuf { |
128b37fe | 37 | |
e95950e4 | 38 | let mut buf = Vec::<u8>::with_capacity(2+1+2+1); |
128b37fe DM |
39 | |
40 | buf.push(HEX_CHARS[(digest[0] as usize) >> 4]); | |
41 | buf.push(HEX_CHARS[(digest[0] as usize) &0xf]); | |
128b37fe | 42 | buf.push('/' as u8); |
35cf5daa | 43 | |
e95950e4 | 44 | buf.push(HEX_CHARS[(digest[1] as usize) >> 4]); |
128b37fe | 45 | buf.push(HEX_CHARS[(digest[1] as usize) & 0xf]); |
128b37fe DM |
46 | buf.push('/' as u8); |
47 | ||
48 | let path = unsafe { String::from_utf8_unchecked(buf)}; | |
49 | ||
50 | path.into() | |
35cf5daa DM |
51 | } |
52 | ||
12bb93b3 | 53 | |
35cf5daa DM |
54 | impl ChunkStore { |
55 | ||
45773720 DM |
56 | fn chunk_dir<P: AsRef<Path>>(path: P) -> PathBuf { |
57 | ||
58 | let mut chunk_dir: PathBuf = PathBuf::from(path.as_ref()); | |
59 | chunk_dir.push(".chunks"); | |
60 | ||
61 | chunk_dir | |
62 | } | |
63 | ||
277fc5a3 | 64 | pub fn create<P: Into<PathBuf>>(name: &str, path: P) -> Result<Self, Error> { |
35cf5daa | 65 | |
45773720 DM |
66 | let base: PathBuf = path.into(); |
67 | let chunk_dir = Self::chunk_dir(&base); | |
35cf5daa | 68 | |
2989f6bf | 69 | if let Err(err) = std::fs::create_dir(&base) { |
277fc5a3 | 70 | bail!("unable to create chunk store '{}' at {:?} - {}", name, base, err); |
2989f6bf DM |
71 | } |
72 | ||
73 | if let Err(err) = std::fs::create_dir(&chunk_dir) { | |
277fc5a3 | 74 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, chunk_dir, err); |
2989f6bf | 75 | } |
35cf5daa | 76 | |
e95950e4 DM |
77 | // create 256*256 subdirs |
78 | let mut last_percentage = 0; | |
79 | ||
80 | for i in 0..256 { | |
af3e7d75 | 81 | let mut l1path = chunk_dir.clone(); |
e95950e4 | 82 | l1path.push(format!("{:02x}",i)); |
2989f6bf | 83 | if let Err(err) = std::fs::create_dir(&l1path) { |
277fc5a3 | 84 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, l1path, err); |
2989f6bf | 85 | } |
e95950e4 DM |
86 | for j in 0..256 { |
87 | let mut l2path = l1path.clone(); | |
88 | l2path.push(format!("{:02x}",j)); | |
89 | if let Err(err) = std::fs::create_dir(&l2path) { | |
90 | bail!("unable to create chunk store '{}' subdir {:?} - {}", name, l2path, err); | |
91 | } | |
92 | let percentage = ((i*256+j)*100)/(256*256); | |
93 | if percentage != last_percentage { | |
94 | eprintln!("Percentage done: {}", percentage); | |
95 | last_percentage = percentage; | |
96 | } | |
97 | } | |
35cf5daa DM |
98 | } |
99 | ||
277fc5a3 | 100 | Self::open(name, base) |
35cf5daa DM |
101 | } |
102 | ||
277fc5a3 | 103 | pub fn open<P: Into<PathBuf>>(name: &str, path: P) -> Result<Self, Error> { |
35cf5daa | 104 | |
45773720 DM |
105 | let base: PathBuf = path.into(); |
106 | let chunk_dir = Self::chunk_dir(&base); | |
107 | ||
ce55dbbc | 108 | if let Err(err) = std::fs::metadata(&chunk_dir) { |
277fc5a3 | 109 | bail!("unable to open chunk store '{}' at {:?} - {}", name, chunk_dir, err); |
ce55dbbc | 110 | } |
45773720 DM |
111 | |
112 | let mut lockfile_path = base.clone(); | |
113 | lockfile_path.push(".lock"); | |
114 | ||
5ba69689 | 115 | // make sure only one process/thread/task can use it |
1628a4c7 WB |
116 | let lockfile = tools::open_file_locked( |
117 | lockfile_path, Duration::from_secs(10))?; | |
35cf5daa | 118 | |
b8d4766a | 119 | Ok(ChunkStore { |
277fc5a3 | 120 | name: name.to_owned(), |
b8d4766a DM |
121 | base, |
122 | chunk_dir, | |
03e4753d | 123 | _lockfile: lockfile, |
b8d4766a DM |
124 | mutex: Mutex::new(false) |
125 | }) | |
35cf5daa DM |
126 | } |
127 | ||
03e4753d | 128 | pub fn touch_chunk(&self, digest:&[u8]) -> Result<(), Error> { |
3d5c11e5 | 129 | |
08481a0b | 130 | // fixme: nix::sys::stat::utimensat |
3d5c11e5 | 131 | let mut chunk_path = self.chunk_dir.clone(); |
08481a0b | 132 | let prefix = digest_to_prefix(&digest); |
3d5c11e5 | 133 | chunk_path.push(&prefix); |
08481a0b | 134 | let digest_str = digest_to_hex(&digest); |
3d5c11e5 DM |
135 | chunk_path.push(&digest_str); |
136 | ||
137 | std::fs::metadata(&chunk_path)?; | |
138 | Ok(()) | |
139 | } | |
140 | ||
277fc5a3 | 141 | fn sweep_old_files(&self, handle: &mut nix::dir::Dir) -> Result<(), Error> { |
08481a0b DM |
142 | |
143 | let rawfd = handle.as_raw_fd(); | |
144 | ||
145 | let now = unsafe { libc::time(std::ptr::null_mut()) }; | |
146 | ||
147 | for entry in handle.iter() { | |
eae8aa3a DM |
148 | let entry = match entry { |
149 | Ok(entry) => entry, | |
150 | Err(_) => continue /* ignore */, | |
151 | }; | |
152 | let file_type = match entry.file_type() { | |
153 | Some(file_type) => file_type, | |
277fc5a3 | 154 | None => bail!("unsupported file system type on chunk store '{}'", self.name), |
eae8aa3a DM |
155 | }; |
156 | if file_type != nix::dir::Type::File { continue; } | |
157 | ||
158 | let filename = entry.file_name(); | |
159 | if let Ok(stat) = nix::sys::stat::fstatat(rawfd, filename, nix::fcntl::AtFlags::AT_SYMLINK_NOFOLLOW) { | |
160 | let age = now - stat.st_atime; | |
e95950e4 | 161 | //println!("FOUND {} {:?}", age/(3600*24), filename); |
eae8aa3a DM |
162 | if age/(3600*24) >= 2 { |
163 | println!("UNLINK {} {:?}", age/(3600*24), filename); | |
277fc5a3 DM |
164 | let res = unsafe { libc::unlinkat(rawfd, filename.as_ptr(), 0) }; |
165 | if res != 0 { | |
166 | let err = nix::Error::last(); | |
167 | bail!("unlink chunk {:?} failed on store '{}' - {}", filename, self.name, err); | |
168 | } | |
08481a0b | 169 | } |
eae8aa3a DM |
170 | } |
171 | } | |
277fc5a3 | 172 | Ok(()) |
08481a0b DM |
173 | } |
174 | ||
03e4753d | 175 | pub fn sweep_used_chunks(&self) -> Result<(), Error> { |
6ea3a0b7 | 176 | |
1c43c56b DM |
177 | use nix::fcntl::OFlag; |
178 | use nix::sys::stat::Mode; | |
179 | use nix::dir::Dir; | |
180 | ||
eae8aa3a | 181 | let base_handle = match Dir::open( |
1c43c56b | 182 | &self.chunk_dir, OFlag::O_RDONLY, Mode::empty()) { |
2bf5f6b2 | 183 | Ok(h) => h, |
277fc5a3 DM |
184 | Err(err) => bail!("unable to open store '{}' chunk dir {:?} - {}", |
185 | self.name, self.chunk_dir, err), | |
2bf5f6b2 DM |
186 | }; |
187 | ||
188 | let base_fd = base_handle.as_raw_fd(); | |
189 | ||
e95950e4 DM |
190 | for i in 0..256 { |
191 | let l1name = PathBuf::from(format!("{:02x}", i)); | |
2bf5f6b2 | 192 | let mut l1_handle = match nix::dir::Dir::openat( |
1c43c56b | 193 | base_fd, &l1name, OFlag::O_RDONLY, Mode::empty()) { |
2bf5f6b2 | 194 | Ok(h) => h, |
277fc5a3 DM |
195 | Err(err) => bail!("unable to open store '{}' dir {:?}/{:?} - {}", |
196 | self.name, self.chunk_dir, l1name, err), | |
2bf5f6b2 DM |
197 | }; |
198 | ||
199 | let l1_fd = l1_handle.as_raw_fd(); | |
200 | ||
e95950e4 DM |
201 | for j in 0..256 { |
202 | let l2name = PathBuf::from(format!("{:02x}", j)); | |
6c20a13d | 203 | |
e95950e4 | 204 | println!("SCAN {:?} {:?}", l1name, l2name); |
6c20a13d DM |
205 | |
206 | let mut l2_handle = match Dir::openat( | |
e95950e4 | 207 | l1_fd, &l2name, OFlag::O_RDONLY, Mode::empty()) { |
6c20a13d DM |
208 | Ok(h) => h, |
209 | Err(err) => bail!( | |
277fc5a3 DM |
210 | "unable to open store '{}' dir {:?}/{:?}/{:?} - {}", |
211 | self.name, self.chunk_dir, l1name, l2name, err), | |
6c20a13d | 212 | }; |
277fc5a3 | 213 | self.sweep_old_files(&mut l2_handle)?; |
08481a0b DM |
214 | } |
215 | } | |
6ea3a0b7 DM |
216 | Ok(()) |
217 | } | |
218 | ||
03e4753d | 219 | pub fn insert_chunk(&self, chunk: &[u8]) -> Result<(bool, [u8; 32]), Error> { |
128b37fe | 220 | |
7b2b40a8 DM |
221 | // fixme: use Sha512/256 when available |
222 | let mut hasher = sha::Sha256::new(); | |
223 | hasher.update(chunk); | |
224 | ||
225 | let digest = hasher.finish(); | |
226 | ||
08481a0b | 227 | //println!("DIGEST {}", digest_to_hex(&digest)); |
128b37fe | 228 | |
af3e7d75 | 229 | let mut chunk_path = self.chunk_dir.clone(); |
08481a0b | 230 | let prefix = digest_to_prefix(&digest); |
c5d82e5f | 231 | chunk_path.push(&prefix); |
08481a0b | 232 | let digest_str = digest_to_hex(&digest); |
c5d82e5f DM |
233 | chunk_path.push(&digest_str); |
234 | ||
235 | let lock = self.mutex.lock(); | |
236 | ||
237 | if let Ok(metadata) = std::fs::metadata(&chunk_path) { | |
238 | if metadata.is_file() { | |
391a2e43 | 239 | return Ok((true, digest)); |
c5d82e5f | 240 | } else { |
f7dd683b | 241 | bail!("Got unexpected file type on store '{}' for chunk {}", self.name, digest_str); |
c5d82e5f DM |
242 | } |
243 | } | |
128b37fe | 244 | |
c5d82e5f DM |
245 | let mut tmp_path = chunk_path.clone(); |
246 | tmp_path.set_extension("tmp"); | |
247 | let mut f = std::fs::File::create(&tmp_path)?; | |
128b37fe DM |
248 | f.write_all(chunk)?; |
249 | ||
c5d82e5f DM |
250 | if let Err(err) = std::fs::rename(&tmp_path, &chunk_path) { |
251 | if let Err(_) = std::fs::remove_file(&tmp_path) { /* ignore */ } | |
f7dd683b | 252 | bail!("Atomic rename on store '{}' failed for chunk {} - {}", self.name, digest_str, err); |
c5d82e5f DM |
253 | } |
254 | ||
128b37fe DM |
255 | println!("PATH {:?}", chunk_path); |
256 | ||
c5d82e5f DM |
257 | drop(lock); |
258 | ||
391a2e43 | 259 | Ok((false, digest)) |
128b37fe DM |
260 | } |
261 | ||
606ce64b DM |
262 | pub fn relative_path(&self, path: &Path) -> PathBuf { |
263 | ||
264 | let mut full_path = self.base.clone(); | |
265 | full_path.push(path); | |
266 | full_path | |
267 | } | |
268 | ||
3d5c11e5 DM |
269 | pub fn base_path(&self) -> PathBuf { |
270 | self.base.clone() | |
271 | } | |
35cf5daa DM |
272 | } |
273 | ||
274 | ||
275 | #[test] | |
276 | fn test_chunk_store1() { | |
277 | ||
278 | if let Err(_e) = std::fs::remove_dir_all(".testdir") { /* ignore */ } | |
279 | ||
f7dd683b | 280 | let chunk_store = ChunkStore::open("test", ".testdir"); |
35cf5daa DM |
281 | assert!(chunk_store.is_err()); |
282 | ||
03e4753d | 283 | let chunk_store = ChunkStore::create("test", ".testdir").unwrap(); |
391a2e43 DM |
284 | let (exists, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap(); |
285 | assert!(!exists); | |
286 | ||
287 | let (exists, _) = chunk_store.insert_chunk(&[0u8, 1u8]).unwrap(); | |
288 | assert!(exists); | |
128b37fe | 289 | |
35cf5daa | 290 | |
f7dd683b | 291 | let chunk_store = ChunkStore::create("test", ".testdir"); |
35cf5daa DM |
292 | assert!(chunk_store.is_err()); |
293 | ||
294 | ||
295 | } |