2 use std
::path
::{Path, PathBuf}
;
5 use crypto
::digest
::Digest
;
6 use crypto
::sha2
::Sha512Trunc256
;
9 use std
::fs
::{File, OpenOptions}
;
10 use nix
::fcntl
::{flock, FlockArg}
;
11 use std
::os
::unix
::io
::AsRawFd
;
13 pub struct ChunkStore
{
16 hasher
: Sha512Trunc256
,
21 const HEX_CHARS
: &'
static [u8; 16] = b
"0123456789abcdef";
23 pub fn digest_to_hex(digest
: &[u8]) -> String
{
25 let mut buf
= Vec
::<u8>::with_capacity(digest
.len()*2);
27 for i
in 0..digest
.len() {
28 buf
.push(HEX_CHARS
[(digest
[i
] >> 4) as usize]);
29 buf
.push(HEX_CHARS
[(digest
[i
] & 0xf) as usize]);
32 unsafe { String::from_utf8_unchecked(buf) }
35 fn digest_to_prefix(digest
: &[u8]) -> PathBuf
{
37 let mut buf
= Vec
::<u8>::with_capacity(3+1+2+1);
39 buf
.push(HEX_CHARS
[(digest
[0] as usize) >> 4]);
40 buf
.push(HEX_CHARS
[(digest
[0] as usize) &0xf]);
41 buf
.push(HEX_CHARS
[(digest
[1] as usize) >> 4]);
44 buf
.push(HEX_CHARS
[(digest
[1] as usize) & 0xf]);
45 buf
.push(HEX_CHARS
[(digest
[2] as usize) >> 4]);
48 let path
= unsafe { String::from_utf8_unchecked(buf)}
;
53 fn lock_file
<P
: AsRef
<Path
>>(filename
: P
, timeout
: usize) -> Result
<File
, Error
> {
55 let path
= filename
.as_ref();
56 let lockfile
= match OpenOptions
::new()
61 Err(err
) => bail
!("Unable to open lock {:?} - {}",
65 let fd
= lockfile
.as_raw_fd();
67 let now
= std
::time
::SystemTime
::now();
68 let mut print_msg
= true;
70 match flock(fd
, FlockArg
::LockExclusiveNonblock
) {
75 eprintln
!("trying to aquire lock...");
82 if elapsed
.as_secs() >= (timeout
as u64) {
83 bail
!("unable to aquire lock {:?} - got timeout", path
);
87 bail
!("unable to aquire lock {:?} - clock problems - {}", path
, err
);
90 std
::thread
::sleep(std
::time
::Duration
::from_millis(100));
97 fn chunk_dir
<P
: AsRef
<Path
>>(path
: P
) -> PathBuf
{
99 let mut chunk_dir
: PathBuf
= PathBuf
::from(path
.as_ref());
100 chunk_dir
.push(".chunks");
105 pub fn create
<P
: Into
<PathBuf
>>(path
: P
) -> Result
<Self, Error
> {
107 let base
: PathBuf
= path
.into();
108 let chunk_dir
= Self::chunk_dir(&base
);
110 if let Err(err
) = std
::fs
::create_dir(&base
) {
111 bail
!("unable to create chunk store {:?} - {}", base
, err
);
114 if let Err(err
) = std
::fs
::create_dir(&chunk_dir
) {
115 bail
!("unable to create chunk store subdir {:?} - {}", chunk_dir
, err
);
118 // create 4096 subdir
120 let mut l1path
= chunk_dir
.clone();
121 l1path
.push(format
!("{:03x}",i
));
122 if let Err(err
) = std
::fs
::create_dir(&l1path
) {
123 bail
!("unable to create chunk subdir {:?} - {}", l1path
, err
);
130 pub fn open
<P
: Into
<PathBuf
>>(path
: P
) -> Result
<Self, Error
> {
132 let base
: PathBuf
= path
.into();
133 let chunk_dir
= Self::chunk_dir(&base
);
135 if let Err(err
) = std
::fs
::metadata(&chunk_dir
) {
136 bail
!("unable to open chunk store {:?} - {}", chunk_dir
, err
);
139 let mut lockfile_path
= base
.clone();
140 lockfile_path
.push(".lock");
142 // make sure only one process/thread/task can use it
143 let lockfile
= lock_file(lockfile_path
, 10)?
;
148 hasher
: Sha512Trunc256
::new(),
150 mutex
: Mutex
::new(false)
154 pub fn touch_chunk(&mut self, digest
:&[u8]) -> Result
<(), Error
> {
156 // fixme: nix::sys::stat::utimensat
157 let mut chunk_path
= self.chunk_dir
.clone();
158 let prefix
= digest_to_prefix(&digest
);
159 chunk_path
.push(&prefix
);
160 let digest_str
= digest_to_hex(&digest
);
161 chunk_path
.push(&digest_str
);
163 std
::fs
::metadata(&chunk_path
)?
;
167 fn sweep_old_files(&self, dir
: &Path
) {
169 let mut handle
= match nix
::dir
::Dir
::open(
170 dir
, nix
::fcntl
::OFlag
::O_RDONLY
, nix
::sys
::stat
::Mode
::empty()) {
175 let rawfd
= handle
.as_raw_fd();
177 let now
= unsafe { libc::time(std::ptr::null_mut()) }
;
179 for entry
in handle
.iter() {
182 if let Some(file_type
) = entry
.file_type() {
183 if file_type
== nix
::dir
::Type
::File
{
184 let filename
= entry
.file_name();
185 if let Ok(stat
) = nix
::sys
::stat
::fstatat(rawfd
, filename
, nix
::fcntl
::AtFlags
::AT_SYMLINK_NOFOLLOW
) {
186 let age
= now
- stat
.st_atime
;
187 println
!("FOUND {} {:?}", age
/(3600*24), filename
);
188 if age
/(3600*24) >= 2 {
189 println
!("UNLINK {} {:?}", age
/(3600*24), filename
);
190 unsafe { libc::unlinkat(rawfd, filename.as_ptr(), 0); }
204 pub fn sweep_used_chunks(&mut self) -> Result
<(), Error
> {
207 let mut l1path
= self.chunk_dir
.clone();
208 l1path
.push(format
!("{:03x}", i
));
210 let mut l2path
= l1path
.clone();
211 l2path
.push(format
!("{:02x}", j
));
212 self.sweep_old_files(&l2path
);
219 pub fn insert_chunk(&mut self, chunk
: &[u8]) -> Result
<(bool
, [u8; 32]), Error
> {
222 self.hasher
.input(chunk
);
224 let mut digest
= [0u8; 32];
225 self.hasher
.result(&mut digest
);
226 //println!("DIGEST {}", digest_to_hex(&digest));
228 let mut chunk_path
= self.chunk_dir
.clone();
229 let prefix
= digest_to_prefix(&digest
);
230 chunk_path
.push(&prefix
);
231 let digest_str
= digest_to_hex(&digest
);
232 chunk_path
.push(&digest_str
);
234 let lock
= self.mutex
.lock();
236 if let Ok(metadata
) = std
::fs
::metadata(&chunk_path
) {
237 if metadata
.is_file() {
238 return Ok((true, digest
));
240 bail
!("Got unexpected file type for chunk {}", digest_str
);
244 let mut chunk_dir
= self.chunk_dir
.clone();
245 chunk_dir
.push(&prefix
);
247 if let Err(_
) = std
::fs
::create_dir(&chunk_dir
) { /* ignore */ }
249 let mut tmp_path
= chunk_path
.clone();
250 tmp_path
.set_extension("tmp");
251 let mut f
= std
::fs
::File
::create(&tmp_path
)?
;
254 if let Err(err
) = std
::fs
::rename(&tmp_path
, &chunk_path
) {
255 if let Err(_
) = std
::fs
::remove_file(&tmp_path
) { /* ignore */ }
256 bail
!("Atomic rename failed for chunk {} - {}", digest_str
, err
);
259 println
!("PATH {:?}", chunk_path
);
266 pub fn relative_path(&self, path
: &Path
) -> PathBuf
{
268 let mut full_path
= self.base
.clone();
269 full_path
.push(path
);
273 pub fn base_path(&self) -> PathBuf
{
281 fn test_chunk_store1() {
283 if let Err(_e
) = std
::fs
::remove_dir_all(".testdir") { /* ignore */ }
285 let chunk_store
= ChunkStore
::open(".testdir");
286 assert
!(chunk_store
.is_err());
288 let mut chunk_store
= ChunkStore
::create(".testdir").unwrap();
289 let (exists
, _
) = chunk_store
.insert_chunk(&[0u8, 1u8]).unwrap();
292 let (exists
, _
) = chunk_store
.insert_chunk(&[0u8, 1u8]).unwrap();
296 let chunk_store
= ChunkStore
::create(".testdir");
297 assert
!(chunk_store
.is_err());