2 use std
::path
::{Path, PathBuf}
;
5 use crypto
::digest
::Digest
;
6 use crypto
::sha2
::Sha512Trunc256
;
9 use std
::fs
::{File, OpenOptions}
;
10 use nix
::fcntl
::{flock, FlockArg}
;
11 use std
::os
::unix
::io
::AsRawFd
;
13 pub struct ChunkStore
{
16 hasher
: Sha512Trunc256
,
21 const HEX_CHARS
: &'
static [u8; 16] = b
"0123456789abcdef";
23 fn u256_to_hex(digest
: &[u8; 32]) -> String
{
25 let mut buf
= Vec
::<u8>::with_capacity(64);
28 buf
.push(HEX_CHARS
[(digest
[i
] >> 4) as usize]);
29 buf
.push(HEX_CHARS
[(digest
[i
] & 0xf) as usize]);
32 unsafe { String::from_utf8_unchecked(buf) }
35 fn u256_to_prefix(digest
: &[u8; 32]) -> PathBuf
{
37 let mut buf
= Vec
::<u8>::with_capacity(3+1+2+1);
39 buf
.push(HEX_CHARS
[(digest
[0] as usize) >> 4]);
40 buf
.push(HEX_CHARS
[(digest
[0] as usize) &0xf]);
41 buf
.push(HEX_CHARS
[(digest
[1] as usize) >> 4]);
44 buf
.push(HEX_CHARS
[(digest
[1] as usize) & 0xf]);
45 buf
.push(HEX_CHARS
[(digest
[2] as usize) >> 4]);
48 let path
= unsafe { String::from_utf8_unchecked(buf)}
;
53 fn lock_file
<P
: AsRef
<Path
>>(filename
: P
, timeout
: usize) -> Result
<File
, Error
> {
55 let path
= filename
.as_ref();
56 let lockfile
= match OpenOptions
::new()
61 Err(err
) => bail
!("Unable to open lock {:?} - {}",
65 let fd
= lockfile
.as_raw_fd();
67 let now
= std
::time
::SystemTime
::now();
68 let mut print_msg
= true;
70 match flock(fd
, FlockArg
::LockExclusiveNonblock
) {
75 eprintln
!("trying to aquire lock...");
82 if elapsed
.as_secs() >= (timeout
as u64) {
83 bail
!("unable to aquire lock {:?} - got timeout", path
);
87 bail
!("unable to aquire lock {:?} - clock problems - {}", path
, err
);
90 std
::thread
::sleep(std
::time
::Duration
::from_millis(100));
97 fn chunk_dir
<P
: AsRef
<Path
>>(path
: P
) -> PathBuf
{
99 let mut chunk_dir
: PathBuf
= PathBuf
::from(path
.as_ref());
100 chunk_dir
.push(".chunks");
105 pub fn create
<P
: Into
<PathBuf
>>(path
: P
) -> Result
<Self, Error
> {
107 let base
: PathBuf
= path
.into();
108 let chunk_dir
= Self::chunk_dir(&base
);
110 if let Err(err
) = std
::fs
::create_dir(&base
) {
111 bail
!("unable to create chunk store {:?} - {}", base
, err
);
114 if let Err(err
) = std
::fs
::create_dir(&chunk_dir
) {
115 bail
!("unable to create chunk store subdir {:?} - {}", chunk_dir
, err
);
118 // create 4096 subdir
120 let mut l1path
= base
.clone();
121 l1path
.push(format
!("{:03x}",i
));
122 if let Err(err
) = std
::fs
::create_dir(&l1path
) {
123 bail
!("unable to create chunk subdir {:?} - {}", l1path
, err
);
130 pub fn open
<P
: Into
<PathBuf
>>(path
: P
) -> Result
<Self, Error
> {
132 let base
: PathBuf
= path
.into();
133 let chunk_dir
= Self::chunk_dir(&base
);
135 if let Err(err
) = std
::fs
::metadata(&chunk_dir
) {
136 bail
!("unable to open chunk store {:?} - {}", chunk_dir
, err
);
139 let mut lockfile_path
= base
.clone();
140 lockfile_path
.push(".lock");
142 // make sure only one process/thread/task can use it
143 let lockfile
= lock_file(lockfile_path
, 10)?
;
148 hasher
: Sha512Trunc256
::new(),
150 mutex
: Mutex
::new(false)
154 pub fn insert_chunk(&mut self, chunk
: &[u8]) -> Result
<(bool
, [u8; 32]), Error
> {
157 self.hasher
.input(chunk
);
159 let mut digest
= [0u8; 32];
160 self.hasher
.result(&mut digest
);
161 println
!("DIGEST {}", u256_to_hex(&digest
));
163 let mut chunk_path
= self.base
.clone();
164 let prefix
= u256_to_prefix(&digest
);
165 chunk_path
.push(&prefix
);
166 let digest_str
= u256_to_hex(&digest
);
167 chunk_path
.push(&digest_str
);
169 let lock
= self.mutex
.lock();
171 if let Ok(metadata
) = std
::fs
::metadata(&chunk_path
) {
172 if metadata
.is_file() {
173 return Ok((true, digest
));
175 bail
!("Got unexpected file type for chunk {}", digest_str
);
179 let mut chunk_dir
= self.base
.clone();
180 chunk_dir
.push(&prefix
);
182 if let Err(_
) = std
::fs
::create_dir(&chunk_dir
) { /* ignore */ }
184 let mut tmp_path
= chunk_path
.clone();
185 tmp_path
.set_extension("tmp");
186 let mut f
= std
::fs
::File
::create(&tmp_path
)?
;
189 if let Err(err
) = std
::fs
::rename(&tmp_path
, &chunk_path
) {
190 if let Err(_
) = std
::fs
::remove_file(&tmp_path
) { /* ignore */ }
191 bail
!("Atomic rename failed for chunk {} - {}", digest_str
, err
);
194 println
!("PATH {:?}", chunk_path
);
205 fn test_chunk_store1() {
207 if let Err(_e
) = std
::fs
::remove_dir_all(".testdir") { /* ignore */ }
209 let chunk_store
= ChunkStore
::open(".testdir");
210 assert
!(chunk_store
.is_err());
212 let mut chunk_store
= ChunkStore
::create(".testdir").unwrap();
213 let (exists
, _
) = chunk_store
.insert_chunk(&[0u8, 1u8]).unwrap();
216 let (exists
, _
) = chunk_store
.insert_chunk(&[0u8, 1u8]).unwrap();
220 let chunk_store
= ChunkStore
::create(".testdir");
221 assert
!(chunk_store
.is_err());