]>
Commit | Line | Data |
---|---|---|
35cf5daa DM |
1 | use failure::*; |
2 | use std::path::{Path, PathBuf}; | |
128b37fe DM |
3 | use std::io::Write; |
4 | ||
5 | use crypto::digest::Digest; | |
6 | use crypto::sha2::Sha512Trunc256; | |
c5d82e5f DM |
7 | use std::sync::Mutex; |
8 | ||
35cf5daa DM |
9 | |
10 | pub struct ChunkStore { | |
11 | base: PathBuf, | |
12 | chunk_dir: PathBuf, | |
128b37fe | 13 | hasher: Sha512Trunc256, |
c5d82e5f | 14 | mutex: Mutex<bool>, |
128b37fe DM |
15 | } |
16 | ||
17 | const HEX_CHARS: &'static [u8; 16] = b"0123456789abcdef"; | |
18 | ||
19 | fn u256_to_hex(digest: &[u8; 32]) -> String { | |
20 | ||
21 | let mut buf = Vec::<u8>::with_capacity(64); | |
22 | ||
23 | for i in 0..32 { | |
24 | buf.push(HEX_CHARS[(digest[i] >> 4) as usize]); | |
25 | buf.push(HEX_CHARS[(digest[i] & 0xf) as usize]); | |
26 | } | |
27 | ||
28 | unsafe { String::from_utf8_unchecked(buf) } | |
35cf5daa DM |
29 | } |
30 | ||
128b37fe DM |
31 | fn u256_to_prefix(digest: &[u8; 32]) -> PathBuf { |
32 | ||
33 | let mut buf = Vec::<u8>::with_capacity(3+1+2+1); | |
34 | ||
35 | buf.push(HEX_CHARS[(digest[0] as usize) >> 4]); | |
36 | buf.push(HEX_CHARS[(digest[0] as usize) &0xf]); | |
37 | buf.push(HEX_CHARS[(digest[1] as usize) >> 4]); | |
38 | buf.push('/' as u8); | |
35cf5daa | 39 | |
128b37fe DM |
40 | buf.push(HEX_CHARS[(digest[1] as usize) & 0xf]); |
41 | buf.push(HEX_CHARS[(digest[2] as usize) >> 4]); | |
42 | buf.push('/' as u8); | |
43 | ||
44 | let path = unsafe { String::from_utf8_unchecked(buf)}; | |
45 | ||
46 | path.into() | |
35cf5daa DM |
47 | } |
48 | ||
49 | impl ChunkStore { | |
50 | ||
51 | fn new<P: Into<PathBuf>>(path: P) -> Self { | |
52 | let base = path.into(); | |
53 | let mut chunk_dir = base.clone(); | |
54 | chunk_dir.push(".chunks"); | |
55 | ||
128b37fe DM |
56 | let hasher = Sha512Trunc256::new(); |
57 | ||
c5d82e5f | 58 | ChunkStore { base, chunk_dir, hasher, mutex: Mutex::new(false) } |
35cf5daa DM |
59 | } |
60 | ||
c5d82e5f DM |
61 | // fixme: aquire filesystem lock for directory |
62 | ||
35cf5daa DM |
63 | pub fn create<P: Into<PathBuf>>(path: P) -> Result<Self, Error> { |
64 | ||
65 | let me = Self::new(path); | |
66 | ||
67 | std::fs::create_dir(&me.base)?; | |
68 | std::fs::create_dir(&me.chunk_dir)?; | |
69 | ||
128b37fe DM |
70 | // create 4096 subdir |
71 | for i in 0..4096 { | |
35cf5daa | 72 | let mut l1path = me.base.clone(); |
128b37fe | 73 | l1path.push(format!("{:03x}",i)); |
35cf5daa DM |
74 | std::fs::create_dir(&l1path)?; |
75 | } | |
76 | ||
77 | Ok(me) | |
78 | } | |
79 | ||
80 | pub fn open<P: Into<PathBuf>>(path: P) -> Result<Self, Error> { | |
81 | ||
82 | let me = Self::new(path); | |
83 | ||
84 | let metadata = std::fs::metadata(&me.chunk_dir)?; | |
85 | ||
86 | println!("{:?}", metadata.file_type()); | |
87 | ||
88 | Ok(me) | |
89 | } | |
90 | ||
128b37fe DM |
91 | pub fn insert_chunk(&mut self, chunk: &[u8]) -> Result<([u8; 32]), Error> { |
92 | ||
93 | self.hasher.reset(); | |
94 | self.hasher.input(chunk); | |
95 | ||
96 | let mut digest = [0u8; 32]; | |
97 | self.hasher.result(&mut digest); | |
98 | println!("DIGEST {}", u256_to_hex(&digest)); | |
99 | ||
100 | let mut chunk_path = self.base.clone(); | |
128b37fe | 101 | let prefix = u256_to_prefix(&digest); |
c5d82e5f DM |
102 | chunk_path.push(&prefix); |
103 | let digest_str = u256_to_hex(&digest); | |
104 | chunk_path.push(&digest_str); | |
105 | ||
106 | let lock = self.mutex.lock(); | |
107 | ||
108 | if let Ok(metadata) = std::fs::metadata(&chunk_path) { | |
109 | if metadata.is_file() { | |
110 | return Ok(digest); | |
111 | } else { | |
112 | bail!("Got unexpected file type for chunk {}", digest_str); | |
113 | } | |
114 | } | |
128b37fe | 115 | |
c5d82e5f DM |
116 | let mut chunk_dir = self.base.clone(); |
117 | chunk_dir.push(&prefix); | |
128b37fe | 118 | |
c5d82e5f | 119 | if let Err(_) = std::fs::create_dir(&chunk_dir) { /* ignore */ } |
128b37fe | 120 | |
c5d82e5f DM |
121 | let mut tmp_path = chunk_path.clone(); |
122 | tmp_path.set_extension("tmp"); | |
123 | let mut f = std::fs::File::create(&tmp_path)?; | |
128b37fe DM |
124 | f.write_all(chunk)?; |
125 | ||
c5d82e5f DM |
126 | if let Err(err) = std::fs::rename(&tmp_path, &chunk_path) { |
127 | if let Err(_) = std::fs::remove_file(&tmp_path) { /* ignore */ } | |
128 | bail!("Atomic rename failed for chunk {} - {}", digest_str, err); | |
129 | } | |
130 | ||
128b37fe DM |
131 | println!("PATH {:?}", chunk_path); |
132 | ||
c5d82e5f DM |
133 | drop(lock); |
134 | ||
128b37fe DM |
135 | Ok(digest) |
136 | } | |
137 | ||
35cf5daa DM |
138 | } |
139 | ||
140 | ||
141 | #[test] | |
142 | fn test_chunk_store1() { | |
143 | ||
144 | if let Err(_e) = std::fs::remove_dir_all(".testdir") { /* ignore */ } | |
145 | ||
146 | let chunk_store = ChunkStore::open(".testdir"); | |
147 | assert!(chunk_store.is_err()); | |
148 | ||
128b37fe | 149 | let mut chunk_store = ChunkStore::create(".testdir").unwrap(); |
c5d82e5f DM |
150 | chunk_store.insert_chunk(&[0u8, 1u8]).unwrap(); |
151 | chunk_store.insert_chunk(&[0u8, 1u8]).unwrap(); | |
128b37fe | 152 | |
35cf5daa DM |
153 | |
154 | let chunk_store = ChunkStore::create(".testdir"); | |
155 | assert!(chunk_store.is_err()); | |
156 | ||
157 | ||
158 | } |