2 use std
::path
::{Path, PathBuf}
;
4 use std
::time
::Duration
;
10 use std
::os
::unix
::io
::AsRawFd
;
14 pub struct GarbageCollectionStatus
{
15 pub used_bytes
: usize,
16 pub used_chunks
: usize,
17 pub disk_bytes
: usize,
18 pub disk_chunks
: usize,
21 impl Default
for GarbageCollectionStatus
{
22 fn default() -> Self {
23 GarbageCollectionStatus
{
32 /// File system based chunk store
33 pub struct ChunkStore
{
34 name
: String
, // used for error reporting
35 pub (crate) base
: PathBuf
,
41 // TODO: what about sysctl setting vm.vfs_cache_pressure (0 - 100) ?
43 fn digest_to_prefix(digest
: &[u8]) -> PathBuf
{
45 let mut buf
= Vec
::<u8>::with_capacity(2+1+2+1);
47 const HEX_CHARS
: &'
static [u8; 16] = b
"0123456789abcdef";
49 buf
.push(HEX_CHARS
[(digest
[0] as usize) >> 4]);
50 buf
.push(HEX_CHARS
[(digest
[0] as usize) &0xf]);
51 buf
.push(HEX_CHARS
[(digest
[1] as usize) >> 4]);
52 buf
.push(HEX_CHARS
[(digest
[1] as usize) & 0xf]);
55 let path
= unsafe { String::from_utf8_unchecked(buf)}
;
62 fn chunk_dir
<P
: AsRef
<Path
>>(path
: P
) -> PathBuf
{
64 let mut chunk_dir
: PathBuf
= PathBuf
::from(path
.as_ref());
65 chunk_dir
.push(".chunks");
70 pub fn create
<P
: Into
<PathBuf
>>(name
: &str, path
: P
) -> Result
<Self, Error
> {
72 let base
: PathBuf
= path
.into();
74 if !base
.is_absolute() {
75 bail
!("expected absolute path - got {:?}", base
);
78 let chunk_dir
= Self::chunk_dir(&base
);
80 if let Err(err
) = std
::fs
::create_dir(&base
) {
81 bail
!("unable to create chunk store '{}' at {:?} - {}", name
, base
, err
);
84 if let Err(err
) = std
::fs
::create_dir(&chunk_dir
) {
85 bail
!("unable to create chunk store '{}' subdir {:?} - {}", name
, chunk_dir
, err
);
88 // create 64*1024 subdirs
89 let mut last_percentage
= 0;
92 let mut l1path
= chunk_dir
.clone();
93 l1path
.push(format
!("{:04x}", i
));
94 if let Err(err
) = std
::fs
::create_dir(&l1path
) {
95 bail
!("unable to create chunk store '{}' subdir {:?} - {}", name
, l1path
, err
);
97 let percentage
= (i
*100)/(64*1024);
98 if percentage
!= last_percentage
{
99 eprintln
!("Percentage done: {}", percentage
);
100 last_percentage
= percentage
;
104 Self::open(name
, base
)
107 pub fn open
<P
: Into
<PathBuf
>>(name
: &str, path
: P
) -> Result
<Self, Error
> {
109 let base
: PathBuf
= path
.into();
111 if !base
.is_absolute() {
112 bail
!("expected absolute path - got {:?}", base
);
115 let chunk_dir
= Self::chunk_dir(&base
);
117 if let Err(err
) = std
::fs
::metadata(&chunk_dir
) {
118 bail
!("unable to open chunk store '{}' at {:?} - {}", name
, chunk_dir
, err
);
121 let mut lockfile_path
= base
.clone();
122 lockfile_path
.push(".lock");
124 // make sure only one process/thread/task can use it
125 let lockfile
= tools
::open_file_locked(
126 lockfile_path
, Duration
::from_secs(10))?
;
129 name
: name
.to_owned(),
133 mutex
: Mutex
::new(false)
137 pub fn touch_chunk(&self, digest
:&[u8]) -> Result
<(), Error
> {
139 let mut chunk_path
= self.chunk_dir
.clone();
140 let prefix
= digest_to_prefix(&digest
);
141 chunk_path
.push(&prefix
);
142 let digest_str
= tools
::digest_to_hex(&digest
);
143 chunk_path
.push(&digest_str
);
145 const UTIME_NOW
: i64 = ((1 << 30) - 1);
146 const UTIME_OMIT
: i64 = ((1 << 30) - 2);
148 let times
: [libc
::timespec
; 2] = [
149 libc
::timespec { tv_sec: 0, tv_nsec: UTIME_NOW }
,
150 libc
::timespec { tv_sec: 0, tv_nsec: UTIME_OMIT }
155 let res
= chunk_path
.with_nix_path(|cstr
| unsafe {
156 libc
::utimensat(-1, cstr
.as_ptr(), ×
[0], libc
::AT_SYMLINK_NOFOLLOW
)
159 if let Err(err
) = nix
::errno
::Errno
::result(res
) {
160 bail
!("updata atime failed for chunk {:?} - {}", chunk_path
, err
);
166 pub fn read_chunk(&self, digest
:&[u8], buffer
: &mut Vec
<u8>) -> Result
<(), Error
> {
168 let mut chunk_path
= self.chunk_dir
.clone();
169 let prefix
= digest_to_prefix(&digest
);
170 chunk_path
.push(&prefix
);
171 let digest_str
= tools
::digest_to_hex(&digest
);
172 chunk_path
.push(&digest_str
);
174 let mut f
= std
::fs
::File
::open(&chunk_path
)?
;
176 let stat
= nix
::sys
::stat
::fstat(f
.as_raw_fd())?
;
177 let size
= stat
.st_size
as usize;
179 if buffer
.capacity() < size
{
180 let mut newsize
= buffer
.capacity();
181 while newsize
< size { newsize = newsize << 1; }
182 let additional
= newsize
- buffer
.len();
183 buffer
.reserve_exact(additional
);
185 unsafe { buffer.set_len(size); }
189 f
.read_exact(buffer
.as_mut_slice())?
;
194 pub fn get_chunk_iterator(
197 impl Iterator
<Item
= Result
<tools
::fs
::ReadDirEntry
, Error
>>,
201 use nix
::fcntl
::OFlag
;
202 use nix
::sys
::stat
::Mode
;
204 let base_handle
= match Dir
::open(
205 &self.chunk_dir
, OFlag
::O_RDONLY
, Mode
::empty()) {
207 Err(err
) => bail
!("unable to open store '{}' chunk dir {:?} - {}",
208 self.name
, self.chunk_dir
, err
),
211 let mut verbose
= true;
212 let mut last_percentage
= 0;
214 Ok((0..0x10000).filter_map(move |index
| {
215 let percentage
= (index
* 100) / 0x10000;
216 if last_percentage
!= percentage
{
217 last_percentage
= percentage
;
218 eprintln
!("percentage done: {}", percentage
);
220 let subdir
: &str = &format
!("{:04x}", index
);
221 match tools
::fs
::read_subdir(base_handle
.as_raw_fd(), subdir
) {
224 eprintln
!("Error iterating through chunks: {}", e
);
229 Ok(iter
) => Some(iter
),
234 // Check that the file name is actually a hash! (64 hex digits)
235 let entry
= match entry
{
236 Err(_
) => return true, // pass errors onwards
237 Ok(ref entry
) => entry
,
239 let bytes
= entry
.file_name().to_bytes();
240 if bytes
.len() != 64 {
244 if !b
.is_ascii_hexdigit() {
252 pub fn sweep_unused_chunks(&self, status
: &mut GarbageCollectionStatus
) -> Result
<(), Error
> {
253 use nix
::sys
::stat
::fstatat
;
255 let now
= unsafe { libc::time(std::ptr::null_mut()) }
;
257 for entry
in self.get_chunk_iterator()?
{
258 let (dirfd
, entry
) = match entry
{
259 Ok(entry
) => (entry
.parent_fd(), entry
),
260 Err(_
) => continue, // ignore errors
263 let file_type
= match entry
.file_type() {
264 Some(file_type
) => file_type
,
265 None
=> bail
!("unsupported file system type on chunk store '{}'", self.name
),
267 if file_type
!= nix
::dir
::Type
::File
{
271 let filename
= entry
.file_name();
272 if let Ok(stat
) = fstatat(dirfd
, filename
, nix
::fcntl
::AtFlags
::AT_SYMLINK_NOFOLLOW
) {
273 let age
= now
- stat
.st_atime
;
274 //println!("FOUND {} {:?}", age/(3600*24), filename);
275 if age
/(3600*24) >= 2 {
276 println
!("UNLINK {} {:?}", age
/(3600*24), filename
);
277 let res
= unsafe { libc::unlinkat(dirfd, filename.as_ptr(), 0) }
;
279 let err
= nix
::Error
::last();
281 "unlink chunk {:?} failed on store '{}' - {}",
288 status
.disk_chunks
+= 1;
289 status
.disk_bytes
+= stat
.st_size
as usize;
297 pub fn insert_chunk(&self, chunk
: &[u8]) -> Result
<(bool
, [u8; 32]), Error
> {
299 // fixme: use Sha512/256 when available
300 let mut hasher
= sha
::Sha256
::new();
301 hasher
.update(chunk
);
303 let digest
= hasher
.finish();
305 //println!("DIGEST {}", tools::digest_to_hex(&digest));
307 let mut chunk_path
= self.chunk_dir
.clone();
308 let prefix
= digest_to_prefix(&digest
);
309 chunk_path
.push(&prefix
);
310 let digest_str
= tools
::digest_to_hex(&digest
);
311 chunk_path
.push(&digest_str
);
313 let lock
= self.mutex
.lock();
315 if let Ok(metadata
) = std
::fs
::metadata(&chunk_path
) {
316 if metadata
.is_file() {
317 return Ok((true, digest
));
319 bail
!("Got unexpected file type on store '{}' for chunk {}", self.name
, digest_str
);
323 let mut tmp_path
= chunk_path
.clone();
324 tmp_path
.set_extension("tmp");
325 let mut f
= std
::fs
::File
::create(&tmp_path
)?
;
328 if let Err(err
) = std
::fs
::rename(&tmp_path
, &chunk_path
) {
329 if let Err(_
) = std
::fs
::remove_file(&tmp_path
) { /* ignore */ }
331 "Atomic rename on store '{}' failed for chunk {} - {}",
338 //println!("PATH {:?}", chunk_path);
345 pub fn relative_path(&self, path
: &Path
) -> PathBuf
{
347 let mut full_path
= self.base
.clone();
348 full_path
.push(path
);
352 pub fn base_path(&self) -> PathBuf
{
359 fn test_chunk_store1() {
361 let mut path
= std
::fs
::canonicalize(".").unwrap(); // we need absulute path
362 path
.push(".testdir");
364 if let Err(_e
) = std
::fs
::remove_dir_all(".testdir") { /* ignore */ }
366 let chunk_store
= ChunkStore
::open("test", &path
);
367 assert
!(chunk_store
.is_err());
369 let chunk_store
= ChunkStore
::create("test", &path
).unwrap();
370 let (exists
, _
) = chunk_store
.insert_chunk(&[0u8, 1u8]).unwrap();
373 let (exists
, _
) = chunk_store
.insert_chunk(&[0u8, 1u8]).unwrap();
377 let chunk_store
= ChunkStore
::create("test", &path
);
378 assert
!(chunk_store
.is_err());
380 if let Err(_e
) = std
::fs
::remove_dir_all(".testdir") { /* ignore */ }