3 use super::chunk_store
::*;
5 use std
::io
::{Read, Write}
;
6 use std
::path
::{Path, PathBuf}
;
7 use std
::os
::unix
::io
::AsRawFd
;
9 use chrono
::{Local, TimeZone}
;
12 pub struct ImageIndexHeader
{
19 reserved
: [u8; 4040], // oversall size is one page (4096 bytes)
22 // split image into fixed size chunks
24 pub struct ImageIndexReader
<'a
> {
25 store
: &'a
mut ChunkStore
,
34 impl <'a
> Drop
for ImageIndexReader
<'a
> {
37 if let Err(err
) = self.unmap() {
38 eprintln
!("Unable to unmap file {:?} - {}", self.filename
, err
);
43 impl <'a
> ImageIndexReader
<'a
> {
45 pub fn open(store
: &'a
mut ChunkStore
, path
: &Path
) -> Result
<Self, Error
> {
47 let full_path
= store
.relative_path(path
);
49 let mut file
= std
::fs
::File
::open(&full_path
)?
;
51 let header_size
= std
::mem
::size_of
::<ImageIndexHeader
>();
53 // todo: use static assertion when available in rust
54 if header_size
!= 4096 { panic!("got unexpected header size"); }
56 let mut buffer
= vec
![0u8; header_size
];
57 file
.read_exact(&mut buffer
)?
;
59 let header
= unsafe { &mut * (buffer.as_ptr() as *mut ImageIndexHeader) }
;
61 let size
= u64::from_be(header
.size
) as usize;
62 let ctime
= u64::from_be(header
.ctime
);
63 let chunk_size
= u64::from_be(header
.chunk_size
) as usize;
65 let index_size
= ((size
+ chunk_size
- 1)/chunk_size
)*32;
67 // fixme: verify file size ?
69 let data
= unsafe { nix
::sys
::mman
::mmap(
72 nix
::sys
::mman
::ProtFlags
::PROT_READ
,
73 nix
::sys
::mman
::MapFlags
::MAP_PRIVATE
,
75 header_size
as i64) }?
as *mut u8;
88 fn unmap(&mut self) -> Result
<(), Error
> {
90 if self.index
== std
::ptr
::null_mut() { return Ok(()); }
92 let index_size
= ((self.size
+ self.chunk_size
- 1)/self.chunk_size
)*32;
94 if let Err(err
) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) }
{
95 bail
!("unmap file {:?} failed - {}", self.filename
, err
);
98 self.index
= std
::ptr
::null_mut();
103 pub fn print_info(&self) {
104 println
!("Filename: {:?}", self.filename
);
105 println
!("Size: {}", self.size
);
106 println
!("ChunkSize: {}", self.chunk_size
);
107 println
!("CTime: {}", Local
.timestamp(self.ctime
as i64, 0).format("%c"));
108 println
!("UUID: {:?}", self.uuid
);
112 pub struct ImageIndexWriter
<'a
> {
113 store
: &'a
mut ChunkStore
,
115 tmp_filename
: PathBuf
,
123 impl <'a
> Drop
for ImageIndexWriter
<'a
> {
126 let _
= std
::fs
::remove_file(&self.tmp_filename
); // ignore errors
127 if let Err(err
) = self.unmap() {
128 eprintln
!("Unable to unmap file {:?} - {}", self.tmp_filename
, err
);
133 impl <'a
> ImageIndexWriter
<'a
> {
135 pub fn create(store
: &'a
mut ChunkStore
, path
: &Path
, size
: usize) -> Result
<Self, Error
> {
137 let full_path
= store
.relative_path(path
);
138 let mut tmp_path
= full_path
.clone();
139 tmp_path
.set_extension("tmp_iidx");
141 let mut file
= std
::fs
::OpenOptions
::new()
142 .create(true).truncate(true)
147 let chunk_size
= 64*1024; // fixed size for now??
149 let header_size
= std
::mem
::size_of
::<ImageIndexHeader
>();
151 // todo: use static assertion when available in rust
152 if header_size
!= 4096 { panic!("got unexpected header size"); }
154 let ctime
= std
::time
::SystemTime
::now().duration_since(
155 std
::time
::SystemTime
::UNIX_EPOCH
)?
.as_secs();
157 let uuid
= Uuid
::new_v4();
159 let buffer
= vec
![0u8; header_size
];
160 let header
= unsafe { &mut * (buffer.as_ptr() as *mut ImageIndexHeader) }
;
162 header
.magic
= *b
"PROXMOX-IIDX";
163 header
.version
= u32::to_be(1);
164 header
.ctime
= u64::to_be(ctime
);
165 header
.size
= u64::to_be(size
as u64);
166 header
.chunk_size
= u64::to_be(chunk_size
as u64);
167 header
.uuid
= *uuid
.as_bytes();
169 file
.write_all(&buffer
)?
;
171 let index_size
= ((size
+ chunk_size
- 1)/chunk_size
)*32;
172 nix
::unistd
::ftruncate(file
.as_raw_fd(), (header_size
+ index_size
) as i64)?
;
174 let data
= unsafe { nix
::sys
::mman
::mmap(
175 std
::ptr
::null_mut(),
177 nix
::sys
::mman
::ProtFlags
::PROT_READ
| nix
::sys
::mman
::ProtFlags
::PROT_WRITE
,
178 nix
::sys
::mman
::MapFlags
::MAP_SHARED
,
180 header_size
as i64) }?
as *mut u8;
186 tmp_filename
: tmp_path
,
191 uuid
: *uuid
.as_bytes(),
195 fn unmap(&mut self) -> Result
<(), Error
> {
197 if self.index
== std
::ptr
::null_mut() { return Ok(()); }
199 let index_size
= ((self.size
+ self.chunk_size
- 1)/self.chunk_size
)*32;
201 if let Err(err
) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) }
{
202 bail
!("unmap file {:?} failed - {}", self.tmp_filename
, err
);
205 self.index
= std
::ptr
::null_mut();
210 pub fn close(&mut self) -> Result
<(), Error
> {
212 if self.index
== std
::ptr
::null_mut() { bail!("cannot close already closed index file."); }
216 if let Err(err
) = std
::fs
::rename(&self.tmp_filename
, &self.filename
) {
217 bail
!("Atomic rename file {:?} failed - {}", self.filename
, err
);
223 // Note: We want to add data out of order, so do not assume and order here.
224 pub fn add_chunk(&mut self, pos
: usize, chunk
: &[u8]) -> Result
<(), Error
> {
226 if self.index
== std
::ptr
::null_mut() { bail!("cannot write to closed index file."); }
228 let end
= pos
+ chunk
.len();
231 bail
!("write chunk data exceeds size ({} >= {})", end
, self.size
);
234 // last chunk can be smaller
235 if ((end
!= self.size
) && (chunk
.len() != self.chunk_size
)) ||
236 (chunk
.len() > self.chunk_size
) || (chunk
.len() == 0) {
237 bail
!("got chunk with wrong length ({} != {}", chunk
.len(), self.chunk_size
);
240 if pos
>= self.size { bail!("add chunk after end ({}
>= {}
)", pos, self.size); }
242 if pos & (self.chunk_size-1) != 0 { bail!("add unaligned chunk (pos = {})", pos
); }
245 let (is_duplicate
, digest
) = self.store
.insert_chunk(chunk
)?
;
247 println
!("ADD CHUNK {} {} {} {}", pos
, chunk
.len(), is_duplicate
, u256_to_hex(&digest
));
249 let index_pos
= (pos
/self.chunk_size
)*32;
251 let dst
= self.index
.add(index_pos
);
252 dst
.copy_from_nonoverlapping(digest
.as_ptr(), 32);