src/backup/image_index.rs

   1 use failure::*;
   2
   3 use super::chunk_store::*;
   4
   5 use std::io::{Read, Write};
   6 use std::path::{Path, PathBuf};
   7 use std::os::unix::io::AsRawFd;
   8 use uuid::Uuid;
   9 use chrono::{Local, TimeZone};
  10
  11 #[repr(C)]
  12 pub struct ImageIndexHeader {
  13     pub magic: [u8; 12],
  14     pub version: u32,
  15     pub uuid: [u8; 16],
  16     pub ctime: u64,
  17     pub size: u64,
  18     pub chunk_size: u64,
  19     reserved: [u8; 4040], // oversall size is one page (4096 bytes)
  20 }
  21
  22 // split image into fixed size chunks
  23
  24 pub struct ImageIndexReader<'a> {
  25     store: &'a mut ChunkStore,
  26     filename: PathBuf,
  27     chunk_size: usize,
  28     size: usize,
  29     index: *mut u8,
  30     uuid: [u8; 16],
  31     ctime: u64,
  32 }
  33
  34 impl <'a> Drop for ImageIndexReader<'a> {
  35
  36     fn drop(&mut self) {
  37         if let Err(err) = self.unmap() {
  38             eprintln!("Unable to unmap file {:?} - {}", self.filename, err);
  39         }
  40     }
  41 }
  42
  43 impl <'a> ImageIndexReader<'a> {
  44
  45     pub fn open(store: &'a mut ChunkStore, path: &Path) -> Result<Self, Error> {
  46
  47         let full_path = store.relative_path(path);
  48
  49         let mut file = std::fs::File::open(&full_path)?;
  50
  51         let header_size = std::mem::size_of::<ImageIndexHeader>();
  52
  53         // todo: use static assertion when available in rust
  54         if header_size != 4096 { panic!("got unexpected header size"); }
  55
  56         let mut buffer = vec![0u8; header_size];
  57         file.read_exact(&mut buffer)?;
  58
  59         let header = unsafe { &mut * (buffer.as_ptr() as *mut ImageIndexHeader) };
  60
  61         let size = u64::from_be(header.size) as usize;
  62         let ctime = u64::from_be(header.ctime);
  63         let chunk_size = u64::from_be(header.chunk_size) as usize;
  64
  65         let index_size = ((size + chunk_size - 1)/chunk_size)*32;
  66
  67         // fixme: verify file size ?
  68
  69         let data = unsafe { nix::sys::mman::mmap(
  70             std::ptr::null_mut(),
  71             index_size,
  72             nix::sys::mman::ProtFlags::PROT_READ,
  73             nix::sys::mman::MapFlags::MAP_PRIVATE,
  74             file.as_raw_fd(),
  75             header_size as i64) }? as *mut u8;
  76
  77         Ok(Self {
  78             store,
  79             filename: full_path,
  80             chunk_size,
  81             size,
  82             index: data,
  83             ctime,
  84             uuid: header.uuid,
  85         })
  86     }
  87
  88     fn unmap(&mut self) -> Result<(), Error> {
  89
  90         if self.index == std::ptr::null_mut() { return Ok(()); }
  91
  92         let index_size = ((self.size + self.chunk_size - 1)/self.chunk_size)*32;
  93
  94         if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
  95             bail!("unmap file {:?} failed - {}", self.filename, err);
  96         }
  97
  98         self.index = std::ptr::null_mut();
  99
 100         Ok(())
 101     }
 102
 103     pub fn print_info(&self) {
 104         println!("Filename: {:?}", self.filename);
 105         println!("Size: {}", self.size);
 106         println!("ChunkSize: {}", self.chunk_size);
 107         println!("CTime: {}", Local.timestamp(self.ctime as i64, 0).format("%c"));
 108         println!("UUID: {:?}", self.uuid);
 109     }
 110 }
 111
 112 pub struct ImageIndexWriter<'a> {
 113     store: &'a mut ChunkStore,
 114     filename: PathBuf,
 115     tmp_filename: PathBuf,
 116     chunk_size: usize,
 117     size: usize,
 118     index: *mut u8,
 119     uuid: [u8; 16],
 120     ctime: u64,
 121 }
 122
 123 impl <'a> Drop for ImageIndexWriter<'a> {
 124
 125     fn drop(&mut self) {
 126         let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors
 127         if let Err(err) = self.unmap() {
 128             eprintln!("Unable to unmap file {:?} - {}", self.tmp_filename, err);
 129         }
 130     }
 131 }
 132
 133 impl <'a> ImageIndexWriter<'a> {
 134
 135     pub fn create(store: &'a mut ChunkStore, path: &Path, size: usize) -> Result<Self, Error> {
 136
 137         let full_path = store.relative_path(path);
 138         let mut tmp_path = full_path.clone();
 139         tmp_path.set_extension("tmp_iidx");
 140
 141         let mut file = std::fs::OpenOptions::new()
 142             .create(true).truncate(true)
 143             .read(true)
 144             .write(true)
 145             .open(&tmp_path)?;
 146
 147         let chunk_size = 64*1024; // fixed size for now??
 148
 149         let header_size = std::mem::size_of::<ImageIndexHeader>();
 150
 151         // todo: use static assertion when available in rust
 152         if header_size != 4096 { panic!("got unexpected header size"); }
 153
 154         let ctime = std::time::SystemTime::now().duration_since(
 155             std::time::SystemTime::UNIX_EPOCH)?.as_secs();
 156
 157         let uuid = Uuid::new_v4();
 158
 159         let buffer = vec![0u8; header_size];
 160         let header = unsafe { &mut * (buffer.as_ptr() as *mut ImageIndexHeader) };
 161
 162         header.magic = *b"PROXMOX-IIDX";
 163         header.version = u32::to_be(1);
 164         header.ctime = u64::to_be(ctime);
 165         header.size = u64::to_be(size as u64);
 166         header.chunk_size = u64::to_be(chunk_size as u64);
 167         header.uuid = *uuid.as_bytes();
 168
 169         file.write_all(&buffer)?;
 170
 171         let index_size = ((size + chunk_size - 1)/chunk_size)*32;
 172         nix::unistd::ftruncate(file.as_raw_fd(), (header_size + index_size) as i64)?;
 173
 174         let data = unsafe { nix::sys::mman::mmap(
 175             std::ptr::null_mut(),
 176             index_size,
 177             nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
 178             nix::sys::mman::MapFlags::MAP_SHARED,
 179             file.as_raw_fd(),
 180             header_size as i64) }? as *mut u8;
 181
 182
 183         Ok(Self {
 184             store,
 185             filename: full_path,
 186             tmp_filename: tmp_path,
 187             chunk_size,
 188             size,
 189             index: data,
 190             ctime,
 191             uuid: *uuid.as_bytes(),
 192         })
 193     }
 194
 195     fn unmap(&mut self) -> Result<(), Error> {
 196
 197         if self.index == std::ptr::null_mut() { return Ok(()); }
 198
 199         let index_size = ((self.size + self.chunk_size - 1)/self.chunk_size)*32;
 200
 201         if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
 202             bail!("unmap file {:?} failed - {}", self.tmp_filename, err);
 203         }
 204
 205         self.index = std::ptr::null_mut();
 206
 207         Ok(())
 208     }
 209
 210     pub fn close(&mut self)  -> Result<(), Error> {
 211
 212         if self.index == std::ptr::null_mut() { bail!("cannot close already closed index file."); }
 213
 214         self.unmap()?;
 215
 216         if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) {
 217             bail!("Atomic rename file {:?} failed - {}", self.filename, err);
 218         }
 219
 220         Ok(())
 221     }
 222
 223     // Note: We want to add data out of order, so do not assume and order here.
 224     pub fn add_chunk(&mut self, pos: usize, chunk: &[u8]) -> Result<(), Error> {
 225
 226         if self.index == std::ptr::null_mut() { bail!("cannot write to closed index file."); }
 227
 228         let end = pos + chunk.len();
 229
 230         if end > self.size {
 231             bail!("write chunk data exceeds size ({} >= {})", end, self.size);
 232         }
 233
 234         // last chunk can be smaller
 235         if ((end != self.size) && (chunk.len() != self.chunk_size)) ||
 236             (chunk.len() > self.chunk_size) || (chunk.len() == 0) {
 237                 bail!("got chunk with wrong length ({} != {}", chunk.len(), self.chunk_size);
 238             }
 239
 240         if pos >= self.size { bail!("add chunk after end ({} >= {})", pos, self.size); }
 241
 242         if pos & (self.chunk_size-1) != 0 { bail!("add unaligned chunk (pos = {})", pos); }
 243
 244
 245         let (is_duplicate, digest) = self.store.insert_chunk(chunk)?;
 246
 247         println!("ADD CHUNK {} {} {} {}", pos, chunk.len(), is_duplicate,  u256_to_hex(&digest));
 248
 249         let index_pos = (pos/self.chunk_size)*32;
 250         unsafe {
 251             let dst = self.index.add(index_pos);
 252             dst.copy_from_nonoverlapping(digest.as_ptr(), 32);
 253         }
 254
 255         Ok(())
 256     }
 257 }