src/backup/fixed_index.rs

   1 use failure::*;
   2
   3 use crate::tools;
   4 use super::chunk_store::*;
   5
   6 use std::sync::Arc;
   7 use std::io::{Read, Write};
   8 use std::path::{Path, PathBuf};
   9 use std::os::unix::io::AsRawFd;
  10 use uuid::Uuid;
  11 use chrono::{Local, TimeZone};
  12
  13 /// Header format definition for fixed index files (`.fixd`)
  14 #[repr(C)]
  15 pub struct FixedIndexHeader {
  16     /// The string `PROXMOX-FIDX`
  17     pub magic: [u8; 12],
  18     pub version: u32,
  19     pub uuid: [u8; 16],
  20     pub ctime: u64,
  21     pub size: u64,
  22     pub chunk_size: u64,
  23     reserved: [u8; 4040], // overall size is one page (4096 bytes)
  24 }
  25
  26 // split image into fixed size chunks
  27
  28 pub struct FixedIndexReader {
  29     store: Arc<ChunkStore>,
  30     filename: PathBuf,
  31     chunk_size: usize,
  32     pub size: usize,
  33     index: *mut u8,
  34     pub uuid: [u8; 16],
  35     pub ctime: u64,
  36 }
  37
  38 impl Drop for FixedIndexReader {
  39
  40     fn drop(&mut self) {
  41         if let Err(err) = self.unmap() {
  42             eprintln!("Unable to unmap file {:?} - {}", self.filename, err);
  43         }
  44     }
  45 }
  46
  47 impl FixedIndexReader {
  48
  49     pub fn open(store: Arc<ChunkStore>, path: &Path) -> Result<Self, Error> {
  50
  51         let full_path = store.relative_path(path);
  52
  53         let mut file = std::fs::File::open(&full_path)?;
  54
  55         let header_size = std::mem::size_of::<FixedIndexHeader>();
  56
  57         // todo: use static assertion when available in rust
  58         if header_size != 4096 { bail!("got unexpected header size for {:?}", path); }
  59
  60         let mut buffer = vec![0u8; header_size];
  61         file.read_exact(&mut buffer)?;
  62
  63         let header = unsafe { &mut * (buffer.as_ptr() as *mut FixedIndexHeader) };
  64
  65         if header.magic != *b"PROXMOX-FIDX" {
  66             bail!("got unknown magic number for {:?}", path);
  67         }
  68
  69         let version = u32::from_le(header.version);
  70         if  version != 1 {
  71             bail!("got unsupported version number ({})", version);
  72         }
  73
  74         let size = u64::from_le(header.size) as usize;
  75         let ctime = u64::from_le(header.ctime);
  76         let chunk_size = u64::from_le(header.chunk_size) as usize;
  77
  78         let index_size = ((size + chunk_size - 1)/chunk_size)*32;
  79
  80         let rawfd = file.as_raw_fd();
  81
  82         let stat = match nix::sys::stat::fstat(rawfd) {
  83             Ok(stat) => stat,
  84             Err(err) => bail!("fstat {:?} failed - {}", path, err),
  85         };
  86
  87         let expected_index_size = (stat.st_size as usize) - header_size;
  88         if index_size != expected_index_size {
  89             bail!("got unexpected file size for {:?} ({} != {})",
  90                   path, index_size, expected_index_size);
  91         }
  92
  93         let data = unsafe { nix::sys::mman::mmap(
  94             std::ptr::null_mut(),
  95             index_size,
  96             nix::sys::mman::ProtFlags::PROT_READ,
  97             nix::sys::mman::MapFlags::MAP_PRIVATE,
  98             file.as_raw_fd(),
  99             header_size as i64) }? as *mut u8;
 100
 101         Ok(Self {
 102             store,
 103             filename: full_path,
 104             chunk_size,
 105             size,
 106             index: data,
 107             ctime,
 108             uuid: header.uuid,
 109         })
 110     }
 111
 112     fn unmap(&mut self) -> Result<(), Error> {
 113
 114         if self.index == std::ptr::null_mut() { return Ok(()); }
 115
 116         let index_size = ((self.size + self.chunk_size - 1)/self.chunk_size)*32;
 117
 118         if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
 119             bail!("unmap file {:?} failed - {}", self.filename, err);
 120         }
 121
 122         self.index = std::ptr::null_mut();
 123
 124         Ok(())
 125     }
 126
 127     pub fn mark_used_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> {
 128
 129         if self.index == std::ptr::null_mut() { bail!("detected closed index file."); }
 130
 131         let index_count = (self.size + self.chunk_size - 1)/self.chunk_size;
 132
 133         status.used_bytes += index_count * self.chunk_size;
 134         status.used_chunks += index_count;
 135
 136         for pos in 0..index_count {
 137
 138             let digest = unsafe { std::slice::from_raw_parts_mut(self.index.add(pos*32), 32) };
 139             if let Err(err) = self.store.touch_chunk(digest) {
 140                 bail!("unable to access chunk {}, required by {:?} - {}",
 141                       tools::digest_to_hex(digest), self.filename, err);
 142             }
 143         }
 144
 145         Ok(())
 146     }
 147
 148     pub fn print_info(&self) {
 149         println!("Filename: {:?}", self.filename);
 150         println!("Size: {}", self.size);
 151         println!("ChunkSize: {}", self.chunk_size);
 152         println!("CTime: {}", Local.timestamp(self.ctime as i64, 0).format("%c"));
 153         println!("UUID: {:?}", self.uuid);
 154     }
 155 }
 156
 157 pub struct FixedIndexWriter {
 158     store: Arc<ChunkStore>,
 159     filename: PathBuf,
 160     tmp_filename: PathBuf,
 161     chunk_size: usize,
 162     duplicate_chunks: usize,
 163     size: usize,
 164     index: *mut u8,
 165     pub uuid: [u8; 16],
 166     pub ctime: u64,
 167 }
 168
 169 impl Drop for FixedIndexWriter {
 170
 171     fn drop(&mut self) {
 172         let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors
 173         if let Err(err) = self.unmap() {
 174             eprintln!("Unable to unmap file {:?} - {}", self.tmp_filename, err);
 175         }
 176     }
 177 }
 178
 179 impl FixedIndexWriter {
 180
 181     pub fn create(store: Arc<ChunkStore>, path: &Path, size: usize, chunk_size: usize) -> Result<Self, Error> {
 182
 183         let full_path = store.relative_path(path);
 184         let mut tmp_path = full_path.clone();
 185         tmp_path.set_extension("tmp_fidx");
 186
 187         let mut file = std::fs::OpenOptions::new()
 188             .create(true).truncate(true)
 189             .read(true)
 190             .write(true)
 191             .open(&tmp_path)?;
 192
 193         let header_size = std::mem::size_of::<FixedIndexHeader>();
 194
 195         // todo: use static assertion when available in rust
 196         if header_size != 4096 { panic!("got unexpected header size"); }
 197
 198         let ctime = std::time::SystemTime::now().duration_since(
 199             std::time::SystemTime::UNIX_EPOCH)?.as_secs();
 200
 201         let uuid = Uuid::new_v4();
 202
 203         let buffer = vec![0u8; header_size];
 204         let header = unsafe { &mut * (buffer.as_ptr() as *mut FixedIndexHeader) };
 205
 206         header.magic = *b"PROXMOX-FIDX";
 207         header.version = u32::to_le(1);
 208         header.ctime = u64::to_le(ctime);
 209         header.size = u64::to_le(size as u64);
 210         header.chunk_size = u64::to_le(chunk_size as u64);
 211         header.uuid = *uuid.as_bytes();
 212
 213         file.write_all(&buffer)?;
 214
 215         let index_size = ((size + chunk_size - 1)/chunk_size)*32;
 216         nix::unistd::ftruncate(file.as_raw_fd(), (header_size + index_size) as i64)?;
 217
 218         let data = unsafe { nix::sys::mman::mmap(
 219             std::ptr::null_mut(),
 220             index_size,
 221             nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
 222             nix::sys::mman::MapFlags::MAP_SHARED,
 223             file.as_raw_fd(),
 224             header_size as i64) }? as *mut u8;
 225
 226
 227         Ok(Self {
 228             store,
 229             filename: full_path,
 230             tmp_filename: tmp_path,
 231             chunk_size,
 232             duplicate_chunks: 0,
 233             size,
 234             index: data,
 235             ctime,
 236             uuid: *uuid.as_bytes(),
 237         })
 238     }
 239
 240     fn unmap(&mut self) -> Result<(), Error> {
 241
 242         if self.index == std::ptr::null_mut() { return Ok(()); }
 243
 244         let index_size = ((self.size + self.chunk_size - 1)/self.chunk_size)*32;
 245
 246         if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
 247             bail!("unmap file {:?} failed - {}", self.tmp_filename, err);
 248         }
 249
 250         self.index = std::ptr::null_mut();
 251
 252         println!("Original size: {} Compressed size: {} Deduplicated size: {}",
 253                 self.size, self.size, self.size - (self.duplicate_chunks*self.chunk_size));
 254
 255         Ok(())
 256     }
 257
 258     pub fn close(&mut self)  -> Result<(), Error> {
 259
 260         if self.index == std::ptr::null_mut() { bail!("cannot close already closed index file."); }
 261
 262         self.unmap()?;
 263
 264         if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) {
 265             bail!("Atomic rename file {:?} failed - {}", self.filename, err);
 266         }
 267
 268         Ok(())
 269     }
 270
 271     // Note: We want to add data out of order, so do not assume and order here.
 272     pub fn add_chunk(&mut self, pos: usize, chunk: &[u8]) -> Result<(), Error> {
 273
 274         if self.index == std::ptr::null_mut() { bail!("cannot write to closed index file."); }
 275
 276         let end = pos + chunk.len();
 277
 278         if end > self.size {
 279             bail!("write chunk data exceeds size ({} >= {})", end, self.size);
 280         }
 281
 282         // last chunk can be smaller
 283         if ((end != self.size) && (chunk.len() != self.chunk_size)) ||
 284             (chunk.len() > self.chunk_size) || (chunk.len() == 0) {
 285                 bail!("got chunk with wrong length ({} != {}", chunk.len(), self.chunk_size);
 286             }
 287
 288         if pos >= self.size { bail!("add chunk after end ({} >= {})", pos, self.size); }
 289
 290         if pos & (self.chunk_size-1) != 0 { bail!("add unaligned chunk (pos = {})", pos); }
 291
 292
 293         let (is_duplicate, digest) = self.store.insert_chunk(chunk)?;
 294
 295         println!("ADD CHUNK {} {} {} {}", pos, chunk.len(), is_duplicate, tools::digest_to_hex(&digest));
 296
 297         if is_duplicate { self.duplicate_chunks += 1; }
 298
 299         let index_pos = (pos/self.chunk_size)*32;
 300         unsafe {
 301             let dst = self.index.add(index_pos);
 302             dst.copy_from_nonoverlapping(digest.as_ptr(), 32);
 303         }
 304
 305         Ok(())
 306     }
 307 }