src/backup/fixed_index.rs

   1 use failure::*;
   2
   3 use crate::tools;
   4 use super::IndexFile;
   5 use super::chunk_stat::*;
   6 use super::chunk_store::*;
   7
   8 use std::sync::Arc;
   9 use std::io::{Read, Write};
  10 use std::path::{Path, PathBuf};
  11 use std::os::unix::io::AsRawFd;
  12 use uuid::Uuid;
  13 use chrono::{Local, TimeZone};
  14
  15 /// Header format definition for fixed index files (`.fixd`)
  16 #[repr(C)]
  17 pub struct FixedIndexHeader {
  18     /// The string `PROXMOX-FIDX`
  19     pub magic: [u8; 12],
  20     pub version: u32,
  21     pub uuid: [u8; 16],
  22     pub ctime: u64,
  23     pub size: u64,
  24     pub chunk_size: u64,
  25     reserved: [u8; 4040], // overall size is one page (4096 bytes)
  26 }
  27
  28 // split image into fixed size chunks
  29
  30 pub struct FixedIndexReader {
  31     store: Arc<ChunkStore>,
  32     filename: PathBuf,
  33     chunk_size: usize,
  34     pub size: usize,
  35     index: *mut u8,
  36     pub uuid: [u8; 16],
  37     pub ctime: u64,
  38 }
  39
  40 // `index` is mmap()ed which cannot be thread-local so should be sendable
  41 unsafe impl Send for FixedIndexReader {}
  42
  43 impl Drop for FixedIndexReader {
  44
  45     fn drop(&mut self) {
  46         if let Err(err) = self.unmap() {
  47             eprintln!("Unable to unmap file {:?} - {}", self.filename, err);
  48         }
  49     }
  50 }
  51
  52 impl FixedIndexReader {
  53
  54     pub fn open(store: Arc<ChunkStore>, path: &Path) -> Result<Self, Error> {
  55
  56         let full_path = store.relative_path(path);
  57
  58         let mut file = std::fs::File::open(&full_path)?;
  59
  60         let header_size = std::mem::size_of::<FixedIndexHeader>();
  61
  62         // todo: use static assertion when available in rust
  63         if header_size != 4096 { bail!("got unexpected header size for {:?}", path); }
  64
  65         let mut buffer = vec![0u8; header_size];
  66         file.read_exact(&mut buffer)?;
  67
  68         let header = unsafe { &mut * (buffer.as_ptr() as *mut FixedIndexHeader) };
  69
  70         if header.magic != *b"PROXMOX-FIDX" {
  71             bail!("got unknown magic number for {:?}", path);
  72         }
  73
  74         let version = u32::from_le(header.version);
  75         if  version != 1 {
  76             bail!("got unsupported version number ({})", version);
  77         }
  78
  79         let size = u64::from_le(header.size) as usize;
  80         let ctime = u64::from_le(header.ctime);
  81         let chunk_size = u64::from_le(header.chunk_size) as usize;
  82
  83         let index_size = ((size + chunk_size - 1)/chunk_size)*32;
  84
  85         let rawfd = file.as_raw_fd();
  86
  87         let stat = match nix::sys::stat::fstat(rawfd) {
  88             Ok(stat) => stat,
  89             Err(err) => bail!("fstat {:?} failed - {}", path, err),
  90         };
  91
  92         let expected_index_size = (stat.st_size as usize) - header_size;
  93         if index_size != expected_index_size {
  94             bail!("got unexpected file size for {:?} ({} != {})",
  95                   path, index_size, expected_index_size);
  96         }
  97
  98         let data = unsafe { nix::sys::mman::mmap(
  99             std::ptr::null_mut(),
 100             index_size,
 101             nix::sys::mman::ProtFlags::PROT_READ,
 102             nix::sys::mman::MapFlags::MAP_PRIVATE,
 103             file.as_raw_fd(),
 104             header_size as i64) }? as *mut u8;
 105
 106         Ok(Self {
 107             store,
 108             filename: full_path,
 109             chunk_size,
 110             size,
 111             index: data,
 112             ctime,
 113             uuid: header.uuid,
 114         })
 115     }
 116
 117     fn unmap(&mut self) -> Result<(), Error> {
 118
 119         if self.index == std::ptr::null_mut() { return Ok(()); }
 120
 121         let index_size = ((self.size + self.chunk_size - 1)/self.chunk_size)*32;
 122
 123         if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
 124             bail!("unmap file {:?} failed - {}", self.filename, err);
 125         }
 126
 127         self.index = std::ptr::null_mut();
 128
 129         Ok(())
 130     }
 131
 132     pub fn mark_used_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> {
 133
 134         if self.index == std::ptr::null_mut() { bail!("detected closed index file."); }
 135
 136         let index_count = self.index_count();
 137
 138         status.used_bytes += index_count * self.chunk_size;
 139         status.used_chunks += index_count;
 140
 141         for pos in 0..index_count {
 142
 143             let digest = self.index_digest(pos).unwrap();
 144             if let Err(err) = self.store.touch_chunk(digest) {
 145                 bail!("unable to access chunk {}, required by {:?} - {}",
 146                       tools::digest_to_hex(digest), self.filename, err);
 147             }
 148         }
 149
 150         Ok(())
 151     }
 152
 153     pub fn print_info(&self) {
 154         println!("Filename: {:?}", self.filename);
 155         println!("Size: {}", self.size);
 156         println!("ChunkSize: {}", self.chunk_size);
 157         println!("CTime: {}", Local.timestamp(self.ctime as i64, 0).format("%c"));
 158         println!("UUID: {:?}", self.uuid);
 159     }
 160 }
 161
 162 impl IndexFile for FixedIndexReader {
 163     fn index_count(&self) -> usize {
 164         (self.size + self.chunk_size - 1)/self.chunk_size
 165     }
 166
 167     fn index_digest(&self, pos: usize) -> Option<&[u8; 32]> {
 168         if pos >= self.index_count() {
 169             None
 170         } else {
 171             Some(unsafe { std::mem::transmute(self.index.add(pos*32)) })
 172         }
 173     }
 174 }
 175
 176 pub struct FixedIndexWriter {
 177     store: Arc<ChunkStore>,
 178     filename: PathBuf,
 179     tmp_filename: PathBuf,
 180     chunk_size: usize,
 181
 182     stat: ChunkStat,
 183
 184     size: usize,
 185     index: *mut u8,
 186     pub uuid: [u8; 16],
 187     pub ctime: u64,
 188 }
 189
 190 // `index` is mmap()ed which cannot be thread-local so should be sendable
 191 unsafe impl Send for FixedIndexWriter {}
 192
 193 impl Drop for FixedIndexWriter {
 194
 195     fn drop(&mut self) {
 196         let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors
 197         if let Err(err) = self.unmap() {
 198             eprintln!("Unable to unmap file {:?} - {}", self.tmp_filename, err);
 199         }
 200     }
 201 }
 202
 203 impl FixedIndexWriter {
 204
 205     pub fn create(store: Arc<ChunkStore>, path: &Path, size: usize, chunk_size: usize) -> Result<Self, Error> {
 206
 207         let full_path = store.relative_path(path);
 208         let mut tmp_path = full_path.clone();
 209         tmp_path.set_extension("tmp_fidx");
 210
 211         let mut file = std::fs::OpenOptions::new()
 212             .create(true).truncate(true)
 213             .read(true)
 214             .write(true)
 215             .open(&tmp_path)?;
 216
 217         let header_size = std::mem::size_of::<FixedIndexHeader>();
 218
 219         // todo: use static assertion when available in rust
 220         if header_size != 4096 { panic!("got unexpected header size"); }
 221
 222         let ctime = std::time::SystemTime::now().duration_since(
 223             std::time::SystemTime::UNIX_EPOCH)?.as_secs();
 224
 225         let uuid = Uuid::new_v4();
 226
 227         let buffer = vec![0u8; header_size];
 228         let header = unsafe { &mut * (buffer.as_ptr() as *mut FixedIndexHeader) };
 229
 230         header.magic = *b"PROXMOX-FIDX";
 231         header.version = u32::to_le(1);
 232         header.ctime = u64::to_le(ctime);
 233         header.size = u64::to_le(size as u64);
 234         header.chunk_size = u64::to_le(chunk_size as u64);
 235         header.uuid = *uuid.as_bytes();
 236
 237         file.write_all(&buffer)?;
 238
 239         let index_size = ((size + chunk_size - 1)/chunk_size)*32;
 240         nix::unistd::ftruncate(file.as_raw_fd(), (header_size + index_size) as i64)?;
 241
 242         let data = unsafe { nix::sys::mman::mmap(
 243             std::ptr::null_mut(),
 244             index_size,
 245             nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
 246             nix::sys::mman::MapFlags::MAP_SHARED,
 247             file.as_raw_fd(),
 248             header_size as i64) }? as *mut u8;
 249
 250
 251         Ok(Self {
 252             store,
 253             filename: full_path,
 254             tmp_filename: tmp_path,
 255             chunk_size,
 256             size,
 257             stat: ChunkStat::new(size as u64),
 258             index: data,
 259             ctime,
 260             uuid: *uuid.as_bytes(),
 261         })
 262     }
 263
 264     fn unmap(&mut self) -> Result<(), Error> {
 265
 266         if self.index == std::ptr::null_mut() { return Ok(()); }
 267
 268         let index_size = ((self.size + self.chunk_size - 1)/self.chunk_size)*32;
 269
 270         if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
 271             bail!("unmap file {:?} failed - {}", self.tmp_filename, err);
 272         }
 273
 274         self.index = std::ptr::null_mut();
 275
 276         self.stat.disk_size += index_size as u64;
 277
 278         println!("STAT: {:?}", self.stat);
 279
 280         Ok(())
 281     }
 282
 283     pub fn close(&mut self)  -> Result<(), Error> {
 284
 285         if self.index == std::ptr::null_mut() { bail!("cannot close already closed index file."); }
 286
 287         self.unmap()?;
 288
 289         if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) {
 290             bail!("Atomic rename file {:?} failed - {}", self.filename, err);
 291         }
 292
 293         Ok(())
 294     }
 295
 296     pub fn stat(&self) -> &ChunkStat {
 297         &self.stat
 298     }
 299
 300     // Note: We want to add data out of order, so do not assume and order here.
 301     pub fn add_chunk(&mut self, pos: usize, chunk: &[u8]) -> Result<(), Error> {
 302
 303         if self.index == std::ptr::null_mut() { bail!("cannot write to closed index file."); }
 304
 305         let end = pos + chunk.len();
 306
 307         if end > self.size {
 308             bail!("write chunk data exceeds size ({} >= {})", end, self.size);
 309         }
 310
 311         // last chunk can be smaller
 312         if ((end != self.size) && (chunk.len() != self.chunk_size)) ||
 313             (chunk.len() > self.chunk_size) || (chunk.len() == 0) {
 314                 bail!("got chunk with wrong length ({} != {}", chunk.len(), self.chunk_size);
 315             }
 316
 317         if pos >= self.size { bail!("add chunk after end ({} >= {})", pos, self.size); }
 318
 319         if pos & (self.chunk_size-1) != 0 { bail!("add unaligned chunk (pos = {})", pos); }
 320
 321
 322         let (is_duplicate, digest, compressed_size) = self.store.insert_chunk(chunk)?;
 323
 324         self.stat.chunk_count += 1;
 325         self.stat.compressed_size += compressed_size;
 326
 327         println!("ADD CHUNK {} {} {}% {} {}", pos, chunk.len(),
 328                  (compressed_size*100)/(chunk.len() as u64), is_duplicate, tools::digest_to_hex(&digest));
 329
 330         if is_duplicate {
 331             self.stat.duplicate_chunks += 1;
 332         } else {
 333             self.stat.disk_size += compressed_size;
 334         }
 335
 336         self.add_digest(pos / self.chunk_size, &digest)
 337     }
 338
 339     pub fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> {
 340
 341         let index_pos = index*32;
 342         unsafe {
 343             let dst = self.index.add(index_pos);
 344             dst.copy_from_nonoverlapping(digest.as_ptr(), 32);
 345         }
 346
 347         Ok(())
 348     }
 349 }