src/backup/fixed_index.rs

   1 use failure::*;
   2
   3 use crate::tools;
   4 use super::chunk_stat::*;
   5 use super::chunk_store::*;
   6
   7 use std::sync::Arc;
   8 use std::io::{Read, Write};
   9 use std::path::{Path, PathBuf};
  10 use std::os::unix::io::AsRawFd;
  11 use uuid::Uuid;
  12 use chrono::{Local, TimeZone};
  13
  14 /// Header format definition for fixed index files (`.fixd`)
  15 #[repr(C)]
  16 pub struct FixedIndexHeader {
  17     /// The string `PROXMOX-FIDX`
  18     pub magic: [u8; 12],
  19     pub version: u32,
  20     pub uuid: [u8; 16],
  21     pub ctime: u64,
  22     pub size: u64,
  23     pub chunk_size: u64,
  24     reserved: [u8; 4040], // overall size is one page (4096 bytes)
  25 }
  26
  27 // split image into fixed size chunks
  28
  29 pub struct FixedIndexReader {
  30     store: Arc<ChunkStore>,
  31     filename: PathBuf,
  32     chunk_size: usize,
  33     pub size: usize,
  34     index: *mut u8,
  35     pub uuid: [u8; 16],
  36     pub ctime: u64,
  37 }
  38
  39 impl Drop for FixedIndexReader {
  40
  41     fn drop(&mut self) {
  42         if let Err(err) = self.unmap() {
  43             eprintln!("Unable to unmap file {:?} - {}", self.filename, err);
  44         }
  45     }
  46 }
  47
  48 impl FixedIndexReader {
  49
  50     pub fn open(store: Arc<ChunkStore>, path: &Path) -> Result<Self, Error> {
  51
  52         let full_path = store.relative_path(path);
  53
  54         let mut file = std::fs::File::open(&full_path)?;
  55
  56         let header_size = std::mem::size_of::<FixedIndexHeader>();
  57
  58         // todo: use static assertion when available in rust
  59         if header_size != 4096 { bail!("got unexpected header size for {:?}", path); }
  60
  61         let mut buffer = vec![0u8; header_size];
  62         file.read_exact(&mut buffer)?;
  63
  64         let header = unsafe { &mut * (buffer.as_ptr() as *mut FixedIndexHeader) };
  65
  66         if header.magic != *b"PROXMOX-FIDX" {
  67             bail!("got unknown magic number for {:?}", path);
  68         }
  69
  70         let version = u32::from_le(header.version);
  71         if  version != 1 {
  72             bail!("got unsupported version number ({})", version);
  73         }
  74
  75         let size = u64::from_le(header.size) as usize;
  76         let ctime = u64::from_le(header.ctime);
  77         let chunk_size = u64::from_le(header.chunk_size) as usize;
  78
  79         let index_size = ((size + chunk_size - 1)/chunk_size)*32;
  80
  81         let rawfd = file.as_raw_fd();
  82
  83         let stat = match nix::sys::stat::fstat(rawfd) {
  84             Ok(stat) => stat,
  85             Err(err) => bail!("fstat {:?} failed - {}", path, err),
  86         };
  87
  88         let expected_index_size = (stat.st_size as usize) - header_size;
  89         if index_size != expected_index_size {
  90             bail!("got unexpected file size for {:?} ({} != {})",
  91                   path, index_size, expected_index_size);
  92         }
  93
  94         let data = unsafe { nix::sys::mman::mmap(
  95             std::ptr::null_mut(),
  96             index_size,
  97             nix::sys::mman::ProtFlags::PROT_READ,
  98             nix::sys::mman::MapFlags::MAP_PRIVATE,
  99             file.as_raw_fd(),
 100             header_size as i64) }? as *mut u8;
 101
 102         Ok(Self {
 103             store,
 104             filename: full_path,
 105             chunk_size,
 106             size,
 107             index: data,
 108             ctime,
 109             uuid: header.uuid,
 110         })
 111     }
 112
 113     fn unmap(&mut self) -> Result<(), Error> {
 114
 115         if self.index == std::ptr::null_mut() { return Ok(()); }
 116
 117         let index_size = ((self.size + self.chunk_size - 1)/self.chunk_size)*32;
 118
 119         if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
 120             bail!("unmap file {:?} failed - {}", self.filename, err);
 121         }
 122
 123         self.index = std::ptr::null_mut();
 124
 125         Ok(())
 126     }
 127
 128     pub fn mark_used_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> {
 129
 130         if self.index == std::ptr::null_mut() { bail!("detected closed index file."); }
 131
 132         let index_count = (self.size + self.chunk_size - 1)/self.chunk_size;
 133
 134         status.used_bytes += index_count * self.chunk_size;
 135         status.used_chunks += index_count;
 136
 137         for pos in 0..index_count {
 138
 139             let digest = unsafe { std::slice::from_raw_parts_mut(self.index.add(pos*32), 32) };
 140             if let Err(err) = self.store.touch_chunk(digest) {
 141                 bail!("unable to access chunk {}, required by {:?} - {}",
 142                       tools::digest_to_hex(digest), self.filename, err);
 143             }
 144         }
 145
 146         Ok(())
 147     }
 148
 149     pub fn print_info(&self) {
 150         println!("Filename: {:?}", self.filename);
 151         println!("Size: {}", self.size);
 152         println!("ChunkSize: {}", self.chunk_size);
 153         println!("CTime: {}", Local.timestamp(self.ctime as i64, 0).format("%c"));
 154         println!("UUID: {:?}", self.uuid);
 155     }
 156 }
 157
 158 pub struct FixedIndexWriter {
 159     store: Arc<ChunkStore>,
 160     filename: PathBuf,
 161     tmp_filename: PathBuf,
 162     chunk_size: usize,
 163
 164     stat: ChunkStat,
 165
 166     size: usize,
 167     index: *mut u8,
 168     pub uuid: [u8; 16],
 169     pub ctime: u64,
 170 }
 171
 172 impl Drop for FixedIndexWriter {
 173
 174     fn drop(&mut self) {
 175         let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors
 176         if let Err(err) = self.unmap() {
 177             eprintln!("Unable to unmap file {:?} - {}", self.tmp_filename, err);
 178         }
 179     }
 180 }
 181
 182 impl FixedIndexWriter {
 183
 184     pub fn create(store: Arc<ChunkStore>, path: &Path, size: usize, chunk_size: usize) -> Result<Self, Error> {
 185
 186         let full_path = store.relative_path(path);
 187         let mut tmp_path = full_path.clone();
 188         tmp_path.set_extension("tmp_fidx");
 189
 190         let mut file = std::fs::OpenOptions::new()
 191             .create(true).truncate(true)
 192             .read(true)
 193             .write(true)
 194             .open(&tmp_path)?;
 195
 196         let header_size = std::mem::size_of::<FixedIndexHeader>();
 197
 198         // todo: use static assertion when available in rust
 199         if header_size != 4096 { panic!("got unexpected header size"); }
 200
 201         let ctime = std::time::SystemTime::now().duration_since(
 202             std::time::SystemTime::UNIX_EPOCH)?.as_secs();
 203
 204         let uuid = Uuid::new_v4();
 205
 206         let buffer = vec![0u8; header_size];
 207         let header = unsafe { &mut * (buffer.as_ptr() as *mut FixedIndexHeader) };
 208
 209         header.magic = *b"PROXMOX-FIDX";
 210         header.version = u32::to_le(1);
 211         header.ctime = u64::to_le(ctime);
 212         header.size = u64::to_le(size as u64);
 213         header.chunk_size = u64::to_le(chunk_size as u64);
 214         header.uuid = *uuid.as_bytes();
 215
 216         file.write_all(&buffer)?;
 217
 218         let index_size = ((size + chunk_size - 1)/chunk_size)*32;
 219         nix::unistd::ftruncate(file.as_raw_fd(), (header_size + index_size) as i64)?;
 220
 221         let data = unsafe { nix::sys::mman::mmap(
 222             std::ptr::null_mut(),
 223             index_size,
 224             nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
 225             nix::sys::mman::MapFlags::MAP_SHARED,
 226             file.as_raw_fd(),
 227             header_size as i64) }? as *mut u8;
 228
 229
 230         Ok(Self {
 231             store,
 232             filename: full_path,
 233             tmp_filename: tmp_path,
 234             chunk_size,
 235             size,
 236             stat: ChunkStat::new(size as u64),
 237             index: data,
 238             ctime,
 239             uuid: *uuid.as_bytes(),
 240         })
 241     }
 242
 243     fn unmap(&mut self) -> Result<(), Error> {
 244
 245         if self.index == std::ptr::null_mut() { return Ok(()); }
 246
 247         let index_size = ((self.size + self.chunk_size - 1)/self.chunk_size)*32;
 248
 249         if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
 250             bail!("unmap file {:?} failed - {}", self.tmp_filename, err);
 251         }
 252
 253         self.index = std::ptr::null_mut();
 254
 255         self.stat.disk_size += index_size as u64;
 256
 257         println!("STAT: {:?}", self.stat);
 258
 259         Ok(())
 260     }
 261
 262     pub fn close(&mut self)  -> Result<(), Error> {
 263
 264         if self.index == std::ptr::null_mut() { bail!("cannot close already closed index file."); }
 265
 266         self.unmap()?;
 267
 268         if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) {
 269             bail!("Atomic rename file {:?} failed - {}", self.filename, err);
 270         }
 271
 272         Ok(())
 273     }
 274
 275     pub fn stat(&self) -> &ChunkStat {
 276         &self.stat
 277     }
 278
 279     // Note: We want to add data out of order, so do not assume and order here.
 280     pub fn add_chunk(&mut self, pos: usize, chunk: &[u8]) -> Result<(), Error> {
 281
 282         if self.index == std::ptr::null_mut() { bail!("cannot write to closed index file."); }
 283
 284         let end = pos + chunk.len();
 285
 286         if end > self.size {
 287             bail!("write chunk data exceeds size ({} >= {})", end, self.size);
 288         }
 289
 290         // last chunk can be smaller
 291         if ((end != self.size) && (chunk.len() != self.chunk_size)) ||
 292             (chunk.len() > self.chunk_size) || (chunk.len() == 0) {
 293                 bail!("got chunk with wrong length ({} != {}", chunk.len(), self.chunk_size);
 294             }
 295
 296         if pos >= self.size { bail!("add chunk after end ({} >= {})", pos, self.size); }
 297
 298         if pos & (self.chunk_size-1) != 0 { bail!("add unaligned chunk (pos = {})", pos); }
 299
 300
 301         let (is_duplicate, digest, compressed_size) = self.store.insert_chunk(chunk)?;
 302
 303         self.stat.chunk_count += 1;
 304         self.stat.compressed_size += compressed_size;
 305
 306         println!("ADD CHUNK {} {} {}% {} {}", pos, chunk.len(),
 307                  (compressed_size*100)/(chunk.len() as u64), is_duplicate, tools::digest_to_hex(&digest));
 308
 309         if is_duplicate {
 310             self.stat.duplicate_chunks += 1;
 311         } else {
 312             self.stat.disk_size += compressed_size;
 313         }
 314
 315         let index_pos = (pos/self.chunk_size)*32;
 316         unsafe {
 317             let dst = self.index.add(index_pos);
 318             dst.copy_from_nonoverlapping(digest.as_ptr(), 32);
 319         }
 320
 321         Ok(())
 322     }
 323 }