src/backup/fixed_index.rs

   1 use failure::*;
   2
   3 use crate::tools;
   4 use super::IndexFile;
   5 use super::chunk_stat::*;
   6 use super::chunk_store::*;
   7
   8 use std::sync::Arc;
   9 use std::io::{Read, Write};
  10 use std::fs::File;
  11 use std::path::{Path, PathBuf};
  12 use std::os::unix::io::AsRawFd;
  13 use uuid::Uuid;
  14 use chrono::{Local, TimeZone};
  15
  16 /// Header format definition for fixed index files (`.fidx`)
  17 #[repr(C)]
  18 pub struct FixedIndexHeader {
  19     /// The string `PROXMOX-FIDX`
  20     pub magic: [u8; 12],
  21     pub version: u32,
  22     pub uuid: [u8; 16],
  23     pub ctime: u64,
  24     pub size: u64,
  25     pub chunk_size: u64,
  26     reserved: [u8; 4040], // overall size is one page (4096 bytes)
  27 }
  28
  29 // split image into fixed size chunks
  30
  31 pub struct FixedIndexReader {
  32     store: Arc<ChunkStore>,
  33     _file: File,
  34     filename: PathBuf,
  35     pub chunk_size: usize,
  36     pub size: usize,
  37     index_length: usize,
  38     index: *mut u8,
  39     pub uuid: [u8; 16],
  40     pub ctime: u64,
  41 }
  42
  43 // `index` is mmap()ed which cannot be thread-local so should be sendable
  44 unsafe impl Send for FixedIndexReader {}
  45
  46 impl Drop for FixedIndexReader {
  47
  48     fn drop(&mut self) {
  49         if let Err(err) = self.unmap() {
  50             eprintln!("Unable to unmap file {:?} - {}", self.filename, err);
  51         }
  52     }
  53 }
  54
  55 impl FixedIndexReader {
  56
  57     pub fn open(store: Arc<ChunkStore>, path: &Path) -> Result<Self, Error> {
  58
  59         let full_path = store.relative_path(path);
  60
  61         let mut file = File::open(&full_path)?;
  62
  63         if let Err(err) = nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock) {
  64             bail!("unable to get shared lock on {:?} - {}", full_path, err);
  65         }
  66
  67         let header_size = std::mem::size_of::<FixedIndexHeader>();
  68
  69         // todo: use static assertion when available in rust
  70         if header_size != 4096 { bail!("got unexpected header size for {:?}", path); }
  71
  72         let mut buffer = vec![0u8; header_size];
  73         file.read_exact(&mut buffer)?;
  74
  75         let header = unsafe { &mut * (buffer.as_ptr() as *mut FixedIndexHeader) };
  76
  77         if header.magic != *b"PROXMOX-FIDX" {
  78             bail!("got unknown magic number for {:?}", path);
  79         }
  80
  81         let version = u32::from_le(header.version);
  82         if  version != 1 {
  83             bail!("got unsupported version number ({})", version);
  84         }
  85
  86         let size = u64::from_le(header.size) as usize;
  87         let ctime = u64::from_le(header.ctime);
  88         let chunk_size = u64::from_le(header.chunk_size) as usize;
  89
  90         let index_length = (size + chunk_size - 1)/chunk_size;
  91         let index_size = index_length*32;
  92
  93         let rawfd = file.as_raw_fd();
  94
  95         let stat = match nix::sys::stat::fstat(rawfd) {
  96             Ok(stat) => stat,
  97             Err(err) => bail!("fstat {:?} failed - {}", path, err),
  98         };
  99
 100         let expected_index_size = (stat.st_size as usize) - header_size;
 101         if index_size != expected_index_size {
 102             bail!("got unexpected file size for {:?} ({} != {})",
 103                   path, index_size, expected_index_size);
 104         }
 105
 106         let data = unsafe { nix::sys::mman::mmap(
 107             std::ptr::null_mut(),
 108             index_size,
 109             nix::sys::mman::ProtFlags::PROT_READ,
 110             nix::sys::mman::MapFlags::MAP_PRIVATE,
 111             file.as_raw_fd(),
 112             header_size as i64) }? as *mut u8;
 113
 114         Ok(Self {
 115             store,
 116             filename: full_path,
 117             _file: file,
 118             chunk_size,
 119             size,
 120             index_length,
 121             index: data,
 122             ctime,
 123             uuid: header.uuid,
 124         })
 125     }
 126
 127     fn unmap(&mut self) -> Result<(), Error> {
 128
 129         if self.index == std::ptr::null_mut() { return Ok(()); }
 130
 131         let index_size = self.index_length*32;
 132
 133         if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
 134             bail!("unmap file {:?} failed - {}", self.filename, err);
 135         }
 136
 137         self.index = std::ptr::null_mut();
 138
 139         Ok(())
 140     }
 141
 142     pub fn mark_used_chunks(&self, status: &mut GarbageCollectionStatus) -> Result<(), Error> {
 143
 144         if self.index == std::ptr::null_mut() { bail!("detected closed index file."); }
 145
 146         status.used_bytes += self.index_length * self.chunk_size;
 147         status.used_chunks += self.index_length;
 148
 149         for pos in 0..self.index_length {
 150
 151             tools::fail_on_shutdown()?;
 152
 153             let digest = self.index_digest(pos).unwrap();
 154             if let Err(err) = self.store.touch_chunk(digest) {
 155                 bail!("unable to access chunk {}, required by {:?} - {}",
 156                       tools::digest_to_hex(digest), self.filename, err);
 157             }
 158         }
 159
 160         Ok(())
 161     }
 162
 163     pub fn print_info(&self) {
 164         println!("Filename: {:?}", self.filename);
 165         println!("Size: {}", self.size);
 166         println!("ChunkSize: {}", self.chunk_size);
 167         println!("CTime: {}", Local.timestamp(self.ctime as i64, 0).format("%c"));
 168         println!("UUID: {:?}", self.uuid);
 169     }
 170 }
 171
 172 impl IndexFile for FixedIndexReader {
 173     fn index_count(&self) -> usize {
 174         self.index_length
 175     }
 176
 177     fn index_digest(&self, pos: usize) -> Option<&[u8; 32]> {
 178         if pos >= self.index_length {
 179             None
 180         } else {
 181             Some(unsafe { std::mem::transmute(self.index.add(pos*32)) })
 182         }
 183     }
 184 }
 185
 186 pub struct FixedIndexWriter {
 187     store: Arc<ChunkStore>,
 188     _lock: tools::ProcessLockSharedGuard,
 189     filename: PathBuf,
 190     tmp_filename: PathBuf,
 191     chunk_size: usize,
 192     size: usize,
 193     index_length: usize,
 194     index: *mut u8,
 195     pub uuid: [u8; 16],
 196     pub ctime: u64,
 197 }
 198
 199 // `index` is mmap()ed which cannot be thread-local so should be sendable
 200 unsafe impl Send for FixedIndexWriter {}
 201
 202 impl Drop for FixedIndexWriter {
 203
 204     fn drop(&mut self) {
 205         let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors
 206         if let Err(err) = self.unmap() {
 207             eprintln!("Unable to unmap file {:?} - {}", self.tmp_filename, err);
 208         }
 209     }
 210 }
 211
 212 impl FixedIndexWriter {
 213
 214     pub fn create(store: Arc<ChunkStore>, path: &Path, size: usize, chunk_size: usize) -> Result<Self, Error> {
 215
 216         let shared_lock = store.try_shared_lock()?;
 217
 218         let full_path = store.relative_path(path);
 219         let mut tmp_path = full_path.clone();
 220         tmp_path.set_extension("tmp_fidx");
 221
 222         let mut file = std::fs::OpenOptions::new()
 223             .create(true).truncate(true)
 224             .read(true)
 225             .write(true)
 226             .open(&tmp_path)?;
 227
 228         let header_size = std::mem::size_of::<FixedIndexHeader>();
 229
 230         // todo: use static assertion when available in rust
 231         if header_size != 4096 { panic!("got unexpected header size"); }
 232
 233         let ctime = std::time::SystemTime::now().duration_since(
 234             std::time::SystemTime::UNIX_EPOCH)?.as_secs();
 235
 236         let uuid = Uuid::new_v4();
 237
 238         let buffer = vec![0u8; header_size];
 239         let header = unsafe { &mut * (buffer.as_ptr() as *mut FixedIndexHeader) };
 240
 241         header.magic = *b"PROXMOX-FIDX";
 242         header.version = u32::to_le(1);
 243         header.ctime = u64::to_le(ctime);
 244         header.size = u64::to_le(size as u64);
 245         header.chunk_size = u64::to_le(chunk_size as u64);
 246         header.uuid = *uuid.as_bytes();
 247
 248         file.write_all(&buffer)?;
 249
 250         let index_length = (size + chunk_size - 1)/chunk_size;
 251         let index_size = index_length*32;
 252         nix::unistd::ftruncate(file.as_raw_fd(), (header_size + index_size) as i64)?;
 253
 254         let data = unsafe { nix::sys::mman::mmap(
 255             std::ptr::null_mut(),
 256             index_size,
 257             nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
 258             nix::sys::mman::MapFlags::MAP_SHARED,
 259             file.as_raw_fd(),
 260             header_size as i64) }? as *mut u8;
 261
 262
 263         Ok(Self {
 264             store,
 265             _lock: shared_lock,
 266             filename: full_path,
 267             tmp_filename: tmp_path,
 268             chunk_size,
 269             size,
 270             index_length,
 271             index: data,
 272             ctime,
 273             uuid: *uuid.as_bytes(),
 274         })
 275     }
 276
 277     pub fn index_length(&self) -> usize {
 278         self.index_length
 279     }
 280
 281     fn unmap(&mut self) -> Result<(), Error> {
 282
 283         if self.index == std::ptr::null_mut() { return Ok(()); }
 284
 285         let index_size = self.index_length*32;
 286
 287         if let Err(err) = unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) } {
 288             bail!("unmap file {:?} failed - {}", self.tmp_filename, err);
 289         }
 290
 291         self.index = std::ptr::null_mut();
 292
 293         Ok(())
 294     }
 295
 296     pub fn close(&mut self)  -> Result<(), Error> {
 297
 298         if self.index == std::ptr::null_mut() { bail!("cannot close already closed index file."); }
 299
 300         self.unmap()?;
 301
 302         if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) {
 303             bail!("Atomic rename file {:?} failed - {}", self.filename, err);
 304         }
 305
 306         Ok(())
 307     }
 308
 309     // Note: We want to add data out of order, so do not assume any order here.
 310     pub fn add_chunk(&mut self, pos: usize, chunk: &[u8], stat: &mut ChunkStat) -> Result<(), Error> {
 311
 312         let end = pos + chunk.len();
 313
 314         if end > self.size {
 315             bail!("write chunk data exceeds size ({} >= {})", end, self.size);
 316         }
 317
 318         // last chunk can be smaller
 319         if ((end != self.size) && (chunk.len() != self.chunk_size)) ||
 320             (chunk.len() > self.chunk_size) || (chunk.len() == 0) {
 321                 bail!("got chunk with wrong length ({} != {}", chunk.len(), self.chunk_size);
 322             }
 323
 324         if pos & (self.chunk_size-1) != 0 { bail!("add unaligned chunk (pos = {})", pos); }
 325
 326         let (is_duplicate, digest, compressed_size) = self.store.insert_chunk(chunk)?;
 327
 328         stat.chunk_count += 1;
 329         stat.compressed_size += compressed_size;
 330
 331         println!("ADD CHUNK {} {} {}% {} {}", pos, chunk.len(),
 332                  (compressed_size*100)/(chunk.len() as u64), is_duplicate, tools::digest_to_hex(&digest));
 333
 334         if is_duplicate {
 335             stat.duplicate_chunks += 1;
 336         } else {
 337             stat.disk_size += compressed_size;
 338         }
 339
 340         self.add_digest(pos / self.chunk_size, &digest)
 341     }
 342
 343     pub fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> {
 344
 345         if index >= self.index_length {
 346             bail!("add digest failed - index out of range ({} >= {})", index, self.index_length);
 347         }
 348
 349         if self.index == std::ptr::null_mut() { bail!("cannot write to closed index file."); }
 350
 351         let index_pos = index*32;
 352         unsafe {
 353             let dst = self.index.add(index_pos);
 354             dst.copy_from_nonoverlapping(digest.as_ptr(), 32);
 355         }
 356
 357         Ok(())
 358     }
 359 }