src/backup/fixed_index.rs

   1 use anyhow::{bail, format_err, Error};
   2 use std::io::{Seek, SeekFrom};
   3
   4 use super::chunk_stat::*;
   5 use super::chunk_store::*;
   6 use super::{IndexFile, ChunkReadInfo};
   7 use crate::tools;
   8
   9 use std::fs::File;
  10 use std::io::Write;
  11 use std::os::unix::io::AsRawFd;
  12 use std::path::{Path, PathBuf};
  13 use std::sync::Arc;
  14
  15 use super::ChunkInfo;
  16
  17 use proxmox::tools::io::ReadExt;
  18 use proxmox::tools::Uuid;
  19
  20 /// Header format definition for fixed index files (`.fidx`)
  21 #[repr(C)]
  22 pub struct FixedIndexHeader {
  23     pub magic: [u8; 8],
  24     pub uuid: [u8; 16],
  25     pub ctime: i64,
  26     /// Sha256 over the index ``SHA256(digest1||digest2||...)``
  27     pub index_csum: [u8; 32],
  28     pub size: u64,
  29     pub chunk_size: u64,
  30     reserved: [u8; 4016], // overall size is one page (4096 bytes)
  31 }
  32 proxmox::static_assert_size!(FixedIndexHeader, 4096);
  33
  34 // split image into fixed size chunks
  35
  36 pub struct FixedIndexReader {
  37     _file: File,
  38     pub chunk_size: usize,
  39     pub size: u64,
  40     index_length: usize,
  41     index: *mut u8,
  42     pub uuid: [u8; 16],
  43     pub ctime: i64,
  44     pub index_csum: [u8; 32],
  45 }
  46
  47 // `index` is mmap()ed which cannot be thread-local so should be sendable
  48 unsafe impl Send for FixedIndexReader {}
  49 unsafe impl Sync for FixedIndexReader {}
  50
  51 impl Drop for FixedIndexReader {
  52     fn drop(&mut self) {
  53         if let Err(err) = self.unmap() {
  54             eprintln!("Unable to unmap file - {}", err);
  55         }
  56     }
  57 }
  58
  59 impl FixedIndexReader {
  60     pub fn open(path: &Path) -> Result<Self, Error> {
  61         File::open(path)
  62             .map_err(Error::from)
  63             .and_then(|file| Self::new(file))
  64             .map_err(|err| format_err!("Unable to open fixed index {:?} - {}", path, err))
  65     }
  66
  67     pub fn new(mut file: std::fs::File) -> Result<Self, Error> {
  68         file.seek(SeekFrom::Start(0))?;
  69
  70         let header_size = std::mem::size_of::<FixedIndexHeader>();
  71         let header: Box<FixedIndexHeader> = unsafe { file.read_host_value_boxed()? };
  72
  73         if header.magic != super::FIXED_SIZED_CHUNK_INDEX_1_0 {
  74             bail!("got unknown magic number");
  75         }
  76
  77         let size = u64::from_le(header.size);
  78         let ctime = i64::from_le(header.ctime);
  79         let chunk_size = u64::from_le(header.chunk_size);
  80
  81         let index_length = ((size + chunk_size - 1) / chunk_size) as usize;
  82         let index_size = index_length * 32;
  83
  84         let rawfd = file.as_raw_fd();
  85
  86         let stat = match nix::sys::stat::fstat(rawfd) {
  87             Ok(stat) => stat,
  88             Err(err) => bail!("fstat failed - {}", err),
  89         };
  90
  91         let expected_index_size = (stat.st_size as usize) - header_size;
  92         if index_size != expected_index_size {
  93             bail!(
  94                 "got unexpected file size ({} != {})",
  95                 index_size,
  96                 expected_index_size
  97             );
  98         }
  99
 100         let data = unsafe {
 101             nix::sys::mman::mmap(
 102                 std::ptr::null_mut(),
 103                 index_size,
 104                 nix::sys::mman::ProtFlags::PROT_READ,
 105                 nix::sys::mman::MapFlags::MAP_PRIVATE,
 106                 file.as_raw_fd(),
 107                 header_size as i64,
 108             )
 109         }? as *mut u8;
 110
 111         Ok(Self {
 112             _file: file,
 113             chunk_size: chunk_size as usize,
 114             size,
 115             index_length,
 116             index: data,
 117             ctime,
 118             uuid: header.uuid,
 119             index_csum: header.index_csum,
 120         })
 121     }
 122
 123     fn unmap(&mut self) -> Result<(), Error> {
 124         if self.index == std::ptr::null_mut() {
 125             return Ok(());
 126         }
 127
 128         let index_size = self.index_length * 32;
 129
 130         if let Err(err) =
 131             unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) }
 132         {
 133             bail!("unmap file failed - {}", err);
 134         }
 135
 136         self.index = std::ptr::null_mut();
 137
 138         Ok(())
 139     }
 140
 141     pub fn print_info(&self) {
 142         println!("Size: {}", self.size);
 143         println!("ChunkSize: {}", self.chunk_size);
 144
 145         let mut ctime_str = self.ctime.to_string();
 146         if let Ok(s) = proxmox::tools::time::strftime_local("%c",self.ctime) {
 147             ctime_str = s;
 148         }
 149
 150         println!("CTime: {}", ctime_str);
 151         println!("UUID: {:?}", self.uuid);
 152     }
 153 }
 154
 155 impl IndexFile for FixedIndexReader {
 156     fn index_count(&self) -> usize {
 157         self.index_length
 158     }
 159
 160     fn index_digest(&self, pos: usize) -> Option<&[u8; 32]> {
 161         if pos >= self.index_length {
 162             None
 163         } else {
 164             Some(unsafe { std::mem::transmute(self.index.add(pos * 32)) })
 165         }
 166     }
 167
 168     fn index_bytes(&self) -> u64 {
 169         self.size
 170     }
 171
 172     fn chunk_info(&self, pos: usize) -> Option<ChunkReadInfo> {
 173         if pos >= self.index_length {
 174             return None;
 175         }
 176
 177         let start = (pos * self.chunk_size) as u64;
 178         let mut end = start + self.chunk_size as u64;
 179
 180         if end > self.size {
 181             end = self.size;
 182         }
 183
 184         let digest = self.index_digest(pos).unwrap();
 185         Some(ChunkReadInfo {
 186             range: start..end,
 187             digest: *digest,
 188         })
 189     }
 190
 191     fn compute_csum(&self) -> ([u8; 32], u64) {
 192         let mut csum = openssl::sha::Sha256::new();
 193         let mut chunk_end = 0;
 194         for pos in 0..self.index_count() {
 195             let info = self.chunk_info(pos).unwrap();
 196             chunk_end = info.range.end;
 197             csum.update(&info.digest);
 198         }
 199         let csum = csum.finish();
 200
 201         (csum, chunk_end)
 202     }
 203
 204     fn chunk_from_offset(&self, offset: u64) -> Option<(usize, u64)> {
 205         if offset >= self.size {
 206             return None;
 207         }
 208
 209         Some((
 210             (offset / self.chunk_size as u64) as usize,
 211             offset & (self.chunk_size - 1) as u64 // fast modulo, valid for 2^x chunk_size
 212         ))
 213     }
 214 }
 215
 216 pub struct FixedIndexWriter {
 217     store: Arc<ChunkStore>,
 218     file: File,
 219     _lock: tools::ProcessLockSharedGuard,
 220     filename: PathBuf,
 221     tmp_filename: PathBuf,
 222     chunk_size: usize,
 223     size: usize,
 224     index_length: usize,
 225     index: *mut u8,
 226     pub uuid: [u8; 16],
 227     pub ctime: i64,
 228 }
 229
 230 // `index` is mmap()ed which cannot be thread-local so should be sendable
 231 unsafe impl Send for FixedIndexWriter {}
 232
 233 impl Drop for FixedIndexWriter {
 234     fn drop(&mut self) {
 235         let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors
 236         if let Err(err) = self.unmap() {
 237             eprintln!("Unable to unmap file {:?} - {}", self.tmp_filename, err);
 238         }
 239     }
 240 }
 241
 242 impl FixedIndexWriter {
 243     #[allow(clippy::cast_ptr_alignment)]
 244     pub fn create(
 245         store: Arc<ChunkStore>,
 246         path: &Path,
 247         size: usize,
 248         chunk_size: usize,
 249     ) -> Result<Self, Error> {
 250         let shared_lock = store.try_shared_lock()?;
 251
 252         let full_path = store.relative_path(path);
 253         let mut tmp_path = full_path.clone();
 254         tmp_path.set_extension("tmp_fidx");
 255
 256         let mut file = std::fs::OpenOptions::new()
 257             .create(true)
 258             .truncate(true)
 259             .read(true)
 260             .write(true)
 261             .open(&tmp_path)?;
 262
 263         let header_size = std::mem::size_of::<FixedIndexHeader>();
 264
 265         // todo: use static assertion when available in rust
 266         if header_size != 4096 {
 267             panic!("got unexpected header size");
 268         }
 269
 270         let ctime = proxmox::tools::time::epoch_i64();
 271
 272         let uuid = Uuid::generate();
 273
 274         let buffer = vec![0u8; header_size];
 275         let header = unsafe { &mut *(buffer.as_ptr() as *mut FixedIndexHeader) };
 276
 277         header.magic = super::FIXED_SIZED_CHUNK_INDEX_1_0;
 278         header.ctime = i64::to_le(ctime);
 279         header.size = u64::to_le(size as u64);
 280         header.chunk_size = u64::to_le(chunk_size as u64);
 281         header.uuid = *uuid.as_bytes();
 282
 283         header.index_csum = [0u8; 32];
 284
 285         file.write_all(&buffer)?;
 286
 287         let index_length = (size + chunk_size - 1) / chunk_size;
 288         let index_size = index_length * 32;
 289         nix::unistd::ftruncate(file.as_raw_fd(), (header_size + index_size) as i64)?;
 290
 291         let data = unsafe {
 292             nix::sys::mman::mmap(
 293                 std::ptr::null_mut(),
 294                 index_size,
 295                 nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
 296                 nix::sys::mman::MapFlags::MAP_SHARED,
 297                 file.as_raw_fd(),
 298                 header_size as i64,
 299             )
 300         }? as *mut u8;
 301
 302         Ok(Self {
 303             store,
 304             file,
 305             _lock: shared_lock,
 306             filename: full_path,
 307             tmp_filename: tmp_path,
 308             chunk_size,
 309             size,
 310             index_length,
 311             index: data,
 312             ctime,
 313             uuid: *uuid.as_bytes(),
 314         })
 315     }
 316
 317     pub fn index_length(&self) -> usize {
 318         self.index_length
 319     }
 320
 321     fn unmap(&mut self) -> Result<(), Error> {
 322         if self.index == std::ptr::null_mut() {
 323             return Ok(());
 324         }
 325
 326         let index_size = self.index_length * 32;
 327
 328         if let Err(err) =
 329             unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) }
 330         {
 331             bail!("unmap file {:?} failed - {}", self.tmp_filename, err);
 332         }
 333
 334         self.index = std::ptr::null_mut();
 335
 336         Ok(())
 337     }
 338
 339     pub fn close(&mut self) -> Result<[u8; 32], Error> {
 340         if self.index == std::ptr::null_mut() {
 341             bail!("cannot close already closed index file.");
 342         }
 343
 344         let index_size = self.index_length * 32;
 345         let data = unsafe { std::slice::from_raw_parts(self.index, index_size) };
 346         let index_csum = openssl::sha::sha256(data);
 347
 348         self.unmap()?;
 349
 350         let csum_offset = proxmox::offsetof!(FixedIndexHeader, index_csum);
 351         self.file.seek(SeekFrom::Start(csum_offset as u64))?;
 352         self.file.write_all(&index_csum)?;
 353         self.file.flush()?;
 354
 355         if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) {
 356             bail!("Atomic rename file {:?} failed - {}", self.filename, err);
 357         }
 358
 359         Ok(index_csum)
 360     }
 361
 362     pub fn check_chunk_alignment(&self, offset: usize, chunk_len: usize) -> Result<usize, Error> {
 363         if offset < chunk_len {
 364             bail!("got chunk with small offset ({} < {}", offset, chunk_len);
 365         }
 366
 367         let pos = offset - chunk_len;
 368
 369         if offset > self.size {
 370             bail!("chunk data exceeds size ({} >= {})", offset, self.size);
 371         }
 372
 373         // last chunk can be smaller
 374         if ((offset != self.size) && (chunk_len != self.chunk_size))
 375             || (chunk_len > self.chunk_size)
 376             || (chunk_len == 0)
 377         {
 378             bail!(
 379                 "chunk with unexpected length ({} != {}",
 380                 chunk_len,
 381                 self.chunk_size
 382             );
 383         }
 384
 385         if pos & (self.chunk_size - 1) != 0 {
 386             bail!("got unaligned chunk (pos = {})", pos);
 387         }
 388
 389         Ok(pos / self.chunk_size)
 390     }
 391
 392     // Note: We want to add data out of order, so do not assume any order here.
 393     pub fn add_chunk(&mut self, chunk_info: &ChunkInfo, stat: &mut ChunkStat) -> Result<(), Error> {
 394         let chunk_len = chunk_info.chunk_len as usize;
 395         let offset = chunk_info.offset as usize; // end of chunk
 396
 397         let idx = self.check_chunk_alignment(offset, chunk_len)?;
 398
 399         let (is_duplicate, compressed_size) = self
 400             .store
 401             .insert_chunk(&chunk_info.chunk, &chunk_info.digest)?;
 402
 403         stat.chunk_count += 1;
 404         stat.compressed_size += compressed_size;
 405
 406         let digest = &chunk_info.digest;
 407
 408         println!(
 409             "ADD CHUNK {} {} {}% {} {}",
 410             idx,
 411             chunk_len,
 412             (compressed_size * 100) / (chunk_len as u64),
 413             is_duplicate,
 414             proxmox::tools::digest_to_hex(digest)
 415         );
 416
 417         if is_duplicate {
 418             stat.duplicate_chunks += 1;
 419         } else {
 420             stat.disk_size += compressed_size;
 421         }
 422
 423         self.add_digest(idx, digest)
 424     }
 425
 426     pub fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> {
 427         if index >= self.index_length {
 428             bail!(
 429                 "add digest failed - index out of range ({} >= {})",
 430                 index,
 431                 self.index_length
 432             );
 433         }
 434
 435         if self.index == std::ptr::null_mut() {
 436             bail!("cannot write to closed index file.");
 437         }
 438
 439         let index_pos = index * 32;
 440         unsafe {
 441             let dst = self.index.add(index_pos);
 442             dst.copy_from_nonoverlapping(digest.as_ptr(), 32);
 443         }
 444
 445         Ok(())
 446     }
 447
 448     pub fn clone_data_from(&mut self, reader: &FixedIndexReader) -> Result<(), Error> {
 449         if self.index_length != reader.index_count() {
 450             bail!("clone_data_from failed - index sizes not equal");
 451         }
 452
 453         for i in 0..self.index_length {
 454             self.add_digest(i, reader.index_digest(i).unwrap())?;
 455         }
 456
 457         Ok(())
 458     }
 459 }