src/backup/fixed_index.rs

   1 use anyhow::{bail, format_err, Error};
   2 use std::io::{Seek, SeekFrom};
   3
   4 use super::chunk_stat::*;
   5 use super::chunk_store::*;
   6 use super::{IndexFile, ChunkReadInfo};
   7 use crate::tools::{self, epoch_now_u64};
   8
   9 use chrono::{Local, TimeZone};
  10 use std::fs::File;
  11 use std::io::Write;
  12 use std::os::unix::io::AsRawFd;
  13 use std::path::{Path, PathBuf};
  14 use std::sync::Arc;
  15
  16 use super::ChunkInfo;
  17
  18 use proxmox::tools::io::ReadExt;
  19 use proxmox::tools::Uuid;
  20
  21 /// Header format definition for fixed index files (`.fidx`)
  22 #[repr(C)]
  23 pub struct FixedIndexHeader {
  24     pub magic: [u8; 8],
  25     pub uuid: [u8; 16],
  26     pub ctime: u64,
  27     /// Sha256 over the index ``SHA256(digest1||digest2||...)``
  28     pub index_csum: [u8; 32],
  29     pub size: u64,
  30     pub chunk_size: u64,
  31     reserved: [u8; 4016], // overall size is one page (4096 bytes)
  32 }
  33 proxmox::static_assert_size!(FixedIndexHeader, 4096);
  34
  35 // split image into fixed size chunks
  36
  37 pub struct FixedIndexReader {
  38     _file: File,
  39     pub chunk_size: usize,
  40     pub size: u64,
  41     index_length: usize,
  42     index: *mut u8,
  43     pub uuid: [u8; 16],
  44     pub ctime: u64,
  45     pub index_csum: [u8; 32],
  46 }
  47
  48 // `index` is mmap()ed which cannot be thread-local so should be sendable
  49 unsafe impl Send for FixedIndexReader {}
  50 unsafe impl Sync for FixedIndexReader {}
  51
  52 impl Drop for FixedIndexReader {
  53     fn drop(&mut self) {
  54         if let Err(err) = self.unmap() {
  55             eprintln!("Unable to unmap file - {}", err);
  56         }
  57     }
  58 }
  59
  60 impl FixedIndexReader {
  61     pub fn open(path: &Path) -> Result<Self, Error> {
  62         File::open(path)
  63             .map_err(Error::from)
  64             .and_then(|file| Self::new(file))
  65             .map_err(|err| format_err!("Unable to open fixed index {:?} - {}", path, err))
  66     }
  67
  68     pub fn new(mut file: std::fs::File) -> Result<Self, Error> {
  69         if let Err(err) =
  70             nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock)
  71         {
  72             bail!("unable to get shared lock - {}", err);
  73         }
  74
  75         file.seek(SeekFrom::Start(0))?;
  76
  77         let header_size = std::mem::size_of::<FixedIndexHeader>();
  78         let header: Box<FixedIndexHeader> = unsafe { file.read_host_value_boxed()? };
  79
  80         if header.magic != super::FIXED_SIZED_CHUNK_INDEX_1_0 {
  81             bail!("got unknown magic number");
  82         }
  83
  84         let size = u64::from_le(header.size);
  85         let ctime = u64::from_le(header.ctime);
  86         let chunk_size = u64::from_le(header.chunk_size);
  87
  88         let index_length = ((size + chunk_size - 1) / chunk_size) as usize;
  89         let index_size = index_length * 32;
  90
  91         let rawfd = file.as_raw_fd();
  92
  93         let stat = match nix::sys::stat::fstat(rawfd) {
  94             Ok(stat) => stat,
  95             Err(err) => bail!("fstat failed - {}", err),
  96         };
  97
  98         let expected_index_size = (stat.st_size as usize) - header_size;
  99         if index_size != expected_index_size {
 100             bail!(
 101                 "got unexpected file size ({} != {})",
 102                 index_size,
 103                 expected_index_size
 104             );
 105         }
 106
 107         let data = unsafe {
 108             nix::sys::mman::mmap(
 109                 std::ptr::null_mut(),
 110                 index_size,
 111                 nix::sys::mman::ProtFlags::PROT_READ,
 112                 nix::sys::mman::MapFlags::MAP_PRIVATE,
 113                 file.as_raw_fd(),
 114                 header_size as i64,
 115             )
 116         }? as *mut u8;
 117
 118         Ok(Self {
 119             _file: file,
 120             chunk_size: chunk_size as usize,
 121             size,
 122             index_length,
 123             index: data,
 124             ctime,
 125             uuid: header.uuid,
 126             index_csum: header.index_csum,
 127         })
 128     }
 129
 130     fn unmap(&mut self) -> Result<(), Error> {
 131         if self.index == std::ptr::null_mut() {
 132             return Ok(());
 133         }
 134
 135         let index_size = self.index_length * 32;
 136
 137         if let Err(err) =
 138             unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) }
 139         {
 140             bail!("unmap file failed - {}", err);
 141         }
 142
 143         self.index = std::ptr::null_mut();
 144
 145         Ok(())
 146     }
 147
 148     pub fn print_info(&self) {
 149         println!("Size: {}", self.size);
 150         println!("ChunkSize: {}", self.chunk_size);
 151         println!(
 152             "CTime: {}",
 153             Local.timestamp(self.ctime as i64, 0).format("%c")
 154         );
 155         println!("UUID: {:?}", self.uuid);
 156     }
 157 }
 158
 159 impl IndexFile for FixedIndexReader {
 160     fn index_count(&self) -> usize {
 161         self.index_length
 162     }
 163
 164     fn index_digest(&self, pos: usize) -> Option<&[u8; 32]> {
 165         if pos >= self.index_length {
 166             None
 167         } else {
 168             Some(unsafe { std::mem::transmute(self.index.add(pos * 32)) })
 169         }
 170     }
 171
 172     fn index_bytes(&self) -> u64 {
 173         self.size
 174     }
 175
 176     fn chunk_info(&self, pos: usize) -> Option<ChunkReadInfo> {
 177         if pos >= self.index_length {
 178             return None;
 179         }
 180
 181         let start = (pos * self.chunk_size) as u64;
 182         let mut end = start + self.chunk_size as u64;
 183
 184         if end > self.size {
 185             end = self.size;
 186         }
 187
 188         let digest = self.index_digest(pos).unwrap();
 189         Some(ChunkReadInfo {
 190             range: start..end,
 191             digest: *digest,
 192         })
 193     }
 194
 195     fn compute_csum(&self) -> ([u8; 32], u64) {
 196         let mut csum = openssl::sha::Sha256::new();
 197         let mut chunk_end = 0;
 198         for pos in 0..self.index_count() {
 199             let info = self.chunk_info(pos).unwrap();
 200             chunk_end = info.range.end;
 201             csum.update(&info.digest);
 202         }
 203         let csum = csum.finish();
 204
 205         (csum, chunk_end)
 206     }
 207
 208     fn chunk_from_offset(&self, offset: u64) -> Option<(usize, u64)> {
 209         if offset >= self.size {
 210             return None;
 211         }
 212
 213         Some((
 214             (offset / self.chunk_size as u64) as usize,
 215             offset % self.chunk_size as u64
 216         ))
 217     }
 218 }
 219
 220 pub struct FixedIndexWriter {
 221     store: Arc<ChunkStore>,
 222     file: File,
 223     _lock: tools::ProcessLockSharedGuard,
 224     filename: PathBuf,
 225     tmp_filename: PathBuf,
 226     chunk_size: usize,
 227     size: usize,
 228     index_length: usize,
 229     index: *mut u8,
 230     pub uuid: [u8; 16],
 231     pub ctime: u64,
 232 }
 233
 234 // `index` is mmap()ed which cannot be thread-local so should be sendable
 235 unsafe impl Send for FixedIndexWriter {}
 236
 237 impl Drop for FixedIndexWriter {
 238     fn drop(&mut self) {
 239         let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors
 240         if let Err(err) = self.unmap() {
 241             eprintln!("Unable to unmap file {:?} - {}", self.tmp_filename, err);
 242         }
 243     }
 244 }
 245
 246 impl FixedIndexWriter {
 247     #[allow(clippy::cast_ptr_alignment)]
 248     pub fn create(
 249         store: Arc<ChunkStore>,
 250         path: &Path,
 251         size: usize,
 252         chunk_size: usize,
 253     ) -> Result<Self, Error> {
 254         let shared_lock = store.try_shared_lock()?;
 255
 256         let full_path = store.relative_path(path);
 257         let mut tmp_path = full_path.clone();
 258         tmp_path.set_extension("tmp_fidx");
 259
 260         let mut file = std::fs::OpenOptions::new()
 261             .create(true)
 262             .truncate(true)
 263             .read(true)
 264             .write(true)
 265             .open(&tmp_path)?;
 266
 267         let header_size = std::mem::size_of::<FixedIndexHeader>();
 268
 269         // todo: use static assertion when available in rust
 270         if header_size != 4096 {
 271             panic!("got unexpected header size");
 272         }
 273
 274         let ctime = epoch_now_u64()?;
 275
 276         let uuid = Uuid::generate();
 277
 278         let buffer = vec![0u8; header_size];
 279         let header = unsafe { &mut *(buffer.as_ptr() as *mut FixedIndexHeader) };
 280
 281         header.magic = super::FIXED_SIZED_CHUNK_INDEX_1_0;
 282         header.ctime = u64::to_le(ctime);
 283         header.size = u64::to_le(size as u64);
 284         header.chunk_size = u64::to_le(chunk_size as u64);
 285         header.uuid = *uuid.as_bytes();
 286
 287         header.index_csum = [0u8; 32];
 288
 289         file.write_all(&buffer)?;
 290
 291         let index_length = (size + chunk_size - 1) / chunk_size;
 292         let index_size = index_length * 32;
 293         nix::unistd::ftruncate(file.as_raw_fd(), (header_size + index_size) as i64)?;
 294
 295         let data = unsafe {
 296             nix::sys::mman::mmap(
 297                 std::ptr::null_mut(),
 298                 index_size,
 299                 nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
 300                 nix::sys::mman::MapFlags::MAP_SHARED,
 301                 file.as_raw_fd(),
 302                 header_size as i64,
 303             )
 304         }? as *mut u8;
 305
 306         Ok(Self {
 307             store,
 308             file,
 309             _lock: shared_lock,
 310             filename: full_path,
 311             tmp_filename: tmp_path,
 312             chunk_size,
 313             size,
 314             index_length,
 315             index: data,
 316             ctime,
 317             uuid: *uuid.as_bytes(),
 318         })
 319     }
 320
 321     pub fn index_length(&self) -> usize {
 322         self.index_length
 323     }
 324
 325     fn unmap(&mut self) -> Result<(), Error> {
 326         if self.index == std::ptr::null_mut() {
 327             return Ok(());
 328         }
 329
 330         let index_size = self.index_length * 32;
 331
 332         if let Err(err) =
 333             unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) }
 334         {
 335             bail!("unmap file {:?} failed - {}", self.tmp_filename, err);
 336         }
 337
 338         self.index = std::ptr::null_mut();
 339
 340         Ok(())
 341     }
 342
 343     pub fn close(&mut self) -> Result<[u8; 32], Error> {
 344         if self.index == std::ptr::null_mut() {
 345             bail!("cannot close already closed index file.");
 346         }
 347
 348         let index_size = self.index_length * 32;
 349         let data = unsafe { std::slice::from_raw_parts(self.index, index_size) };
 350         let index_csum = openssl::sha::sha256(data);
 351
 352         self.unmap()?;
 353
 354         let csum_offset = proxmox::offsetof!(FixedIndexHeader, index_csum);
 355         self.file.seek(SeekFrom::Start(csum_offset as u64))?;
 356         self.file.write_all(&index_csum)?;
 357         self.file.flush()?;
 358
 359         if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) {
 360             bail!("Atomic rename file {:?} failed - {}", self.filename, err);
 361         }
 362
 363         Ok(index_csum)
 364     }
 365
 366     pub fn check_chunk_alignment(&self, offset: usize, chunk_len: usize) -> Result<usize, Error> {
 367         if offset < chunk_len {
 368             bail!("got chunk with small offset ({} < {}", offset, chunk_len);
 369         }
 370
 371         let pos = offset - chunk_len;
 372
 373         if offset > self.size {
 374             bail!("chunk data exceeds size ({} >= {})", offset, self.size);
 375         }
 376
 377         // last chunk can be smaller
 378         if ((offset != self.size) && (chunk_len != self.chunk_size))
 379             || (chunk_len > self.chunk_size)
 380             || (chunk_len == 0)
 381         {
 382             bail!(
 383                 "chunk with unexpected length ({} != {}",
 384                 chunk_len,
 385                 self.chunk_size
 386             );
 387         }
 388
 389         if pos & (self.chunk_size - 1) != 0 {
 390             bail!("got unaligned chunk (pos = {})", pos);
 391         }
 392
 393         Ok(pos / self.chunk_size)
 394     }
 395
 396     // Note: We want to add data out of order, so do not assume any order here.
 397     pub fn add_chunk(&mut self, chunk_info: &ChunkInfo, stat: &mut ChunkStat) -> Result<(), Error> {
 398         let chunk_len = chunk_info.chunk_len as usize;
 399         let offset = chunk_info.offset as usize; // end of chunk
 400
 401         let idx = self.check_chunk_alignment(offset, chunk_len)?;
 402
 403         let (is_duplicate, compressed_size) = self
 404             .store
 405             .insert_chunk(&chunk_info.chunk, &chunk_info.digest)?;
 406
 407         stat.chunk_count += 1;
 408         stat.compressed_size += compressed_size;
 409
 410         let digest = &chunk_info.digest;
 411
 412         println!(
 413             "ADD CHUNK {} {} {}% {} {}",
 414             idx,
 415             chunk_len,
 416             (compressed_size * 100) / (chunk_len as u64),
 417             is_duplicate,
 418             proxmox::tools::digest_to_hex(digest)
 419         );
 420
 421         if is_duplicate {
 422             stat.duplicate_chunks += 1;
 423         } else {
 424             stat.disk_size += compressed_size;
 425         }
 426
 427         self.add_digest(idx, digest)
 428     }
 429
 430     pub fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> {
 431         if index >= self.index_length {
 432             bail!(
 433                 "add digest failed - index out of range ({} >= {})",
 434                 index,
 435                 self.index_length
 436             );
 437         }
 438
 439         if self.index == std::ptr::null_mut() {
 440             bail!("cannot write to closed index file.");
 441         }
 442
 443         let index_pos = index * 32;
 444         unsafe {
 445             let dst = self.index.add(index_pos);
 446             dst.copy_from_nonoverlapping(digest.as_ptr(), 32);
 447         }
 448
 449         Ok(())
 450     }
 451
 452     pub fn clone_data_from(&mut self, reader: &FixedIndexReader) -> Result<(), Error> {
 453         if self.index_length != reader.index_count() {
 454             bail!("clone_data_from failed - index sizes not equal");
 455         }
 456
 457         for i in 0..self.index_length {
 458             self.add_digest(i, reader.index_digest(i).unwrap())?;
 459         }
 460
 461         Ok(())
 462     }
 463 }