src/backup/fixed_index.rs

   1 use anyhow::{bail, format_err, Error};
   2 use std::io::{Seek, SeekFrom};
   3
   4 use super::chunk_stat::*;
   5 use super::chunk_store::*;
   6 use super::{IndexFile, ChunkReadInfo};
   7 use crate::tools::{self, epoch_now_u64};
   8
   9 use chrono::{Local, LocalResult, TimeZone};
  10 use std::fs::File;
  11 use std::io::Write;
  12 use std::os::unix::io::AsRawFd;
  13 use std::path::{Path, PathBuf};
  14 use std::sync::Arc;
  15
  16 use super::ChunkInfo;
  17
  18 use proxmox::tools::io::ReadExt;
  19 use proxmox::tools::Uuid;
  20
  21 /// Header format definition for fixed index files (`.fidx`)
  22 #[repr(C)]
  23 pub struct FixedIndexHeader {
  24     pub magic: [u8; 8],
  25     pub uuid: [u8; 16],
  26     pub ctime: u64,
  27     /// Sha256 over the index ``SHA256(digest1||digest2||...)``
  28     pub index_csum: [u8; 32],
  29     pub size: u64,
  30     pub chunk_size: u64,
  31     reserved: [u8; 4016], // overall size is one page (4096 bytes)
  32 }
  33 proxmox::static_assert_size!(FixedIndexHeader, 4096);
  34
  35 // split image into fixed size chunks
  36
  37 pub struct FixedIndexReader {
  38     _file: File,
  39     pub chunk_size: usize,
  40     pub size: u64,
  41     index_length: usize,
  42     index: *mut u8,
  43     pub uuid: [u8; 16],
  44     pub ctime: u64,
  45     pub index_csum: [u8; 32],
  46 }
  47
  48 // `index` is mmap()ed which cannot be thread-local so should be sendable
  49 unsafe impl Send for FixedIndexReader {}
  50 unsafe impl Sync for FixedIndexReader {}
  51
  52 impl Drop for FixedIndexReader {
  53     fn drop(&mut self) {
  54         if let Err(err) = self.unmap() {
  55             eprintln!("Unable to unmap file - {}", err);
  56         }
  57     }
  58 }
  59
  60 impl FixedIndexReader {
  61     pub fn open(path: &Path) -> Result<Self, Error> {
  62         File::open(path)
  63             .map_err(Error::from)
  64             .and_then(|file| Self::new(file))
  65             .map_err(|err| format_err!("Unable to open fixed index {:?} - {}", path, err))
  66     }
  67
  68     pub fn new(mut file: std::fs::File) -> Result<Self, Error> {
  69         if let Err(err) =
  70             nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockSharedNonblock)
  71         {
  72             bail!("unable to get shared lock - {}", err);
  73         }
  74
  75         file.seek(SeekFrom::Start(0))?;
  76
  77         let header_size = std::mem::size_of::<FixedIndexHeader>();
  78         let header: Box<FixedIndexHeader> = unsafe { file.read_host_value_boxed()? };
  79
  80         if header.magic != super::FIXED_SIZED_CHUNK_INDEX_1_0 {
  81             bail!("got unknown magic number");
  82         }
  83
  84         let size = u64::from_le(header.size);
  85         let ctime = u64::from_le(header.ctime);
  86         let chunk_size = u64::from_le(header.chunk_size);
  87
  88         let index_length = ((size + chunk_size - 1) / chunk_size) as usize;
  89         let index_size = index_length * 32;
  90
  91         let rawfd = file.as_raw_fd();
  92
  93         let stat = match nix::sys::stat::fstat(rawfd) {
  94             Ok(stat) => stat,
  95             Err(err) => bail!("fstat failed - {}", err),
  96         };
  97
  98         let expected_index_size = (stat.st_size as usize) - header_size;
  99         if index_size != expected_index_size {
 100             bail!(
 101                 "got unexpected file size ({} != {})",
 102                 index_size,
 103                 expected_index_size
 104             );
 105         }
 106
 107         let data = unsafe {
 108             nix::sys::mman::mmap(
 109                 std::ptr::null_mut(),
 110                 index_size,
 111                 nix::sys::mman::ProtFlags::PROT_READ,
 112                 nix::sys::mman::MapFlags::MAP_PRIVATE,
 113                 file.as_raw_fd(),
 114                 header_size as i64,
 115             )
 116         }? as *mut u8;
 117
 118         Ok(Self {
 119             _file: file,
 120             chunk_size: chunk_size as usize,
 121             size,
 122             index_length,
 123             index: data,
 124             ctime,
 125             uuid: header.uuid,
 126             index_csum: header.index_csum,
 127         })
 128     }
 129
 130     fn unmap(&mut self) -> Result<(), Error> {
 131         if self.index == std::ptr::null_mut() {
 132             return Ok(());
 133         }
 134
 135         let index_size = self.index_length * 32;
 136
 137         if let Err(err) =
 138             unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) }
 139         {
 140             bail!("unmap file failed - {}", err);
 141         }
 142
 143         self.index = std::ptr::null_mut();
 144
 145         Ok(())
 146     }
 147
 148     pub fn print_info(&self) {
 149         println!("Size: {}", self.size);
 150         println!("ChunkSize: {}", self.chunk_size);
 151         println!(
 152             "CTime: {}",
 153             match Local.timestamp_opt(self.ctime as i64, 0) {
 154                 LocalResult::Single(ctime) => ctime.format("%c").to_string(),
 155                 _ => (self.ctime as i64).to_string(),
 156             }
 157         );
 158         println!("UUID: {:?}", self.uuid);
 159     }
 160 }
 161
 162 impl IndexFile for FixedIndexReader {
 163     fn index_count(&self) -> usize {
 164         self.index_length
 165     }
 166
 167     fn index_digest(&self, pos: usize) -> Option<&[u8; 32]> {
 168         if pos >= self.index_length {
 169             None
 170         } else {
 171             Some(unsafe { std::mem::transmute(self.index.add(pos * 32)) })
 172         }
 173     }
 174
 175     fn index_bytes(&self) -> u64 {
 176         self.size
 177     }
 178
 179     fn chunk_info(&self, pos: usize) -> Option<ChunkReadInfo> {
 180         if pos >= self.index_length {
 181             return None;
 182         }
 183
 184         let start = (pos * self.chunk_size) as u64;
 185         let mut end = start + self.chunk_size as u64;
 186
 187         if end > self.size {
 188             end = self.size;
 189         }
 190
 191         let digest = self.index_digest(pos).unwrap();
 192         Some(ChunkReadInfo {
 193             range: start..end,
 194             digest: *digest,
 195         })
 196     }
 197
 198     fn compute_csum(&self) -> ([u8; 32], u64) {
 199         let mut csum = openssl::sha::Sha256::new();
 200         let mut chunk_end = 0;
 201         for pos in 0..self.index_count() {
 202             let info = self.chunk_info(pos).unwrap();
 203             chunk_end = info.range.end;
 204             csum.update(&info.digest);
 205         }
 206         let csum = csum.finish();
 207
 208         (csum, chunk_end)
 209     }
 210
 211     fn chunk_from_offset(&self, offset: u64) -> Option<(usize, u64)> {
 212         if offset >= self.size {
 213             return None;
 214         }
 215
 216         Some((
 217             (offset / self.chunk_size as u64) as usize,
 218             offset & (self.chunk_size - 1) as u64 // fast modulo, valid for 2^x chunk_size
 219         ))
 220     }
 221 }
 222
 223 pub struct FixedIndexWriter {
 224     store: Arc<ChunkStore>,
 225     file: File,
 226     _lock: tools::ProcessLockSharedGuard,
 227     filename: PathBuf,
 228     tmp_filename: PathBuf,
 229     chunk_size: usize,
 230     size: usize,
 231     index_length: usize,
 232     index: *mut u8,
 233     pub uuid: [u8; 16],
 234     pub ctime: u64,
 235 }
 236
 237 // `index` is mmap()ed which cannot be thread-local so should be sendable
 238 unsafe impl Send for FixedIndexWriter {}
 239
 240 impl Drop for FixedIndexWriter {
 241     fn drop(&mut self) {
 242         let _ = std::fs::remove_file(&self.tmp_filename); // ignore errors
 243         if let Err(err) = self.unmap() {
 244             eprintln!("Unable to unmap file {:?} - {}", self.tmp_filename, err);
 245         }
 246     }
 247 }
 248
 249 impl FixedIndexWriter {
 250     #[allow(clippy::cast_ptr_alignment)]
 251     pub fn create(
 252         store: Arc<ChunkStore>,
 253         path: &Path,
 254         size: usize,
 255         chunk_size: usize,
 256     ) -> Result<Self, Error> {
 257         let shared_lock = store.try_shared_lock()?;
 258
 259         let full_path = store.relative_path(path);
 260         let mut tmp_path = full_path.clone();
 261         tmp_path.set_extension("tmp_fidx");
 262
 263         let mut file = std::fs::OpenOptions::new()
 264             .create(true)
 265             .truncate(true)
 266             .read(true)
 267             .write(true)
 268             .open(&tmp_path)?;
 269
 270         let header_size = std::mem::size_of::<FixedIndexHeader>();
 271
 272         // todo: use static assertion when available in rust
 273         if header_size != 4096 {
 274             panic!("got unexpected header size");
 275         }
 276
 277         let ctime = epoch_now_u64()?;
 278
 279         let uuid = Uuid::generate();
 280
 281         let buffer = vec![0u8; header_size];
 282         let header = unsafe { &mut *(buffer.as_ptr() as *mut FixedIndexHeader) };
 283
 284         header.magic = super::FIXED_SIZED_CHUNK_INDEX_1_0;
 285         header.ctime = u64::to_le(ctime);
 286         header.size = u64::to_le(size as u64);
 287         header.chunk_size = u64::to_le(chunk_size as u64);
 288         header.uuid = *uuid.as_bytes();
 289
 290         header.index_csum = [0u8; 32];
 291
 292         file.write_all(&buffer)?;
 293
 294         let index_length = (size + chunk_size - 1) / chunk_size;
 295         let index_size = index_length * 32;
 296         nix::unistd::ftruncate(file.as_raw_fd(), (header_size + index_size) as i64)?;
 297
 298         let data = unsafe {
 299             nix::sys::mman::mmap(
 300                 std::ptr::null_mut(),
 301                 index_size,
 302                 nix::sys::mman::ProtFlags::PROT_READ | nix::sys::mman::ProtFlags::PROT_WRITE,
 303                 nix::sys::mman::MapFlags::MAP_SHARED,
 304                 file.as_raw_fd(),
 305                 header_size as i64,
 306             )
 307         }? as *mut u8;
 308
 309         Ok(Self {
 310             store,
 311             file,
 312             _lock: shared_lock,
 313             filename: full_path,
 314             tmp_filename: tmp_path,
 315             chunk_size,
 316             size,
 317             index_length,
 318             index: data,
 319             ctime,
 320             uuid: *uuid.as_bytes(),
 321         })
 322     }
 323
 324     pub fn index_length(&self) -> usize {
 325         self.index_length
 326     }
 327
 328     fn unmap(&mut self) -> Result<(), Error> {
 329         if self.index == std::ptr::null_mut() {
 330             return Ok(());
 331         }
 332
 333         let index_size = self.index_length * 32;
 334
 335         if let Err(err) =
 336             unsafe { nix::sys::mman::munmap(self.index as *mut std::ffi::c_void, index_size) }
 337         {
 338             bail!("unmap file {:?} failed - {}", self.tmp_filename, err);
 339         }
 340
 341         self.index = std::ptr::null_mut();
 342
 343         Ok(())
 344     }
 345
 346     pub fn close(&mut self) -> Result<[u8; 32], Error> {
 347         if self.index == std::ptr::null_mut() {
 348             bail!("cannot close already closed index file.");
 349         }
 350
 351         let index_size = self.index_length * 32;
 352         let data = unsafe { std::slice::from_raw_parts(self.index, index_size) };
 353         let index_csum = openssl::sha::sha256(data);
 354
 355         self.unmap()?;
 356
 357         let csum_offset = proxmox::offsetof!(FixedIndexHeader, index_csum);
 358         self.file.seek(SeekFrom::Start(csum_offset as u64))?;
 359         self.file.write_all(&index_csum)?;
 360         self.file.flush()?;
 361
 362         if let Err(err) = std::fs::rename(&self.tmp_filename, &self.filename) {
 363             bail!("Atomic rename file {:?} failed - {}", self.filename, err);
 364         }
 365
 366         Ok(index_csum)
 367     }
 368
 369     pub fn check_chunk_alignment(&self, offset: usize, chunk_len: usize) -> Result<usize, Error> {
 370         if offset < chunk_len {
 371             bail!("got chunk with small offset ({} < {}", offset, chunk_len);
 372         }
 373
 374         let pos = offset - chunk_len;
 375
 376         if offset > self.size {
 377             bail!("chunk data exceeds size ({} >= {})", offset, self.size);
 378         }
 379
 380         // last chunk can be smaller
 381         if ((offset != self.size) && (chunk_len != self.chunk_size))
 382             || (chunk_len > self.chunk_size)
 383             || (chunk_len == 0)
 384         {
 385             bail!(
 386                 "chunk with unexpected length ({} != {}",
 387                 chunk_len,
 388                 self.chunk_size
 389             );
 390         }
 391
 392         if pos & (self.chunk_size - 1) != 0 {
 393             bail!("got unaligned chunk (pos = {})", pos);
 394         }
 395
 396         Ok(pos / self.chunk_size)
 397     }
 398
 399     // Note: We want to add data out of order, so do not assume any order here.
 400     pub fn add_chunk(&mut self, chunk_info: &ChunkInfo, stat: &mut ChunkStat) -> Result<(), Error> {
 401         let chunk_len = chunk_info.chunk_len as usize;
 402         let offset = chunk_info.offset as usize; // end of chunk
 403
 404         let idx = self.check_chunk_alignment(offset, chunk_len)?;
 405
 406         let (is_duplicate, compressed_size) = self
 407             .store
 408             .insert_chunk(&chunk_info.chunk, &chunk_info.digest)?;
 409
 410         stat.chunk_count += 1;
 411         stat.compressed_size += compressed_size;
 412
 413         let digest = &chunk_info.digest;
 414
 415         println!(
 416             "ADD CHUNK {} {} {}% {} {}",
 417             idx,
 418             chunk_len,
 419             (compressed_size * 100) / (chunk_len as u64),
 420             is_duplicate,
 421             proxmox::tools::digest_to_hex(digest)
 422         );
 423
 424         if is_duplicate {
 425             stat.duplicate_chunks += 1;
 426         } else {
 427             stat.disk_size += compressed_size;
 428         }
 429
 430         self.add_digest(idx, digest)
 431     }
 432
 433     pub fn add_digest(&mut self, index: usize, digest: &[u8; 32]) -> Result<(), Error> {
 434         if index >= self.index_length {
 435             bail!(
 436                 "add digest failed - index out of range ({} >= {})",
 437                 index,
 438                 self.index_length
 439             );
 440         }
 441
 442         if self.index == std::ptr::null_mut() {
 443             bail!("cannot write to closed index file.");
 444         }
 445
 446         let index_pos = index * 32;
 447         unsafe {
 448             let dst = self.index.add(index_pos);
 449             dst.copy_from_nonoverlapping(digest.as_ptr(), 32);
 450         }
 451
 452         Ok(())
 453     }
 454
 455     pub fn clone_data_from(&mut self, reader: &FixedIndexReader) -> Result<(), Error> {
 456         if self.index_length != reader.index_count() {
 457             bail!("clone_data_from failed - index sizes not equal");
 458         }
 459
 460         for i in 0..self.index_length {
 461             self.add_digest(i, reader.index_digest(i).unwrap())?;
 462         }
 463
 464         Ok(())
 465     }
 466 }