src/backup/verify.rs

   1 use std::collections::HashSet;
   2
   3 use anyhow::{bail, Error};
   4
   5 use crate::server::WorkerTask;
   6
   7 use super::{
   8     DataStore, BackupGroup, BackupDir, BackupInfo, IndexFile,
   9     ENCR_COMPR_BLOB_MAGIC_1_0, ENCRYPTED_BLOB_MAGIC_1_0,
  10     FileInfo, ArchiveType, archive_type,
  11 };
  12
  13 fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
  14
  15     let blob = datastore.load_blob(backup_dir, &info.filename)?;
  16
  17     let raw_size = blob.raw_size();
  18     if raw_size != info.size {
  19         bail!("wrong size ({} != {})", info.size, raw_size);
  20     }
  21
  22     let csum = openssl::sha::sha256(blob.raw_data());
  23     if csum != info.csum {
  24         bail!("wrong index checksum");
  25     }
  26
  27     let magic = blob.magic();
  28
  29     if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
  30         return Ok(());
  31     }
  32
  33     blob.decode(None)?;
  34
  35     Ok(())
  36 }
  37
  38 fn verify_index_chunks(
  39     datastore: &DataStore,
  40     index: Box<dyn IndexFile>,
  41     verified_chunks: &mut HashSet<[u8;32]>,
  42     worker: &WorkerTask,
  43 ) -> Result<(), Error> {
  44
  45     let mut errors = 0;
  46     for pos in 0..index.index_count() {
  47
  48         worker.fail_on_abort()?;
  49
  50         let info = index.chunk_info(pos).unwrap();
  51         let size = info.range.end - info.range.start;
  52
  53         if !verified_chunks.contains(&info.digest) {
  54             if let Err(err) = datastore.verify_stored_chunk(&info.digest, size) {
  55                 worker.log(format!("{}", err));
  56                 errors += 1;
  57             } else {
  58                 verified_chunks.insert(info.digest);
  59             }
  60         }
  61     }
  62
  63     if errors > 0 {
  64         bail!("chunks could not be verified");
  65     }
  66
  67     Ok(())
  68 }
  69
  70 fn verify_fixed_index(
  71     datastore: &DataStore,
  72     backup_dir: &BackupDir,
  73     info: &FileInfo,
  74     verified_chunks: &mut HashSet<[u8;32]>,
  75     worker: &WorkerTask,
  76 ) -> Result<(), Error> {
  77
  78     let mut path = backup_dir.relative_path();
  79     path.push(&info.filename);
  80
  81     let index = datastore.open_fixed_reader(&path)?;
  82
  83     let (csum, size) = index.compute_csum();
  84     if size != info.size {
  85         bail!("wrong size ({} != {})", info.size, size);
  86     }
  87
  88     if csum != info.csum {
  89         bail!("wrong index checksum");
  90     }
  91
  92     verify_index_chunks(datastore, Box::new(index), verified_chunks, worker)
  93 }
  94
  95 fn verify_dynamic_index(
  96     datastore: &DataStore,
  97     backup_dir: &BackupDir,
  98     info: &FileInfo,
  99     verified_chunks: &mut HashSet<[u8;32]>,
 100     worker: &WorkerTask,
 101 ) -> Result<(), Error> {
 102
 103     let mut path = backup_dir.relative_path();
 104     path.push(&info.filename);
 105
 106     let index = datastore.open_dynamic_reader(&path)?;
 107
 108     let (csum, size) = index.compute_csum();
 109     if size != info.size {
 110         bail!("wrong size ({} != {})", info.size, size);
 111     }
 112
 113     if csum != info.csum {
 114         bail!("wrong index checksum");
 115     }
 116
 117     verify_index_chunks(datastore, Box::new(index), verified_chunks, worker)
 118 }
 119
 120 /// Verify a single backup snapshot
 121 ///
 122 /// This checks all archives inside a backup snapshot.
 123 /// Errors are logged to the worker log.
 124 ///
 125 /// Returns
 126 /// - Ok(true) if verify is successful
 127 /// - Ok(false) if there were verification errors
 128 /// - Err(_) if task was aborted
 129 pub fn verify_backup_dir(
 130     datastore: &DataStore,
 131     backup_dir: &BackupDir,
 132     verified_chunks: &mut HashSet<[u8;32]>,
 133     worker: &WorkerTask
 134 ) -> Result<bool, Error> {
 135
 136     let manifest = match datastore.load_manifest(&backup_dir) {
 137         Ok((manifest, _crypt_mode, _)) => manifest,
 138         Err(err) => {
 139             worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err));
 140             return Ok(false);
 141         }
 142     };
 143
 144     worker.log(format!("verify {}:{}", datastore.name(), backup_dir));
 145
 146     let mut error_count = 0;
 147
 148     for info in manifest.files() {
 149         let result = proxmox::try_block!({
 150             worker.log(format!("  check {}", info.filename));
 151             match archive_type(&info.filename)? {
 152                 ArchiveType::FixedIndex => verify_fixed_index(&datastore, &backup_dir, info, verified_chunks, worker),
 153                 ArchiveType::DynamicIndex => verify_dynamic_index(&datastore, &backup_dir, info, verified_chunks, worker),
 154                 ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info),
 155             }
 156         });
 157
 158         worker.fail_on_abort()?;
 159
 160         if let Err(err) = result {
 161             worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err));
 162             error_count += 1;
 163         }
 164     }
 165
 166     Ok(error_count == 0)
 167 }
 168
 169 /// Verify all backups inside a backup group
 170 ///
 171 /// Errors are logged to the worker log.
 172 ///
 173 /// Returns
 174 /// - Ok(true) if verify is successful
 175 /// - Ok(false) if there were verification errors
 176 /// - Err(_) if task was aborted
 177 pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<bool, Error> {
 178
 179     let mut list = match group.list_backups(&datastore.base_path()) {
 180         Ok(list) => list,
 181         Err(err) => {
 182             worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err));
 183             return Ok(false);
 184         }
 185     };
 186
 187     worker.log(format!("verify group {}:{}", datastore.name(), group));
 188
 189     let mut error_count = 0;
 190
 191     let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB)
 192
 193     BackupInfo::sort_list(&mut list, false); // newest first
 194     for info in list {
 195         if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, worker)? {
 196             error_count += 1;
 197         }
 198     }
 199
 200     Ok(error_count == 0)
 201 }
 202
 203 /// Verify all backups inside a datastore
 204 ///
 205 /// Errors are logged to the worker log.
 206 ///
 207 /// Returns
 208 /// - Ok(true) if verify is successful
 209 /// - Ok(false) if there were verification errors
 210 /// - Err(_) if task was aborted
 211 pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<bool, Error> {
 212
 213     let list = match BackupGroup::list_groups(&datastore.base_path()) {
 214         Ok(list) => list,
 215         Err(err) => {
 216             worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
 217             return Ok(false);
 218         }
 219     };
 220
 221     worker.log(format!("verify datastore {}", datastore.name()));
 222
 223     let mut error_count = 0;
 224     for group in list {
 225         if !verify_backup_group(datastore, &group, worker)? {
 226             error_count += 1;
 227         }
 228     }
 229
 230     Ok(error_count == 0)
 231 }