src/backup/verify.rs

   1 use std::collections::HashSet;
   2
   3 use anyhow::{bail, Error};
   4
   5 use crate::server::WorkerTask;
   6
   7 use super::{
   8     DataStore, BackupGroup, BackupDir, BackupInfo, IndexFile,
   9     ENCR_COMPR_BLOB_MAGIC_1_0, ENCRYPTED_BLOB_MAGIC_1_0,
  10     FileInfo, ArchiveType, archive_type,
  11 };
  12
  13 fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
  14
  15     let blob = datastore.load_blob(backup_dir, &info.filename)?;
  16
  17     let raw_size = blob.raw_size();
  18     if raw_size != info.size {
  19         bail!("wrong size ({} != {})", info.size, raw_size);
  20     }
  21
  22     let csum = openssl::sha::sha256(blob.raw_data());
  23     if csum != info.csum {
  24         bail!("wrong index checksum");
  25     }
  26
  27     let magic = blob.magic();
  28
  29     if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
  30         return Ok(());
  31     }
  32
  33     blob.decode(None)?;
  34
  35     Ok(())
  36 }
  37
  38 fn verify_index_chunks(
  39     datastore: &DataStore,
  40     index: Box<dyn IndexFile>,
  41     verified_chunks: &mut HashSet<[u8;32]>,
  42     corrupt_chunks: &mut HashSet<[u8; 32]>,
  43     worker: &WorkerTask,
  44 ) -> Result<(), Error> {
  45
  46     let mut errors = 0;
  47     for pos in 0..index.index_count() {
  48
  49         worker.fail_on_abort()?;
  50
  51         let info = index.chunk_info(pos).unwrap();
  52         let size = info.range.end - info.range.start;
  53
  54         if !verified_chunks.contains(&info.digest) {
  55             if !corrupt_chunks.contains(&info.digest) {
  56                 if let Err(err) = datastore.verify_stored_chunk(&info.digest, size) {
  57                     corrupt_chunks.insert(info.digest);
  58                     worker.log(format!("{}", err));
  59                     errors += 1;
  60                 } else {
  61                     verified_chunks.insert(info.digest);
  62                 }
  63             } else {
  64                 let digest_str = proxmox::tools::digest_to_hex(&info.digest);
  65                 worker.log(format!("chunk {} was marked as corrupt", digest_str));
  66                 errors += 1;
  67             }
  68         }
  69     }
  70
  71     if errors > 0 {
  72         bail!("chunks could not be verified");
  73     }
  74
  75     Ok(())
  76 }
  77
  78 fn verify_fixed_index(
  79     datastore: &DataStore,
  80     backup_dir: &BackupDir,
  81     info: &FileInfo,
  82     verified_chunks: &mut HashSet<[u8;32]>,
  83     corrupt_chunks: &mut HashSet<[u8;32]>,
  84     worker: &WorkerTask,
  85 ) -> Result<(), Error> {
  86
  87     let mut path = backup_dir.relative_path();
  88     path.push(&info.filename);
  89
  90     let index = datastore.open_fixed_reader(&path)?;
  91
  92     let (csum, size) = index.compute_csum();
  93     if size != info.size {
  94         bail!("wrong size ({} != {})", info.size, size);
  95     }
  96
  97     if csum != info.csum {
  98         bail!("wrong index checksum");
  99     }
 100
 101     verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, worker)
 102 }
 103
 104 fn verify_dynamic_index(
 105     datastore: &DataStore,
 106     backup_dir: &BackupDir,
 107     info: &FileInfo,
 108     verified_chunks: &mut HashSet<[u8;32]>,
 109     corrupt_chunks: &mut HashSet<[u8;32]>,
 110     worker: &WorkerTask,
 111 ) -> Result<(), Error> {
 112
 113     let mut path = backup_dir.relative_path();
 114     path.push(&info.filename);
 115
 116     let index = datastore.open_dynamic_reader(&path)?;
 117
 118     let (csum, size) = index.compute_csum();
 119     if size != info.size {
 120         bail!("wrong size ({} != {})", info.size, size);
 121     }
 122
 123     if csum != info.csum {
 124         bail!("wrong index checksum");
 125     }
 126
 127     verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, worker)
 128 }
 129
 130 /// Verify a single backup snapshot
 131 ///
 132 /// This checks all archives inside a backup snapshot.
 133 /// Errors are logged to the worker log.
 134 ///
 135 /// Returns
 136 /// - Ok(true) if verify is successful
 137 /// - Ok(false) if there were verification errors
 138 /// - Err(_) if task was aborted
 139 pub fn verify_backup_dir(
 140     datastore: &DataStore,
 141     backup_dir: &BackupDir,
 142     verified_chunks: &mut HashSet<[u8;32]>,
 143     corrupt_chunks: &mut HashSet<[u8;32]>,
 144     worker: &WorkerTask
 145 ) -> Result<bool, Error> {
 146
 147     let manifest = match datastore.load_manifest(&backup_dir) {
 148         Ok((manifest, _)) => manifest,
 149         Err(err) => {
 150             worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err));
 151             return Ok(false);
 152         }
 153     };
 154
 155     worker.log(format!("verify {}:{}", datastore.name(), backup_dir));
 156
 157     let mut error_count = 0;
 158
 159     for info in manifest.files() {
 160         let result = proxmox::try_block!({
 161             worker.log(format!("  check {}", info.filename));
 162             match archive_type(&info.filename)? {
 163                 ArchiveType::FixedIndex =>
 164                     verify_fixed_index(
 165                         &datastore,
 166                         &backup_dir,
 167                         info,
 168                         verified_chunks,
 169                         corrupt_chunks,
 170                         worker
 171                     ),
 172                 ArchiveType::DynamicIndex =>
 173                     verify_dynamic_index(
 174                         &datastore,
 175                         &backup_dir,
 176                         info,
 177                         verified_chunks,
 178                         corrupt_chunks,
 179                         worker
 180                     ),
 181                 ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info),
 182             }
 183         });
 184
 185         worker.fail_on_abort()?;
 186
 187         if let Err(err) = result {
 188             worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err));
 189             error_count += 1;
 190         }
 191     }
 192
 193     Ok(error_count == 0)
 194 }
 195
 196 /// Verify all backups inside a backup group
 197 ///
 198 /// Errors are logged to the worker log.
 199 ///
 200 /// Returns
 201 /// - Ok(failed_dirs) where failed_dirs had verification errors
 202 /// - Err(_) if task was aborted
 203 pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<Vec<String>, Error> {
 204
 205     let mut errors = Vec::new();
 206     let mut list = match group.list_backups(&datastore.base_path()) {
 207         Ok(list) => list,
 208         Err(err) => {
 209             worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err));
 210             return Ok(errors);
 211         }
 212     };
 213
 214     worker.log(format!("verify group {}:{}", datastore.name(), group));
 215
 216     let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB)
 217     let mut corrupt_chunks = HashSet::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones
 218
 219     BackupInfo::sort_list(&mut list, false); // newest first
 220     for info in list {
 221         if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, &mut corrupt_chunks, worker)?{
 222             errors.push(info.backup_dir.to_string());
 223         }
 224     }
 225
 226     Ok(errors)
 227 }
 228
 229 /// Verify all backups inside a datastore
 230 ///
 231 /// Errors are logged to the worker log.
 232 ///
 233 /// Returns
 234 /// - Ok(failed_dirs) where failed_dirs had verification errors
 235 /// - Err(_) if task was aborted
 236 pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<Vec<String>, Error> {
 237
 238     let mut errors = Vec::new();
 239
 240     let list = match BackupGroup::list_groups(&datastore.base_path()) {
 241         Ok(list) => list,
 242         Err(err) => {
 243             worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
 244             return Ok(errors);
 245         }
 246     };
 247
 248     worker.log(format!("verify datastore {}", datastore.name()));
 249
 250     for group in list {
 251         let mut group_errors = verify_backup_group(datastore, &group, worker)?;
 252         errors.append(&mut group_errors);
 253     }
 254
 255     Ok(errors)
 256 }