src/backup/verify.rs

   1 use std::collections::HashSet;
   2
   3 use anyhow::{bail, format_err, Error};
   4
   5 use crate::server::WorkerTask;
   6 use crate::api2::types::*;
   7
   8 use super::{
   9     DataStore, BackupGroup, BackupDir, BackupInfo, IndexFile,
  10     CryptMode,
  11     FileInfo, ArchiveType, archive_type,
  12 };
  13
  14 fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> {
  15
  16     let blob = datastore.load_blob(backup_dir, &info.filename)?;
  17
  18     let raw_size = blob.raw_size();
  19     if raw_size != info.size {
  20         bail!("wrong size ({} != {})", info.size, raw_size);
  21     }
  22
  23     let csum = openssl::sha::sha256(blob.raw_data());
  24     if csum != info.csum {
  25         bail!("wrong index checksum");
  26     }
  27
  28     match blob.crypt_mode()? {
  29         CryptMode::Encrypt => Ok(()),
  30         CryptMode::None => {
  31             // digest already verified above
  32             blob.decode(None, None)?;
  33             Ok(())
  34         },
  35         CryptMode::SignOnly => bail!("Invalid CryptMode for blob"),
  36     }
  37 }
  38
  39 fn verify_index_chunks(
  40     datastore: &DataStore,
  41     index: Box<dyn IndexFile>,
  42     verified_chunks: &mut HashSet<[u8;32]>,
  43     corrupt_chunks: &mut HashSet<[u8; 32]>,
  44     crypt_mode: CryptMode,
  45     worker: &WorkerTask,
  46 ) -> Result<(), Error> {
  47
  48     let mut errors = 0;
  49     for pos in 0..index.index_count() {
  50
  51         worker.fail_on_abort()?;
  52
  53         let info = index.chunk_info(pos).unwrap();
  54
  55         if verified_chunks.contains(&info.digest) {
  56             continue; // already verified
  57         }
  58
  59         if corrupt_chunks.contains(&info.digest) {
  60             let digest_str = proxmox::tools::digest_to_hex(&info.digest);
  61             worker.log(format!("chunk {} was marked as corrupt", digest_str));
  62             errors += 1;
  63             continue;
  64         }
  65
  66         let chunk = match datastore.load_chunk(&info.digest) {
  67             Err(err) => {
  68                 corrupt_chunks.insert(info.digest);
  69                 worker.log(format!("can't verify chunk, load failed - {}", err));
  70                 errors += 1;
  71                 continue;
  72             },
  73             Ok(chunk) => chunk,
  74         };
  75
  76         let chunk_crypt_mode = match chunk.crypt_mode() {
  77             Err(err) => {
  78                 corrupt_chunks.insert(info.digest);
  79                 worker.log(format!("can't verify chunk, unknown CryptMode - {}", err));
  80                 errors += 1;
  81                 continue;
  82             },
  83             Ok(mode) => mode,
  84         };
  85
  86         if chunk_crypt_mode != crypt_mode {
  87             worker.log(format!(
  88                 "chunk CryptMode {:?} does not match index CryptMode {:?}",
  89                 chunk_crypt_mode,
  90                 crypt_mode
  91             ));
  92             errors += 1;
  93         }
  94
  95         let size = info.range.end - info.range.start;
  96
  97         if let Err(err) = chunk.verify_unencrypted(size as usize, &info.digest) {
  98             corrupt_chunks.insert(info.digest);
  99             worker.log(format!("{}", err));
 100             errors += 1;
 101         } else {
 102             verified_chunks.insert(info.digest);
 103         }
 104     }
 105
 106     if errors > 0 {
 107         bail!("chunks could not be verified");
 108     }
 109
 110     Ok(())
 111 }
 112
 113 fn verify_fixed_index(
 114     datastore: &DataStore,
 115     backup_dir: &BackupDir,
 116     info: &FileInfo,
 117     verified_chunks: &mut HashSet<[u8;32]>,
 118     corrupt_chunks: &mut HashSet<[u8;32]>,
 119     worker: &WorkerTask,
 120 ) -> Result<(), Error> {
 121
 122     let mut path = backup_dir.relative_path();
 123     path.push(&info.filename);
 124
 125     let index = datastore.open_fixed_reader(&path)?;
 126
 127     let (csum, size) = index.compute_csum();
 128     if size != info.size {
 129         bail!("wrong size ({} != {})", info.size, size);
 130     }
 131
 132     if csum != info.csum {
 133         bail!("wrong index checksum");
 134     }
 135
 136     verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, info.chunk_crypt_mode(), worker)
 137 }
 138
 139 fn verify_dynamic_index(
 140     datastore: &DataStore,
 141     backup_dir: &BackupDir,
 142     info: &FileInfo,
 143     verified_chunks: &mut HashSet<[u8;32]>,
 144     corrupt_chunks: &mut HashSet<[u8;32]>,
 145     worker: &WorkerTask,
 146 ) -> Result<(), Error> {
 147
 148     let mut path = backup_dir.relative_path();
 149     path.push(&info.filename);
 150
 151     let index = datastore.open_dynamic_reader(&path)?;
 152
 153     let (csum, size) = index.compute_csum();
 154     if size != info.size {
 155         bail!("wrong size ({} != {})", info.size, size);
 156     }
 157
 158     if csum != info.csum {
 159         bail!("wrong index checksum");
 160     }
 161
 162     verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, info.chunk_crypt_mode(), worker)
 163 }
 164
 165 /// Verify a single backup snapshot
 166 ///
 167 /// This checks all archives inside a backup snapshot.
 168 /// Errors are logged to the worker log.
 169 ///
 170 /// Returns
 171 /// - Ok(true) if verify is successful
 172 /// - Ok(false) if there were verification errors
 173 /// - Err(_) if task was aborted
 174 pub fn verify_backup_dir(
 175     datastore: &DataStore,
 176     backup_dir: &BackupDir,
 177     verified_chunks: &mut HashSet<[u8;32]>,
 178     corrupt_chunks: &mut HashSet<[u8;32]>,
 179     worker: &WorkerTask
 180 ) -> Result<bool, Error> {
 181
 182     let mut manifest = match datastore.load_manifest(&backup_dir) {
 183         Ok((manifest, _)) => manifest,
 184         Err(err) => {
 185             worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err));
 186             return Ok(false);
 187         }
 188     };
 189
 190     worker.log(format!("verify {}:{}", datastore.name(), backup_dir));
 191
 192     let mut error_count = 0;
 193
 194     let mut verify_result = "ok";
 195     for info in manifest.files() {
 196         let result = proxmox::try_block!({
 197             worker.log(format!("  check {}", info.filename));
 198             match archive_type(&info.filename)? {
 199                 ArchiveType::FixedIndex =>
 200                     verify_fixed_index(
 201                         &datastore,
 202                         &backup_dir,
 203                         info,
 204                         verified_chunks,
 205                         corrupt_chunks,
 206                         worker
 207                     ),
 208                 ArchiveType::DynamicIndex =>
 209                     verify_dynamic_index(
 210                         &datastore,
 211                         &backup_dir,
 212                         info,
 213                         verified_chunks,
 214                         corrupt_chunks,
 215                         worker
 216                     ),
 217                 ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info),
 218             }
 219         });
 220
 221         worker.fail_on_abort()?;
 222
 223         if let Err(err) = result {
 224             worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err));
 225             error_count += 1;
 226             verify_result = "failed";
 227         }
 228
 229     }
 230
 231     let verify_state = SnapshotVerifyState {
 232         state: verify_result.to_string(),
 233         upid: worker.upid().clone(),
 234     };
 235     manifest.unprotected["verify_state"] = serde_json::to_value(verify_state)?;
 236     datastore.store_manifest(&backup_dir, serde_json::to_value(manifest)?)
 237         .map_err(|err| format_err!("unable to store manifest blob - {}", err))?;
 238
 239
 240     Ok(error_count == 0)
 241 }
 242
 243 /// Verify all backups inside a backup group
 244 ///
 245 /// Errors are logged to the worker log.
 246 ///
 247 /// Returns
 248 /// - Ok(failed_dirs) where failed_dirs had verification errors
 249 /// - Err(_) if task was aborted
 250 pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<Vec<String>, Error> {
 251
 252     let mut errors = Vec::new();
 253     let mut list = match group.list_backups(&datastore.base_path()) {
 254         Ok(list) => list,
 255         Err(err) => {
 256             worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err));
 257             return Ok(errors);
 258         }
 259     };
 260
 261     worker.log(format!("verify group {}:{}", datastore.name(), group));
 262
 263     let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB)
 264     let mut corrupt_chunks = HashSet::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones
 265
 266     BackupInfo::sort_list(&mut list, false); // newest first
 267     for info in list {
 268         if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, &mut corrupt_chunks, worker)?{
 269             errors.push(info.backup_dir.to_string());
 270         }
 271     }
 272
 273     Ok(errors)
 274 }
 275
 276 /// Verify all backups inside a datastore
 277 ///
 278 /// Errors are logged to the worker log.
 279 ///
 280 /// Returns
 281 /// - Ok(failed_dirs) where failed_dirs had verification errors
 282 /// - Err(_) if task was aborted
 283 pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<Vec<String>, Error> {
 284
 285     let mut errors = Vec::new();
 286
 287     let mut list = match BackupGroup::list_groups(&datastore.base_path()) {
 288         Ok(list) => list,
 289         Err(err) => {
 290             worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err));
 291             return Ok(errors);
 292         }
 293     };
 294
 295     list.sort_unstable();
 296
 297     worker.log(format!("verify datastore {}", datastore.name()));
 298
 299     for group in list {
 300         let mut group_errors = verify_backup_group(datastore, &group, worker)?;
 301         errors.append(&mut group_errors);
 302     }
 303
 304     Ok(errors)
 305 }