1 use std
::collections
::HashSet
;
3 use anyhow
::{bail, Error}
;
5 use crate::server
::WorkerTask
;
8 DataStore
, BackupGroup
, BackupDir
, BackupInfo
, IndexFile
,
9 ENCR_COMPR_BLOB_MAGIC_1_0
, ENCRYPTED_BLOB_MAGIC_1_0
,
10 FileInfo
, ArchiveType
, archive_type
,
13 fn verify_blob(datastore
: &DataStore
, backup_dir
: &BackupDir
, info
: &FileInfo
) -> Result
<(), Error
> {
15 let blob
= datastore
.load_blob(backup_dir
, &info
.filename
)?
;
17 let raw_size
= blob
.raw_size();
18 if raw_size
!= info
.size
{
19 bail
!("wrong size ({} != {})", info
.size
, raw_size
);
22 let csum
= openssl
::sha
::sha256(blob
.raw_data());
23 if csum
!= info
.csum
{
24 bail
!("wrong index checksum");
27 let magic
= blob
.magic();
29 if magic
== &ENCR_COMPR_BLOB_MAGIC_1_0
|| magic
== &ENCRYPTED_BLOB_MAGIC_1_0
{
38 fn verify_index_chunks(
39 datastore
: &DataStore
,
40 index
: Box
<dyn IndexFile
>,
41 verified_chunks
: &mut HashSet
<[u8;32]>,
42 corrupt_chunks
: &mut HashSet
<[u8; 32]>,
44 ) -> Result
<(), Error
> {
47 for pos
in 0..index
.index_count() {
49 worker
.fail_on_abort()?
;
51 let info
= index
.chunk_info(pos
).unwrap();
52 let size
= info
.range
.end
- info
.range
.start
;
54 if !verified_chunks
.contains(&info
.digest
) {
55 if !corrupt_chunks
.contains(&info
.digest
) {
56 if let Err(err
) = datastore
.verify_stored_chunk(&info
.digest
, size
) {
57 corrupt_chunks
.insert(info
.digest
);
58 worker
.log(format
!("{}", err
));
61 verified_chunks
.insert(info
.digest
);
64 let digest_str
= proxmox
::tools
::digest_to_hex(&info
.digest
);
65 worker
.log(format
!("chunk {} was marked as corrupt", digest_str
));
72 bail
!("chunks could not be verified");
78 fn verify_fixed_index(
79 datastore
: &DataStore
,
80 backup_dir
: &BackupDir
,
82 verified_chunks
: &mut HashSet
<[u8;32]>,
83 corrupt_chunks
: &mut HashSet
<[u8;32]>,
85 ) -> Result
<(), Error
> {
87 let mut path
= backup_dir
.relative_path();
88 path
.push(&info
.filename
);
90 let index
= datastore
.open_fixed_reader(&path
)?
;
92 let (csum
, size
) = index
.compute_csum();
93 if size
!= info
.size
{
94 bail
!("wrong size ({} != {})", info
.size
, size
);
97 if csum
!= info
.csum
{
98 bail
!("wrong index checksum");
101 verify_index_chunks(datastore
, Box
::new(index
), verified_chunks
, corrupt_chunks
, worker
)
104 fn verify_dynamic_index(
105 datastore
: &DataStore
,
106 backup_dir
: &BackupDir
,
108 verified_chunks
: &mut HashSet
<[u8;32]>,
109 corrupt_chunks
: &mut HashSet
<[u8;32]>,
111 ) -> Result
<(), Error
> {
113 let mut path
= backup_dir
.relative_path();
114 path
.push(&info
.filename
);
116 let index
= datastore
.open_dynamic_reader(&path
)?
;
118 let (csum
, size
) = index
.compute_csum();
119 if size
!= info
.size
{
120 bail
!("wrong size ({} != {})", info
.size
, size
);
123 if csum
!= info
.csum
{
124 bail
!("wrong index checksum");
127 verify_index_chunks(datastore
, Box
::new(index
), verified_chunks
, corrupt_chunks
, worker
)
130 /// Verify a single backup snapshot
132 /// This checks all archives inside a backup snapshot.
133 /// Errors are logged to the worker log.
136 /// - Ok(true) if verify is successful
137 /// - Ok(false) if there were verification errors
138 /// - Err(_) if task was aborted
139 pub fn verify_backup_dir(
140 datastore
: &DataStore
,
141 backup_dir
: &BackupDir
,
142 verified_chunks
: &mut HashSet
<[u8;32]>,
143 corrupt_chunks
: &mut HashSet
<[u8;32]>,
145 ) -> Result
<bool
, Error
> {
147 let manifest
= match datastore
.load_manifest(&backup_dir
) {
148 Ok((manifest
, _crypt_mode
, _
)) => manifest
,
150 worker
.log(format
!("verify {}:{} - manifest load error: {}", datastore
.name(), backup_dir
, err
));
155 worker
.log(format
!("verify {}:{}", datastore
.name(), backup_dir
));
157 let mut error_count
= 0;
159 for info
in manifest
.files() {
160 let result
= proxmox
::try_block
!({
161 worker
.log(format
!(" check {}", info
.filename
));
162 match archive_type(&info
.filename
)?
{
163 ArchiveType
::FixedIndex
=>
172 ArchiveType
::DynamicIndex
=>
173 verify_dynamic_index(
181 ArchiveType
::Blob
=> verify_blob(&datastore
, &backup_dir
, info
),
185 worker
.fail_on_abort()?
;
187 if let Err(err
) = result
{
188 worker
.log(format
!("verify {}:{}/{} failed: {}", datastore
.name(), backup_dir
, info
.filename
, err
));
196 /// Verify all backups inside a backup group
198 /// Errors are logged to the worker log.
201 /// - Ok(failed_dirs) where failed_dirs had verification errors
202 /// - Err(_) if task was aborted
203 pub fn verify_backup_group(datastore
: &DataStore
, group
: &BackupGroup
, worker
: &WorkerTask
) -> Result
<Vec
<String
>, Error
> {
205 let mut errors
= Vec
::new();
206 let mut list
= match group
.list_backups(&datastore
.base_path()) {
209 worker
.log(format
!("verify group {}:{} - unable to list backups: {}", datastore
.name(), group
, err
));
214 worker
.log(format
!("verify group {}:{}", datastore
.name(), group
));
216 let mut verified_chunks
= HashSet
::with_capacity(1024*16); // start with 16384 chunks (up to 65GB)
217 let mut corrupt_chunks
= HashSet
::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones
219 BackupInfo
::sort_list(&mut list
, false); // newest first
221 if !verify_backup_dir(datastore
, &info
.backup_dir
, &mut verified_chunks
, &mut corrupt_chunks
, worker
)?
{
222 errors
.push(info
.backup_dir
.to_string());
229 /// Verify all backups inside a datastore
231 /// Errors are logged to the worker log.
234 /// - Ok(failed_dirs) where failed_dirs had verification errors
235 /// - Err(_) if task was aborted
236 pub fn verify_all_backups(datastore
: &DataStore
, worker
: &WorkerTask
) -> Result
<Vec
<String
>, Error
> {
238 let mut errors
= Vec
::new();
240 let list
= match BackupGroup
::list_groups(&datastore
.base_path()) {
243 worker
.log(format
!("verify datastore {} - unable to list backups: {}", datastore
.name(), err
));
248 worker
.log(format
!("verify datastore {}", datastore
.name()));
251 let mut group_errors
= verify_backup_group(datastore
, &group
, worker
)?
;
252 errors
.append(&mut group_errors
);