]>
Commit | Line | Data |
---|---|---|
2aaae970 DM |
1 | use std::collections::HashSet; |
2 | ||
c2009e53 DM |
3 | use anyhow::{bail, Error}; |
4 | ||
5 | use crate::server::WorkerTask; | |
6 | ||
7 | use super::{ | |
8 | DataStore, BackupGroup, BackupDir, BackupInfo, IndexFile, | |
8819d1f2 | 9 | CryptMode, |
c2009e53 DM |
10 | FileInfo, ArchiveType, archive_type, |
11 | }; | |
12 | ||
13 | fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> { | |
14 | ||
39f18b30 | 15 | let blob = datastore.load_blob(backup_dir, &info.filename)?; |
c2009e53 | 16 | |
2aaae970 | 17 | let raw_size = blob.raw_size(); |
c2009e53 DM |
18 | if raw_size != info.size { |
19 | bail!("wrong size ({} != {})", info.size, raw_size); | |
20 | } | |
21 | ||
39f18b30 | 22 | let csum = openssl::sha::sha256(blob.raw_data()); |
c2009e53 DM |
23 | if csum != info.csum { |
24 | bail!("wrong index checksum"); | |
25 | } | |
26 | ||
8819d1f2 FG |
27 | match blob.crypt_mode()? { |
28 | CryptMode::Encrypt => Ok(()), | |
29 | CryptMode::None => { | |
30 | // digest already verified above | |
31 | blob.decode(None, None)?; | |
32 | Ok(()) | |
33 | }, | |
34 | CryptMode::SignOnly => bail!("Invalid CryptMode for blob"), | |
c2009e53 | 35 | } |
c2009e53 DM |
36 | } |
37 | ||
fdaab0df DM |
38 | fn verify_index_chunks( |
39 | datastore: &DataStore, | |
40 | index: Box<dyn IndexFile>, | |
2aaae970 | 41 | verified_chunks: &mut HashSet<[u8;32]>, |
d8594d87 | 42 | corrupt_chunks: &mut HashSet<[u8; 32]>, |
fdaab0df DM |
43 | worker: &WorkerTask, |
44 | ) -> Result<(), Error> { | |
45 | ||
f66f537d | 46 | let mut errors = 0; |
fdaab0df DM |
47 | for pos in 0..index.index_count() { |
48 | ||
49 | worker.fail_on_abort()?; | |
fdaab0df DM |
50 | |
51 | let info = index.chunk_info(pos).unwrap(); | |
52 | let size = info.range.end - info.range.start; | |
2aaae970 DM |
53 | |
54 | if !verified_chunks.contains(&info.digest) { | |
d8594d87 DC |
55 | if !corrupt_chunks.contains(&info.digest) { |
56 | if let Err(err) = datastore.verify_stored_chunk(&info.digest, size) { | |
57 | corrupt_chunks.insert(info.digest); | |
58 | worker.log(format!("{}", err)); | |
59 | errors += 1; | |
60 | } else { | |
61 | verified_chunks.insert(info.digest); | |
62 | } | |
f66f537d | 63 | } else { |
d8594d87 DC |
64 | let digest_str = proxmox::tools::digest_to_hex(&info.digest); |
65 | worker.log(format!("chunk {} was marked as corrupt", digest_str)); | |
66 | errors += 1; | |
f66f537d | 67 | } |
2aaae970 | 68 | } |
fdaab0df DM |
69 | } |
70 | ||
f66f537d DC |
71 | if errors > 0 { |
72 | bail!("chunks could not be verified"); | |
73 | } | |
74 | ||
fdaab0df DM |
75 | Ok(()) |
76 | } | |
77 | ||
2aaae970 DM |
78 | fn verify_fixed_index( |
79 | datastore: &DataStore, | |
80 | backup_dir: &BackupDir, | |
81 | info: &FileInfo, | |
82 | verified_chunks: &mut HashSet<[u8;32]>, | |
d8594d87 | 83 | corrupt_chunks: &mut HashSet<[u8;32]>, |
2aaae970 DM |
84 | worker: &WorkerTask, |
85 | ) -> Result<(), Error> { | |
c2009e53 DM |
86 | |
87 | let mut path = backup_dir.relative_path(); | |
88 | path.push(&info.filename); | |
89 | ||
90 | let index = datastore.open_fixed_reader(&path)?; | |
91 | ||
92 | let (csum, size) = index.compute_csum(); | |
93 | if size != info.size { | |
94 | bail!("wrong size ({} != {})", info.size, size); | |
95 | } | |
96 | ||
97 | if csum != info.csum { | |
98 | bail!("wrong index checksum"); | |
99 | } | |
100 | ||
d8594d87 | 101 | verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, worker) |
c2009e53 DM |
102 | } |
103 | ||
2aaae970 DM |
104 | fn verify_dynamic_index( |
105 | datastore: &DataStore, | |
106 | backup_dir: &BackupDir, | |
107 | info: &FileInfo, | |
108 | verified_chunks: &mut HashSet<[u8;32]>, | |
d8594d87 | 109 | corrupt_chunks: &mut HashSet<[u8;32]>, |
2aaae970 DM |
110 | worker: &WorkerTask, |
111 | ) -> Result<(), Error> { | |
112 | ||
c2009e53 DM |
113 | let mut path = backup_dir.relative_path(); |
114 | path.push(&info.filename); | |
115 | ||
116 | let index = datastore.open_dynamic_reader(&path)?; | |
117 | ||
118 | let (csum, size) = index.compute_csum(); | |
119 | if size != info.size { | |
120 | bail!("wrong size ({} != {})", info.size, size); | |
121 | } | |
122 | ||
123 | if csum != info.csum { | |
124 | bail!("wrong index checksum"); | |
125 | } | |
126 | ||
d8594d87 | 127 | verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, worker) |
c2009e53 DM |
128 | } |
129 | ||
130 | /// Verify a single backup snapshot | |
131 | /// | |
132 | /// This checks all archives inside a backup snapshot. | |
133 | /// Errors are logged to the worker log. | |
134 | /// | |
8ea00f6e DM |
135 | /// Returns |
136 | /// - Ok(true) if verify is successful | |
137 | /// - Ok(false) if there were verification errors | |
138 | /// - Err(_) if task was aborted | |
2aaae970 DM |
139 | pub fn verify_backup_dir( |
140 | datastore: &DataStore, | |
141 | backup_dir: &BackupDir, | |
142 | verified_chunks: &mut HashSet<[u8;32]>, | |
d8594d87 | 143 | corrupt_chunks: &mut HashSet<[u8;32]>, |
2aaae970 DM |
144 | worker: &WorkerTask |
145 | ) -> Result<bool, Error> { | |
c2009e53 DM |
146 | |
147 | let manifest = match datastore.load_manifest(&backup_dir) { | |
ff86ef00 | 148 | Ok((manifest, _)) => manifest, |
c2009e53 DM |
149 | Err(err) => { |
150 | worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err)); | |
8ea00f6e | 151 | return Ok(false); |
c2009e53 DM |
152 | } |
153 | }; | |
154 | ||
155 | worker.log(format!("verify {}:{}", datastore.name(), backup_dir)); | |
156 | ||
157 | let mut error_count = 0; | |
158 | ||
159 | for info in manifest.files() { | |
160 | let result = proxmox::try_block!({ | |
161 | worker.log(format!(" check {}", info.filename)); | |
162 | match archive_type(&info.filename)? { | |
d8594d87 DC |
163 | ArchiveType::FixedIndex => |
164 | verify_fixed_index( | |
165 | &datastore, | |
166 | &backup_dir, | |
167 | info, | |
168 | verified_chunks, | |
169 | corrupt_chunks, | |
170 | worker | |
171 | ), | |
172 | ArchiveType::DynamicIndex => | |
173 | verify_dynamic_index( | |
174 | &datastore, | |
175 | &backup_dir, | |
176 | info, | |
177 | verified_chunks, | |
178 | corrupt_chunks, | |
179 | worker | |
180 | ), | |
c2009e53 DM |
181 | ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info), |
182 | } | |
183 | }); | |
8ea00f6e DM |
184 | |
185 | worker.fail_on_abort()?; | |
8ea00f6e | 186 | |
c2009e53 DM |
187 | if let Err(err) = result { |
188 | worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err)); | |
189 | error_count += 1; | |
190 | } | |
191 | } | |
192 | ||
8ea00f6e | 193 | Ok(error_count == 0) |
c2009e53 DM |
194 | } |
195 | ||
8ea00f6e DM |
196 | /// Verify all backups inside a backup group |
197 | /// | |
198 | /// Errors are logged to the worker log. | |
199 | /// | |
200 | /// Returns | |
adfdc369 | 201 | /// - Ok(failed_dirs) where failed_dirs had verification errors |
8ea00f6e | 202 | /// - Err(_) if task was aborted |
adfdc369 | 203 | pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<Vec<String>, Error> { |
c2009e53 | 204 | |
adfdc369 | 205 | let mut errors = Vec::new(); |
c2009e53 DM |
206 | let mut list = match group.list_backups(&datastore.base_path()) { |
207 | Ok(list) => list, | |
208 | Err(err) => { | |
209 | worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err)); | |
adfdc369 | 210 | return Ok(errors); |
c2009e53 DM |
211 | } |
212 | }; | |
213 | ||
214 | worker.log(format!("verify group {}:{}", datastore.name(), group)); | |
215 | ||
2aaae970 | 216 | let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB) |
d8594d87 | 217 | let mut corrupt_chunks = HashSet::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones |
2aaae970 | 218 | |
c2009e53 DM |
219 | BackupInfo::sort_list(&mut list, false); // newest first |
220 | for info in list { | |
d8594d87 | 221 | if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, &mut corrupt_chunks, worker)?{ |
adfdc369 | 222 | errors.push(info.backup_dir.to_string()); |
c2009e53 DM |
223 | } |
224 | } | |
225 | ||
adfdc369 | 226 | Ok(errors) |
c2009e53 DM |
227 | } |
228 | ||
8ea00f6e DM |
229 | /// Verify all backups inside a datastore |
230 | /// | |
231 | /// Errors are logged to the worker log. | |
232 | /// | |
233 | /// Returns | |
adfdc369 | 234 | /// - Ok(failed_dirs) where failed_dirs had verification errors |
8ea00f6e | 235 | /// - Err(_) if task was aborted |
adfdc369 DC |
236 | pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<Vec<String>, Error> { |
237 | ||
238 | let mut errors = Vec::new(); | |
c2009e53 DM |
239 | |
240 | let list = match BackupGroup::list_groups(&datastore.base_path()) { | |
241 | Ok(list) => list, | |
242 | Err(err) => { | |
243 | worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err)); | |
adfdc369 | 244 | return Ok(errors); |
c2009e53 DM |
245 | } |
246 | }; | |
247 | ||
248 | worker.log(format!("verify datastore {}", datastore.name())); | |
249 | ||
c2009e53 | 250 | for group in list { |
adfdc369 DC |
251 | let mut group_errors = verify_backup_group(datastore, &group, worker)?; |
252 | errors.append(&mut group_errors); | |
c2009e53 DM |
253 | } |
254 | ||
adfdc369 | 255 | Ok(errors) |
c2009e53 | 256 | } |