]>
Commit | Line | Data |
---|---|---|
2aaae970 DM |
1 | use std::collections::HashSet; |
2 | ||
c2009e53 DM |
3 | use anyhow::{bail, Error}; |
4 | ||
5 | use crate::server::WorkerTask; | |
6 | ||
7 | use super::{ | |
8 | DataStore, BackupGroup, BackupDir, BackupInfo, IndexFile, | |
8819d1f2 | 9 | CryptMode, |
c2009e53 DM |
10 | FileInfo, ArchiveType, archive_type, |
11 | }; | |
12 | ||
13 | fn verify_blob(datastore: &DataStore, backup_dir: &BackupDir, info: &FileInfo) -> Result<(), Error> { | |
14 | ||
39f18b30 | 15 | let blob = datastore.load_blob(backup_dir, &info.filename)?; |
c2009e53 | 16 | |
2aaae970 | 17 | let raw_size = blob.raw_size(); |
c2009e53 DM |
18 | if raw_size != info.size { |
19 | bail!("wrong size ({} != {})", info.size, raw_size); | |
20 | } | |
21 | ||
39f18b30 | 22 | let csum = openssl::sha::sha256(blob.raw_data()); |
c2009e53 DM |
23 | if csum != info.csum { |
24 | bail!("wrong index checksum"); | |
25 | } | |
26 | ||
8819d1f2 FG |
27 | match blob.crypt_mode()? { |
28 | CryptMode::Encrypt => Ok(()), | |
29 | CryptMode::None => { | |
30 | // digest already verified above | |
31 | blob.decode(None, None)?; | |
32 | Ok(()) | |
33 | }, | |
34 | CryptMode::SignOnly => bail!("Invalid CryptMode for blob"), | |
c2009e53 | 35 | } |
c2009e53 DM |
36 | } |
37 | ||
fdaab0df DM |
38 | fn verify_index_chunks( |
39 | datastore: &DataStore, | |
40 | index: Box<dyn IndexFile>, | |
2aaae970 | 41 | verified_chunks: &mut HashSet<[u8;32]>, |
d8594d87 | 42 | corrupt_chunks: &mut HashSet<[u8; 32]>, |
9a38fa29 | 43 | crypt_mode: CryptMode, |
fdaab0df DM |
44 | worker: &WorkerTask, |
45 | ) -> Result<(), Error> { | |
46 | ||
f66f537d | 47 | let mut errors = 0; |
fdaab0df DM |
48 | for pos in 0..index.index_count() { |
49 | ||
50 | worker.fail_on_abort()?; | |
fdaab0df DM |
51 | |
52 | let info = index.chunk_info(pos).unwrap(); | |
53 | let size = info.range.end - info.range.start; | |
2aaae970 | 54 | |
9a38fa29 FG |
55 | let chunk = match datastore.load_chunk(&info.digest) { |
56 | Err(err) => { | |
57 | corrupt_chunks.insert(info.digest); | |
58 | worker.log(format!("can't verify chunk, load failed - {}", err)); | |
59 | errors += 1; | |
60 | continue; | |
61 | }, | |
62 | Ok(chunk) => chunk, | |
63 | }; | |
64 | ||
65 | let chunk_crypt_mode = match chunk.crypt_mode() { | |
66 | Err(err) => { | |
67 | corrupt_chunks.insert(info.digest); | |
68 | worker.log(format!("can't verify chunk, unknown CryptMode - {}", err)); | |
69 | errors += 1; | |
70 | continue; | |
71 | }, | |
72 | Ok(mode) => mode, | |
73 | }; | |
74 | ||
75 | if chunk_crypt_mode != crypt_mode { | |
76 | worker.log(format!( | |
77 | "chunk CryptMode {:?} does not match index CryptMode {:?}", | |
78 | chunk_crypt_mode, | |
79 | crypt_mode | |
80 | )); | |
81 | errors += 1; | |
82 | } | |
83 | ||
2aaae970 | 84 | if !verified_chunks.contains(&info.digest) { |
d8594d87 | 85 | if !corrupt_chunks.contains(&info.digest) { |
9a38fa29 | 86 | if let Err(err) = chunk.verify_unencrypted(size as usize, &info.digest) { |
d8594d87 DC |
87 | corrupt_chunks.insert(info.digest); |
88 | worker.log(format!("{}", err)); | |
89 | errors += 1; | |
90 | } else { | |
91 | verified_chunks.insert(info.digest); | |
92 | } | |
f66f537d | 93 | } else { |
d8594d87 DC |
94 | let digest_str = proxmox::tools::digest_to_hex(&info.digest); |
95 | worker.log(format!("chunk {} was marked as corrupt", digest_str)); | |
96 | errors += 1; | |
f66f537d | 97 | } |
2aaae970 | 98 | } |
fdaab0df DM |
99 | } |
100 | ||
f66f537d DC |
101 | if errors > 0 { |
102 | bail!("chunks could not be verified"); | |
103 | } | |
104 | ||
fdaab0df DM |
105 | Ok(()) |
106 | } | |
107 | ||
2aaae970 DM |
108 | fn verify_fixed_index( |
109 | datastore: &DataStore, | |
110 | backup_dir: &BackupDir, | |
111 | info: &FileInfo, | |
112 | verified_chunks: &mut HashSet<[u8;32]>, | |
d8594d87 | 113 | corrupt_chunks: &mut HashSet<[u8;32]>, |
2aaae970 DM |
114 | worker: &WorkerTask, |
115 | ) -> Result<(), Error> { | |
c2009e53 DM |
116 | |
117 | let mut path = backup_dir.relative_path(); | |
118 | path.push(&info.filename); | |
119 | ||
120 | let index = datastore.open_fixed_reader(&path)?; | |
121 | ||
122 | let (csum, size) = index.compute_csum(); | |
123 | if size != info.size { | |
124 | bail!("wrong size ({} != {})", info.size, size); | |
125 | } | |
126 | ||
127 | if csum != info.csum { | |
128 | bail!("wrong index checksum"); | |
129 | } | |
130 | ||
9a38fa29 | 131 | verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, info.chunk_crypt_mode(), worker) |
c2009e53 DM |
132 | } |
133 | ||
2aaae970 DM |
134 | fn verify_dynamic_index( |
135 | datastore: &DataStore, | |
136 | backup_dir: &BackupDir, | |
137 | info: &FileInfo, | |
138 | verified_chunks: &mut HashSet<[u8;32]>, | |
d8594d87 | 139 | corrupt_chunks: &mut HashSet<[u8;32]>, |
2aaae970 DM |
140 | worker: &WorkerTask, |
141 | ) -> Result<(), Error> { | |
142 | ||
c2009e53 DM |
143 | let mut path = backup_dir.relative_path(); |
144 | path.push(&info.filename); | |
145 | ||
146 | let index = datastore.open_dynamic_reader(&path)?; | |
147 | ||
148 | let (csum, size) = index.compute_csum(); | |
149 | if size != info.size { | |
150 | bail!("wrong size ({} != {})", info.size, size); | |
151 | } | |
152 | ||
153 | if csum != info.csum { | |
154 | bail!("wrong index checksum"); | |
155 | } | |
156 | ||
9a38fa29 | 157 | verify_index_chunks(datastore, Box::new(index), verified_chunks, corrupt_chunks, info.chunk_crypt_mode(), worker) |
c2009e53 DM |
158 | } |
159 | ||
160 | /// Verify a single backup snapshot | |
161 | /// | |
162 | /// This checks all archives inside a backup snapshot. | |
163 | /// Errors are logged to the worker log. | |
164 | /// | |
8ea00f6e DM |
165 | /// Returns |
166 | /// - Ok(true) if verify is successful | |
167 | /// - Ok(false) if there were verification errors | |
168 | /// - Err(_) if task was aborted | |
2aaae970 DM |
169 | pub fn verify_backup_dir( |
170 | datastore: &DataStore, | |
171 | backup_dir: &BackupDir, | |
172 | verified_chunks: &mut HashSet<[u8;32]>, | |
d8594d87 | 173 | corrupt_chunks: &mut HashSet<[u8;32]>, |
2aaae970 DM |
174 | worker: &WorkerTask |
175 | ) -> Result<bool, Error> { | |
c2009e53 DM |
176 | |
177 | let manifest = match datastore.load_manifest(&backup_dir) { | |
ff86ef00 | 178 | Ok((manifest, _)) => manifest, |
c2009e53 DM |
179 | Err(err) => { |
180 | worker.log(format!("verify {}:{} - manifest load error: {}", datastore.name(), backup_dir, err)); | |
8ea00f6e | 181 | return Ok(false); |
c2009e53 DM |
182 | } |
183 | }; | |
184 | ||
185 | worker.log(format!("verify {}:{}", datastore.name(), backup_dir)); | |
186 | ||
187 | let mut error_count = 0; | |
188 | ||
189 | for info in manifest.files() { | |
190 | let result = proxmox::try_block!({ | |
191 | worker.log(format!(" check {}", info.filename)); | |
192 | match archive_type(&info.filename)? { | |
d8594d87 DC |
193 | ArchiveType::FixedIndex => |
194 | verify_fixed_index( | |
195 | &datastore, | |
196 | &backup_dir, | |
197 | info, | |
198 | verified_chunks, | |
199 | corrupt_chunks, | |
200 | worker | |
201 | ), | |
202 | ArchiveType::DynamicIndex => | |
203 | verify_dynamic_index( | |
204 | &datastore, | |
205 | &backup_dir, | |
206 | info, | |
207 | verified_chunks, | |
208 | corrupt_chunks, | |
209 | worker | |
210 | ), | |
c2009e53 DM |
211 | ArchiveType::Blob => verify_blob(&datastore, &backup_dir, info), |
212 | } | |
213 | }); | |
8ea00f6e DM |
214 | |
215 | worker.fail_on_abort()?; | |
8ea00f6e | 216 | |
c2009e53 DM |
217 | if let Err(err) = result { |
218 | worker.log(format!("verify {}:{}/{} failed: {}", datastore.name(), backup_dir, info.filename, err)); | |
219 | error_count += 1; | |
220 | } | |
221 | } | |
222 | ||
8ea00f6e | 223 | Ok(error_count == 0) |
c2009e53 DM |
224 | } |
225 | ||
8ea00f6e DM |
226 | /// Verify all backups inside a backup group |
227 | /// | |
228 | /// Errors are logged to the worker log. | |
229 | /// | |
230 | /// Returns | |
adfdc369 | 231 | /// - Ok(failed_dirs) where failed_dirs had verification errors |
8ea00f6e | 232 | /// - Err(_) if task was aborted |
adfdc369 | 233 | pub fn verify_backup_group(datastore: &DataStore, group: &BackupGroup, worker: &WorkerTask) -> Result<Vec<String>, Error> { |
c2009e53 | 234 | |
adfdc369 | 235 | let mut errors = Vec::new(); |
c2009e53 DM |
236 | let mut list = match group.list_backups(&datastore.base_path()) { |
237 | Ok(list) => list, | |
238 | Err(err) => { | |
239 | worker.log(format!("verify group {}:{} - unable to list backups: {}", datastore.name(), group, err)); | |
adfdc369 | 240 | return Ok(errors); |
c2009e53 DM |
241 | } |
242 | }; | |
243 | ||
244 | worker.log(format!("verify group {}:{}", datastore.name(), group)); | |
245 | ||
2aaae970 | 246 | let mut verified_chunks = HashSet::with_capacity(1024*16); // start with 16384 chunks (up to 65GB) |
d8594d87 | 247 | let mut corrupt_chunks = HashSet::with_capacity(64); // start with 64 chunks since we assume there are few corrupt ones |
2aaae970 | 248 | |
c2009e53 DM |
249 | BackupInfo::sort_list(&mut list, false); // newest first |
250 | for info in list { | |
d8594d87 | 251 | if !verify_backup_dir(datastore, &info.backup_dir, &mut verified_chunks, &mut corrupt_chunks, worker)?{ |
adfdc369 | 252 | errors.push(info.backup_dir.to_string()); |
c2009e53 DM |
253 | } |
254 | } | |
255 | ||
adfdc369 | 256 | Ok(errors) |
c2009e53 DM |
257 | } |
258 | ||
8ea00f6e DM |
259 | /// Verify all backups inside a datastore |
260 | /// | |
261 | /// Errors are logged to the worker log. | |
262 | /// | |
263 | /// Returns | |
adfdc369 | 264 | /// - Ok(failed_dirs) where failed_dirs had verification errors |
8ea00f6e | 265 | /// - Err(_) if task was aborted |
adfdc369 DC |
266 | pub fn verify_all_backups(datastore: &DataStore, worker: &WorkerTask) -> Result<Vec<String>, Error> { |
267 | ||
268 | let mut errors = Vec::new(); | |
c2009e53 DM |
269 | |
270 | let list = match BackupGroup::list_groups(&datastore.base_path()) { | |
271 | Ok(list) => list, | |
272 | Err(err) => { | |
273 | worker.log(format!("verify datastore {} - unable to list backups: {}", datastore.name(), err)); | |
adfdc369 | 274 | return Ok(errors); |
c2009e53 DM |
275 | } |
276 | }; | |
277 | ||
278 | worker.log(format!("verify datastore {}", datastore.name())); | |
279 | ||
c2009e53 | 280 | for group in list { |
adfdc369 DC |
281 | let mut group_errors = verify_backup_group(datastore, &group, worker)?; |
282 | errors.append(&mut group_errors); | |
c2009e53 DM |
283 | } |
284 | ||
adfdc369 | 285 | Ok(errors) |
c2009e53 | 286 | } |