]>
Commit | Line | Data |
---|---|---|
acc3d9df DM |
1 | use std::collections::HashSet; |
2 | use std::sync::{Arc, Mutex}; | |
3 | ||
4 | use anyhow::{bail, Error}; | |
5 | ||
6 | use crate::{ | |
7 | server::WorkerTask, | |
8 | api2::types::*, | |
9 | server::jobstate::Job, | |
10 | config::verify::VerificationJobConfig, | |
11 | backup::{ | |
12 | DataStore, | |
13 | BackupInfo, | |
14 | verify_backup_dir, | |
15 | }, | |
16 | task_log, | |
17 | }; | |
18 | ||
19 | /// Runs a verification job. | |
20 | pub fn do_verification_job( | |
21 | mut job: Job, | |
22 | verification_job: VerificationJobConfig, | |
23 | userid: &Userid, | |
24 | schedule: Option<String>, | |
25 | ) -> Result<String, Error> { | |
26 | let datastore = DataStore::lookup_datastore(&verification_job.store)?; | |
27 | ||
28 | let mut backups_to_verify = BackupInfo::list_backups(&datastore.base_path())?; | |
29 | if verification_job.ignore_verified.unwrap_or(true) { | |
30 | backups_to_verify.retain(|backup_info| { | |
31 | let manifest = match datastore.load_manifest(&backup_info.backup_dir) { | |
32 | Ok((manifest, _)) => manifest, | |
33 | Err(_) => return false, | |
34 | }; | |
35 | ||
36 | let raw_verify_state = manifest.unprotected["verify_state"].clone(); | |
37 | let last_state = match serde_json::from_value::<SnapshotVerifyState>(raw_verify_state) { | |
38 | Ok(last_state) => last_state, | |
39 | Err(_) => return true, | |
40 | }; | |
41 | ||
42 | let now = proxmox::tools::time::epoch_i64(); | |
43 | let days_since_last_verify = (now - last_state.upid.starttime) / 86400; | |
44 | verification_job.outdated_after.is_some() | |
45 | && days_since_last_verify > verification_job.outdated_after.unwrap() | |
46 | }) | |
47 | } | |
48 | ||
49 | let job_id = job.jobname().to_string(); | |
50 | let worker_type = job.jobtype().to_string(); | |
51 | let upid_str = WorkerTask::new_thread( | |
52 | &worker_type, | |
53 | Some(job.jobname().to_string()), | |
54 | userid.clone(), | |
55 | false, | |
56 | move |worker| { | |
57 | job.start(&worker.upid().to_string())?; | |
58 | ||
59 | task_log!(worker,"Starting datastore verify job '{}'", job_id); | |
60 | task_log!(worker,"verifying {} backups", backups_to_verify.len()); | |
61 | if let Some(event_str) = schedule { | |
62 | task_log!(worker,"task triggered by schedule '{}'", event_str); | |
63 | } | |
64 | ||
65 | let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024 * 16))); | |
66 | let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64))); | |
67 | let result = proxmox::try_block!({ | |
68 | let mut failed_dirs: Vec<String> = Vec::new(); | |
69 | ||
70 | for backup_info in backups_to_verify { | |
71 | let verification_result = verify_backup_dir( | |
72 | datastore.clone(), | |
73 | &backup_info.backup_dir, | |
74 | verified_chunks.clone(), | |
75 | corrupt_chunks.clone(), | |
76 | worker.clone(), | |
77 | worker.upid().clone() | |
78 | ); | |
79 | ||
80 | if let Ok(false) = verification_result { | |
81 | failed_dirs.push(backup_info.backup_dir.to_string()); | |
82 | } // otherwise successful or aborted | |
83 | } | |
84 | ||
85 | if !failed_dirs.is_empty() { | |
86 | task_log!(worker,"Failed to verify following snapshots:",); | |
87 | for dir in failed_dirs { | |
88 | task_log!(worker, "\t{}", dir) | |
89 | } | |
90 | bail!("verification failed - please check the log for details"); | |
91 | } | |
92 | Ok(()) | |
93 | }); | |
94 | ||
95 | let status = worker.create_state(&result); | |
96 | ||
97 | match job.finish(status) { | |
98 | Err(err) => eprintln!( | |
99 | "could not finish job state for {}: {}", | |
100 | job.jobtype().to_string(), | |
101 | err | |
102 | ), | |
103 | Ok(_) => (), | |
104 | } | |
105 | ||
106 | result | |
107 | }, | |
108 | )?; | |
109 | Ok(upid_str) | |
110 | } |