]>
Commit | Line | Data |
---|---|---|
acc3d9df DM |
1 | use std::collections::HashSet; |
2 | use std::sync::{Arc, Mutex}; | |
3 | ||
4 | use anyhow::{bail, Error}; | |
5 | ||
6 | use crate::{ | |
7 | server::WorkerTask, | |
8 | api2::types::*, | |
9 | server::jobstate::Job, | |
10 | config::verify::VerificationJobConfig, | |
11 | backup::{ | |
12 | DataStore, | |
13 | BackupInfo, | |
14 | verify_backup_dir, | |
15 | }, | |
16 | task_log, | |
17 | }; | |
18 | ||
19 | /// Runs a verification job. | |
20 | pub fn do_verification_job( | |
21 | mut job: Job, | |
22 | verification_job: VerificationJobConfig, | |
23 | userid: &Userid, | |
24 | schedule: Option<String>, | |
25 | ) -> Result<String, Error> { | |
26 | let datastore = DataStore::lookup_datastore(&verification_job.store)?; | |
27 | ||
28 | let mut backups_to_verify = BackupInfo::list_backups(&datastore.base_path())?; | |
29 | if verification_job.ignore_verified.unwrap_or(true) { | |
30 | backups_to_verify.retain(|backup_info| { | |
31 | let manifest = match datastore.load_manifest(&backup_info.backup_dir) { | |
32 | Ok((manifest, _)) => manifest, | |
33 | Err(_) => return false, | |
34 | }; | |
35 | ||
36 | let raw_verify_state = manifest.unprotected["verify_state"].clone(); | |
37 | let last_state = match serde_json::from_value::<SnapshotVerifyState>(raw_verify_state) { | |
38 | Ok(last_state) => last_state, | |
39 | Err(_) => return true, | |
40 | }; | |
41 | ||
42 | let now = proxmox::tools::time::epoch_i64(); | |
43 | let days_since_last_verify = (now - last_state.upid.starttime) / 86400; | |
44 | verification_job.outdated_after.is_some() | |
45 | && days_since_last_verify > verification_job.outdated_after.unwrap() | |
46 | }) | |
47 | } | |
48 | ||
b9e7bcc2 DM |
49 | let email = crate::server::lookup_user_email(userid); |
50 | ||
acc3d9df DM |
51 | let job_id = job.jobname().to_string(); |
52 | let worker_type = job.jobtype().to_string(); | |
53 | let upid_str = WorkerTask::new_thread( | |
54 | &worker_type, | |
55 | Some(job.jobname().to_string()), | |
56 | userid.clone(), | |
57 | false, | |
58 | move |worker| { | |
59 | job.start(&worker.upid().to_string())?; | |
60 | ||
61 | task_log!(worker,"Starting datastore verify job '{}'", job_id); | |
62 | task_log!(worker,"verifying {} backups", backups_to_verify.len()); | |
63 | if let Some(event_str) = schedule { | |
64 | task_log!(worker,"task triggered by schedule '{}'", event_str); | |
65 | } | |
66 | ||
67 | let verified_chunks = Arc::new(Mutex::new(HashSet::with_capacity(1024 * 16))); | |
68 | let corrupt_chunks = Arc::new(Mutex::new(HashSet::with_capacity(64))); | |
69 | let result = proxmox::try_block!({ | |
70 | let mut failed_dirs: Vec<String> = Vec::new(); | |
71 | ||
72 | for backup_info in backups_to_verify { | |
73 | let verification_result = verify_backup_dir( | |
74 | datastore.clone(), | |
75 | &backup_info.backup_dir, | |
76 | verified_chunks.clone(), | |
77 | corrupt_chunks.clone(), | |
78 | worker.clone(), | |
79 | worker.upid().clone() | |
80 | ); | |
81 | ||
82 | if let Ok(false) = verification_result { | |
83 | failed_dirs.push(backup_info.backup_dir.to_string()); | |
84 | } // otherwise successful or aborted | |
85 | } | |
86 | ||
87 | if !failed_dirs.is_empty() { | |
88 | task_log!(worker,"Failed to verify following snapshots:",); | |
89 | for dir in failed_dirs { | |
90 | task_log!(worker, "\t{}", dir) | |
91 | } | |
92 | bail!("verification failed - please check the log for details"); | |
93 | } | |
94 | Ok(()) | |
95 | }); | |
96 | ||
97 | let status = worker.create_state(&result); | |
98 | ||
99 | match job.finish(status) { | |
100 | Err(err) => eprintln!( | |
101 | "could not finish job state for {}: {}", | |
102 | job.jobtype().to_string(), | |
103 | err | |
104 | ), | |
105 | Ok(_) => (), | |
106 | } | |
107 | ||
b9e7bcc2 DM |
108 | if let Some(email) = email { |
109 | if let Err(err) = crate::server::send_verify_status(&email, verification_job, &result) { | |
110 | eprintln!("send verify notification failed: {}", err); | |
111 | } | |
112 | } | |
113 | ||
acc3d9df DM |
114 | result |
115 | }, | |
116 | )?; | |
117 | Ok(upid_str) | |
118 | } |