]>
Commit | Line | Data |
---|---|---|
7759eef5 | 1 | use std::collections::{HashSet, HashMap}; |
54552dda | 2 | use std::io::{self, Write}; |
367f002e WB |
3 | use std::path::{Path, PathBuf}; |
4 | use std::sync::{Arc, Mutex}; | |
60f9a6ea | 5 | use std::convert::TryFrom; |
cb4b721c | 6 | use std::str::FromStr; |
1a374fcf | 7 | use std::time::Duration; |
367f002e | 8 | |
f7d4e4b5 | 9 | use anyhow::{bail, format_err, Error}; |
2c32fdde | 10 | use lazy_static::lazy_static; |
e4439025 | 11 | |
7526d864 | 12 | use proxmox::tools::fs::{replace_file, file_read_optional_string, CreateOptions}; |
d5790a9f DM |
13 | use proxmox_sys::process_locker::ProcessLockSharedGuard; |
14 | use proxmox_sys::worker_task_context::WorkerTaskContext; | |
15 | use proxmox_sys::{task_log, task_warn}; | |
529de6c7 | 16 | |
a58a5cf7 | 17 | use pbs_api_types::{UPID, DataStoreConfig, Authid, GarbageCollectionStatus, HumanByte}; |
770a36e5 | 18 | use pbs_tools::fs::{lock_dir_noblock, DirLockGuard}; |
21211748 | 19 | use pbs_config::{open_backup_lockfile, BackupLockGuard}; |
529de6c7 | 20 | |
6d5d305d DM |
21 | use crate::DataBlob; |
22 | use crate::backup_info::{BackupGroup, BackupDir}; | |
23 | use crate::chunk_store::ChunkStore; | |
24 | use crate::dynamic_index::{DynamicIndexReader, DynamicIndexWriter}; | |
25 | use crate::fixed_index::{FixedIndexReader, FixedIndexWriter}; | |
26 | use crate::index::IndexFile; | |
27 | use crate::manifest::{ | |
28 | MANIFEST_BLOB_NAME, MANIFEST_LOCK_NAME, CLIENT_LOG_BLOB_NAME, | |
29 | ArchiveType, BackupManifest, | |
30 | archive_type, | |
31 | }; | |
32 | ||
367f002e WB |
33 | lazy_static! { |
34 | static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStore>>> = Mutex::new(HashMap::new()); | |
b3483782 | 35 | } |
ff3d3100 | 36 | |
9751ef4b DC |
37 | /// checks if auth_id is owner, or, if owner is a token, if |
38 | /// auth_id is the user of the token | |
39 | pub fn check_backup_owner( | |
40 | owner: &Authid, | |
41 | auth_id: &Authid, | |
42 | ) -> Result<(), Error> { | |
43 | let correct_owner = owner == auth_id | |
44 | || (owner.is_token() && &Authid::from(owner.user().clone()) == auth_id); | |
45 | if !correct_owner { | |
46 | bail!("backup owner check failed ({} != {})", auth_id, owner); | |
47 | } | |
48 | Ok(()) | |
49 | } | |
50 | ||
e5064ba6 DM |
51 | /// Datastore Management |
52 | /// | |
53 | /// A Datastore can store severals backups, and provides the | |
54 | /// management interface for backup. | |
529de6c7 | 55 | pub struct DataStore { |
1629d2ad | 56 | chunk_store: Arc<ChunkStore>, |
81b2a872 | 57 | gc_mutex: Mutex<()>, |
f2b99c34 | 58 | last_gc_status: Mutex<GarbageCollectionStatus>, |
0698f78d | 59 | verify_new: bool, |
529de6c7 DM |
60 | } |
61 | ||
62 | impl DataStore { | |
63 | ||
2c32fdde DM |
64 | pub fn lookup_datastore(name: &str) -> Result<Arc<DataStore>, Error> { |
65 | ||
e7d4be9d DM |
66 | let (config, _digest) = pbs_config::datastore::config()?; |
67 | let config: DataStoreConfig = config.lookup("datastore", name)?; | |
df729017 | 68 | let path = PathBuf::from(&config.path); |
2c32fdde | 69 | |
515688d1 | 70 | let mut map = DATASTORE_MAP.lock().unwrap(); |
2c32fdde DM |
71 | |
72 | if let Some(datastore) = map.get(name) { | |
73 | // Compare Config - if changed, create new Datastore object! | |
c23192d3 | 74 | if datastore.chunk_store.base() == path && |
0698f78d SR |
75 | datastore.verify_new == config.verify_new.unwrap_or(false) |
76 | { | |
2c32fdde DM |
77 | return Ok(datastore.clone()); |
78 | } | |
79 | } | |
80 | ||
df729017 | 81 | let datastore = DataStore::open_with_path(name, &path, config)?; |
f0a61124 DM |
82 | |
83 | let datastore = Arc::new(datastore); | |
84 | map.insert(name.to_string(), datastore.clone()); | |
2c32fdde | 85 | |
f0a61124 | 86 | Ok(datastore) |
2c32fdde DM |
87 | } |
88 | ||
062cf75c DC |
89 | /// removes all datastores that are not configured anymore |
90 | pub fn remove_unused_datastores() -> Result<(), Error>{ | |
e7d4be9d | 91 | let (config, _digest) = pbs_config::datastore::config()?; |
062cf75c DC |
92 | |
93 | let mut map = DATASTORE_MAP.lock().unwrap(); | |
94 | // removes all elements that are not in the config | |
95 | map.retain(|key, _| { | |
96 | config.sections.contains_key(key) | |
97 | }); | |
98 | Ok(()) | |
99 | } | |
100 | ||
0698f78d | 101 | fn open_with_path(store_name: &str, path: &Path, config: DataStoreConfig) -> Result<Self, Error> { |
277fc5a3 | 102 | let chunk_store = ChunkStore::open(store_name, path)?; |
529de6c7 | 103 | |
b683fd58 DC |
104 | let mut gc_status_path = chunk_store.base_path(); |
105 | gc_status_path.push(".gc-status"); | |
106 | ||
107 | let gc_status = if let Some(state) = file_read_optional_string(gc_status_path)? { | |
108 | match serde_json::from_str(&state) { | |
109 | Ok(state) => state, | |
110 | Err(err) => { | |
111 | eprintln!("error reading gc-status: {}", err); | |
112 | GarbageCollectionStatus::default() | |
113 | } | |
114 | } | |
115 | } else { | |
116 | GarbageCollectionStatus::default() | |
117 | }; | |
f2b99c34 | 118 | |
529de6c7 | 119 | Ok(Self { |
1629d2ad | 120 | chunk_store: Arc::new(chunk_store), |
81b2a872 | 121 | gc_mutex: Mutex::new(()), |
f2b99c34 | 122 | last_gc_status: Mutex::new(gc_status), |
0698f78d | 123 | verify_new: config.verify_new.unwrap_or(false), |
529de6c7 DM |
124 | }) |
125 | } | |
126 | ||
d59397e6 WB |
127 | pub fn get_chunk_iterator( |
128 | &self, | |
129 | ) -> Result< | |
770a36e5 | 130 | impl Iterator<Item = (Result<pbs_tools::fs::ReadDirEntry, Error>, usize, bool)>, |
d59397e6 WB |
131 | Error |
132 | > { | |
a5736098 | 133 | self.chunk_store.get_chunk_iterator() |
d59397e6 WB |
134 | } |
135 | ||
91a905b6 | 136 | pub fn create_fixed_writer<P: AsRef<Path>>(&self, filename: P, size: usize, chunk_size: usize) -> Result<FixedIndexWriter, Error> { |
529de6c7 | 137 | |
91a905b6 | 138 | let index = FixedIndexWriter::create(self.chunk_store.clone(), filename.as_ref(), size, chunk_size)?; |
529de6c7 DM |
139 | |
140 | Ok(index) | |
141 | } | |
142 | ||
91a905b6 | 143 | pub fn open_fixed_reader<P: AsRef<Path>>(&self, filename: P) -> Result<FixedIndexReader, Error> { |
529de6c7 | 144 | |
a7c72ad9 DM |
145 | let full_path = self.chunk_store.relative_path(filename.as_ref()); |
146 | ||
147 | let index = FixedIndexReader::open(&full_path)?; | |
529de6c7 DM |
148 | |
149 | Ok(index) | |
150 | } | |
3d5c11e5 | 151 | |
93d5d779 | 152 | pub fn create_dynamic_writer<P: AsRef<Path>>( |
0433db19 | 153 | &self, filename: P, |
93d5d779 | 154 | ) -> Result<DynamicIndexWriter, Error> { |
0433db19 | 155 | |
93d5d779 | 156 | let index = DynamicIndexWriter::create( |
976595e1 | 157 | self.chunk_store.clone(), filename.as_ref())?; |
0433db19 DM |
158 | |
159 | Ok(index) | |
160 | } | |
ff3d3100 | 161 | |
93d5d779 | 162 | pub fn open_dynamic_reader<P: AsRef<Path>>(&self, filename: P) -> Result<DynamicIndexReader, Error> { |
77703d95 | 163 | |
d48a9955 DM |
164 | let full_path = self.chunk_store.relative_path(filename.as_ref()); |
165 | ||
166 | let index = DynamicIndexReader::open(&full_path)?; | |
77703d95 DM |
167 | |
168 | Ok(index) | |
169 | } | |
170 | ||
5de2bced WB |
171 | pub fn open_index<P>(&self, filename: P) -> Result<Box<dyn IndexFile + Send>, Error> |
172 | where | |
173 | P: AsRef<Path>, | |
174 | { | |
175 | let filename = filename.as_ref(); | |
176 | let out: Box<dyn IndexFile + Send> = | |
1e8da0a7 DM |
177 | match archive_type(filename)? { |
178 | ArchiveType::DynamicIndex => Box::new(self.open_dynamic_reader(filename)?), | |
179 | ArchiveType::FixedIndex => Box::new(self.open_fixed_reader(filename)?), | |
5de2bced WB |
180 | _ => bail!("cannot open index file of unknown type: {:?}", filename), |
181 | }; | |
182 | Ok(out) | |
183 | } | |
184 | ||
1369bcdb | 185 | /// Fast index verification - only check if chunks exists |
28570d19 DM |
186 | pub fn fast_index_verification( |
187 | &self, | |
188 | index: &dyn IndexFile, | |
189 | checked: &mut HashSet<[u8;32]>, | |
190 | ) -> Result<(), Error> { | |
1369bcdb DM |
191 | |
192 | for pos in 0..index.index_count() { | |
193 | let info = index.chunk_info(pos).unwrap(); | |
28570d19 DM |
194 | if checked.contains(&info.digest) { |
195 | continue; | |
196 | } | |
197 | ||
1369bcdb DM |
198 | self.stat_chunk(&info.digest). |
199 | map_err(|err| { | |
200 | format_err!( | |
201 | "fast_index_verification error, stat_chunk {} failed - {}", | |
202 | proxmox::tools::digest_to_hex(&info.digest), | |
203 | err, | |
204 | ) | |
205 | })?; | |
28570d19 DM |
206 | |
207 | checked.insert(info.digest); | |
1369bcdb DM |
208 | } |
209 | ||
210 | Ok(()) | |
211 | } | |
212 | ||
60f9a6ea DM |
213 | pub fn name(&self) -> &str { |
214 | self.chunk_store.name() | |
215 | } | |
216 | ||
ff3d3100 DM |
217 | pub fn base_path(&self) -> PathBuf { |
218 | self.chunk_store.base_path() | |
219 | } | |
220 | ||
c47e294e | 221 | /// Cleanup a backup directory |
7759eef5 DM |
222 | /// |
223 | /// Removes all files not mentioned in the manifest. | |
224 | pub fn cleanup_backup_dir(&self, backup_dir: &BackupDir, manifest: &BackupManifest | |
225 | ) -> Result<(), Error> { | |
226 | ||
227 | let mut full_path = self.base_path(); | |
228 | full_path.push(backup_dir.relative_path()); | |
229 | ||
230 | let mut wanted_files = HashSet::new(); | |
231 | wanted_files.insert(MANIFEST_BLOB_NAME.to_string()); | |
1610c45a | 232 | wanted_files.insert(CLIENT_LOG_BLOB_NAME.to_string()); |
7759eef5 DM |
233 | manifest.files().iter().for_each(|item| { wanted_files.insert(item.filename.clone()); }); |
234 | ||
770a36e5 | 235 | for item in pbs_tools::fs::read_subdir(libc::AT_FDCWD, &full_path)? { |
7759eef5 DM |
236 | if let Ok(item) = item { |
237 | if let Some(file_type) = item.file_type() { | |
238 | if file_type != nix::dir::Type::File { continue; } | |
239 | } | |
240 | let file_name = item.file_name().to_bytes(); | |
241 | if file_name == b"." || file_name == b".." { continue; }; | |
242 | ||
243 | if let Ok(name) = std::str::from_utf8(file_name) { | |
244 | if wanted_files.contains(name) { continue; } | |
245 | } | |
246 | println!("remove unused file {:?}", item.file_name()); | |
247 | let dirfd = item.parent_fd(); | |
248 | let _res = unsafe { libc::unlinkat(dirfd, item.file_name().as_ptr(), 0) }; | |
249 | } | |
250 | } | |
251 | ||
252 | Ok(()) | |
253 | } | |
4b4eba0b | 254 | |
41b373ec DM |
255 | /// Returns the absolute path for a backup_group |
256 | pub fn group_path(&self, backup_group: &BackupGroup) -> PathBuf { | |
4b4eba0b DM |
257 | let mut full_path = self.base_path(); |
258 | full_path.push(backup_group.group_path()); | |
41b373ec DM |
259 | full_path |
260 | } | |
261 | ||
262 | /// Returns the absolute path for backup_dir | |
263 | pub fn snapshot_path(&self, backup_dir: &BackupDir) -> PathBuf { | |
264 | let mut full_path = self.base_path(); | |
265 | full_path.push(backup_dir.relative_path()); | |
266 | full_path | |
267 | } | |
268 | ||
de91418b DC |
269 | /// Remove a complete backup group including all snapshots, returns true |
270 | /// if all snapshots were removed, and false if some were protected | |
271 | pub fn remove_backup_group(&self, backup_group: &BackupGroup) -> Result<bool, Error> { | |
41b373ec DM |
272 | |
273 | let full_path = self.group_path(backup_group); | |
4b4eba0b | 274 | |
770a36e5 | 275 | let _guard = pbs_tools::fs::lock_dir_noblock(&full_path, "backup group", "possible running backup")?; |
c9756b40 | 276 | |
4b4eba0b | 277 | log::info!("removing backup group {:?}", full_path); |
4c0ae82e | 278 | |
de91418b DC |
279 | let mut removed_all = true; |
280 | ||
4c0ae82e SR |
281 | // remove all individual backup dirs first to ensure nothing is using them |
282 | for snap in backup_group.list_backups(&self.base_path())? { | |
de91418b DC |
283 | if snap.backup_dir.is_protected(self.base_path()) { |
284 | removed_all = false; | |
285 | continue; | |
286 | } | |
4c0ae82e SR |
287 | self.remove_backup_dir(&snap.backup_dir, false)?; |
288 | } | |
289 | ||
de91418b DC |
290 | if removed_all { |
291 | // no snapshots left, we can now safely remove the empty folder | |
292 | std::fs::remove_dir_all(&full_path) | |
293 | .map_err(|err| { | |
294 | format_err!( | |
295 | "removing backup group directory {:?} failed - {}", | |
296 | full_path, | |
297 | err, | |
298 | ) | |
299 | })?; | |
300 | } | |
4b4eba0b | 301 | |
de91418b | 302 | Ok(removed_all) |
4b4eba0b DM |
303 | } |
304 | ||
8f579717 | 305 | /// Remove a backup directory including all content |
c9756b40 | 306 | pub fn remove_backup_dir(&self, backup_dir: &BackupDir, force: bool) -> Result<(), Error> { |
8f579717 | 307 | |
41b373ec | 308 | let full_path = self.snapshot_path(backup_dir); |
8f579717 | 309 | |
1a374fcf | 310 | let (_guard, _manifest_guard); |
c9756b40 | 311 | if !force { |
238a872d | 312 | _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or in use")?; |
6bd0a00c | 313 | _manifest_guard = self.lock_manifest(backup_dir)?; |
c9756b40 SR |
314 | } |
315 | ||
de91418b DC |
316 | if backup_dir.is_protected(self.base_path()) { |
317 | bail!("cannot remove protected snapshot"); | |
318 | } | |
319 | ||
8a1d68c8 | 320 | log::info!("removing backup snapshot {:?}", full_path); |
6abce6c2 | 321 | std::fs::remove_dir_all(&full_path) |
8a1d68c8 DM |
322 | .map_err(|err| { |
323 | format_err!( | |
324 | "removing backup snapshot {:?} failed - {}", | |
325 | full_path, | |
326 | err, | |
327 | ) | |
328 | })?; | |
8f579717 | 329 | |
179145dc DC |
330 | // the manifest does not exists anymore, we do not need to keep the lock |
331 | if let Ok(path) = self.manifest_lock_path(backup_dir) { | |
332 | // ignore errors | |
333 | let _ = std::fs::remove_file(path); | |
334 | } | |
335 | ||
8f579717 DM |
336 | Ok(()) |
337 | } | |
338 | ||
41b373ec DM |
339 | /// Returns the time of the last successful backup |
340 | /// | |
341 | /// Or None if there is no backup in the group (or the group dir does not exist). | |
6a7be83e | 342 | pub fn last_successful_backup(&self, backup_group: &BackupGroup) -> Result<Option<i64>, Error> { |
41b373ec DM |
343 | let base_path = self.base_path(); |
344 | let mut group_path = base_path.clone(); | |
345 | group_path.push(backup_group.group_path()); | |
346 | ||
347 | if group_path.exists() { | |
348 | backup_group.last_successful_backup(&base_path) | |
349 | } else { | |
350 | Ok(None) | |
351 | } | |
352 | } | |
353 | ||
54552dda DM |
354 | /// Returns the backup owner. |
355 | /// | |
e6dc35ac FG |
356 | /// The backup owner is the entity who first created the backup group. |
357 | pub fn get_owner(&self, backup_group: &BackupGroup) -> Result<Authid, Error> { | |
54552dda DM |
358 | let mut full_path = self.base_path(); |
359 | full_path.push(backup_group.group_path()); | |
360 | full_path.push("owner"); | |
361 | let owner = proxmox::tools::fs::file_read_firstline(full_path)?; | |
e7cb4dc5 | 362 | Ok(owner.trim_end().parse()?) // remove trailing newline |
54552dda DM |
363 | } |
364 | ||
9751ef4b DC |
365 | pub fn owns_backup(&self, backup_group: &BackupGroup, auth_id: &Authid) -> Result<bool, Error> { |
366 | let owner = self.get_owner(backup_group)?; | |
367 | ||
8e0b852f | 368 | Ok(check_backup_owner(&owner, auth_id).is_ok()) |
9751ef4b DC |
369 | } |
370 | ||
54552dda | 371 | /// Set the backup owner. |
e7cb4dc5 WB |
372 | pub fn set_owner( |
373 | &self, | |
374 | backup_group: &BackupGroup, | |
e6dc35ac | 375 | auth_id: &Authid, |
e7cb4dc5 WB |
376 | force: bool, |
377 | ) -> Result<(), Error> { | |
54552dda DM |
378 | let mut path = self.base_path(); |
379 | path.push(backup_group.group_path()); | |
380 | path.push("owner"); | |
381 | ||
382 | let mut open_options = std::fs::OpenOptions::new(); | |
383 | open_options.write(true); | |
384 | open_options.truncate(true); | |
385 | ||
386 | if force { | |
387 | open_options.create(true); | |
388 | } else { | |
389 | open_options.create_new(true); | |
390 | } | |
391 | ||
392 | let mut file = open_options.open(&path) | |
393 | .map_err(|err| format_err!("unable to create owner file {:?} - {}", path, err))?; | |
394 | ||
e6dc35ac | 395 | writeln!(file, "{}", auth_id) |
54552dda DM |
396 | .map_err(|err| format_err!("unable to write owner file {:?} - {}", path, err))?; |
397 | ||
398 | Ok(()) | |
399 | } | |
400 | ||
1fc82c41 | 401 | /// Create (if it does not already exists) and lock a backup group |
54552dda DM |
402 | /// |
403 | /// And set the owner to 'userid'. If the group already exists, it returns the | |
404 | /// current owner (instead of setting the owner). | |
1fc82c41 | 405 | /// |
1ffe0301 | 406 | /// This also acquires an exclusive lock on the directory and returns the lock guard. |
e7cb4dc5 WB |
407 | pub fn create_locked_backup_group( |
408 | &self, | |
409 | backup_group: &BackupGroup, | |
e6dc35ac FG |
410 | auth_id: &Authid, |
411 | ) -> Result<(Authid, DirLockGuard), Error> { | |
8731e40a | 412 | // create intermediate path first: |
44288184 | 413 | let mut full_path = self.base_path(); |
54552dda | 414 | full_path.push(backup_group.backup_type()); |
8731e40a WB |
415 | std::fs::create_dir_all(&full_path)?; |
416 | ||
54552dda DM |
417 | full_path.push(backup_group.backup_id()); |
418 | ||
419 | // create the last component now | |
420 | match std::fs::create_dir(&full_path) { | |
421 | Ok(_) => { | |
e4342585 | 422 | let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?; |
e6dc35ac | 423 | self.set_owner(backup_group, auth_id, false)?; |
54552dda | 424 | let owner = self.get_owner(backup_group)?; // just to be sure |
1fc82c41 | 425 | Ok((owner, guard)) |
54552dda DM |
426 | } |
427 | Err(ref err) if err.kind() == io::ErrorKind::AlreadyExists => { | |
e4342585 | 428 | let guard = lock_dir_noblock(&full_path, "backup group", "another backup is already running")?; |
54552dda | 429 | let owner = self.get_owner(backup_group)?; // just to be sure |
1fc82c41 | 430 | Ok((owner, guard)) |
54552dda DM |
431 | } |
432 | Err(err) => bail!("unable to create backup group {:?} - {}", full_path, err), | |
433 | } | |
434 | } | |
435 | ||
436 | /// Creates a new backup snapshot inside a BackupGroup | |
437 | /// | |
438 | /// The BackupGroup directory needs to exist. | |
f23f7543 SR |
439 | pub fn create_locked_backup_dir(&self, backup_dir: &BackupDir) |
440 | -> Result<(PathBuf, bool, DirLockGuard), Error> | |
441 | { | |
b3483782 DM |
442 | let relative_path = backup_dir.relative_path(); |
443 | let mut full_path = self.base_path(); | |
444 | full_path.push(&relative_path); | |
ff3d3100 | 445 | |
f23f7543 SR |
446 | let lock = || |
447 | lock_dir_noblock(&full_path, "snapshot", "internal error - tried creating snapshot that's already in use"); | |
448 | ||
8731e40a | 449 | match std::fs::create_dir(&full_path) { |
f23f7543 SR |
450 | Ok(_) => Ok((relative_path, true, lock()?)), |
451 | Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => Ok((relative_path, false, lock()?)), | |
452 | Err(e) => Err(e.into()) | |
8731e40a | 453 | } |
ff3d3100 DM |
454 | } |
455 | ||
3d5c11e5 | 456 | pub fn list_images(&self) -> Result<Vec<PathBuf>, Error> { |
ff3d3100 | 457 | let base = self.base_path(); |
3d5c11e5 DM |
458 | |
459 | let mut list = vec![]; | |
460 | ||
95cea65b DM |
461 | use walkdir::WalkDir; |
462 | ||
84466003 | 463 | let walker = WalkDir::new(&base).into_iter(); |
95cea65b DM |
464 | |
465 | // make sure we skip .chunks (and other hidden files to keep it simple) | |
466 | fn is_hidden(entry: &walkdir::DirEntry) -> bool { | |
467 | entry.file_name() | |
468 | .to_str() | |
d8d8af98 | 469 | .map(|s| s.starts_with('.')) |
95cea65b DM |
470 | .unwrap_or(false) |
471 | } | |
c3b090ac TL |
472 | let handle_entry_err = |err: walkdir::Error| { |
473 | if let Some(inner) = err.io_error() { | |
d08cff51 FG |
474 | if let Some(path) = err.path() { |
475 | if inner.kind() == io::ErrorKind::PermissionDenied { | |
c3b090ac TL |
476 | // only allow to skip ext4 fsck directory, avoid GC if, for example, |
477 | // a user got file permissions wrong on datastore rsync to new server | |
478 | if err.depth() > 1 || !path.ends_with("lost+found") { | |
d08cff51 | 479 | bail!("cannot continue garbage-collection safely, permission denied on: {:?}", path) |
c3b090ac | 480 | } |
d08cff51 FG |
481 | } else { |
482 | bail!("unexpected error on datastore traversal: {} - {:?}", inner, path) | |
483 | } | |
484 | } else { | |
485 | bail!("unexpected error on datastore traversal: {}", inner) | |
c3b090ac TL |
486 | } |
487 | } | |
488 | Ok(()) | |
489 | }; | |
95cea65b | 490 | for entry in walker.filter_entry(|e| !is_hidden(e)) { |
c3b090ac TL |
491 | let path = match entry { |
492 | Ok(entry) => entry.into_path(), | |
493 | Err(err) => { | |
494 | handle_entry_err(err)?; | |
495 | continue | |
496 | }, | |
497 | }; | |
1e8da0a7 DM |
498 | if let Ok(archive_type) = archive_type(&path) { |
499 | if archive_type == ArchiveType::FixedIndex || archive_type == ArchiveType::DynamicIndex { | |
95cea65b | 500 | list.push(path); |
3d5c11e5 DM |
501 | } |
502 | } | |
503 | } | |
504 | ||
505 | Ok(list) | |
506 | } | |
507 | ||
a660978c DM |
508 | // mark chunks used by ``index`` as used |
509 | fn index_mark_used_chunks<I: IndexFile>( | |
510 | &self, | |
511 | index: I, | |
512 | file_name: &Path, // only used for error reporting | |
513 | status: &mut GarbageCollectionStatus, | |
c8449217 | 514 | worker: &dyn WorkerTaskContext, |
a660978c DM |
515 | ) -> Result<(), Error> { |
516 | ||
517 | status.index_file_count += 1; | |
518 | status.index_data_bytes += index.index_bytes(); | |
519 | ||
520 | for pos in 0..index.index_count() { | |
f6b1d1cc | 521 | worker.check_abort()?; |
0fd55b08 | 522 | worker.fail_on_shutdown()?; |
a660978c | 523 | let digest = index.index_digest(pos).unwrap(); |
1399c592 | 524 | if !self.chunk_store.cond_touch_chunk(digest, false)? { |
c23192d3 | 525 | task_warn!( |
f6b1d1cc | 526 | worker, |
d1d74c43 | 527 | "warning: unable to access non-existent chunk {}, required by {:?}", |
f6b1d1cc WB |
528 | proxmox::tools::digest_to_hex(digest), |
529 | file_name, | |
f6b1d1cc | 530 | ); |
fd192564 SR |
531 | |
532 | // touch any corresponding .bad files to keep them around, meaning if a chunk is | |
533 | // rewritten correctly they will be removed automatically, as well as if no index | |
534 | // file requires the chunk anymore (won't get to this loop then) | |
535 | for i in 0..=9 { | |
536 | let bad_ext = format!("{}.bad", i); | |
537 | let mut bad_path = PathBuf::new(); | |
538 | bad_path.push(self.chunk_path(digest).0); | |
539 | bad_path.set_extension(bad_ext); | |
540 | self.chunk_store.cond_touch_path(&bad_path, false)?; | |
541 | } | |
a660978c DM |
542 | } |
543 | } | |
544 | Ok(()) | |
545 | } | |
546 | ||
f6b1d1cc WB |
547 | fn mark_used_chunks( |
548 | &self, | |
549 | status: &mut GarbageCollectionStatus, | |
c8449217 | 550 | worker: &dyn WorkerTaskContext, |
f6b1d1cc | 551 | ) -> Result<(), Error> { |
3d5c11e5 DM |
552 | |
553 | let image_list = self.list_images()?; | |
8317873c DM |
554 | let image_count = image_list.len(); |
555 | ||
8317873c DM |
556 | let mut last_percentage: usize = 0; |
557 | ||
cb4b721c FG |
558 | let mut strange_paths_count: u64 = 0; |
559 | ||
ea368a06 | 560 | for (i, img) in image_list.into_iter().enumerate() { |
92da93b2 | 561 | |
f6b1d1cc | 562 | worker.check_abort()?; |
0fd55b08 | 563 | worker.fail_on_shutdown()?; |
92da93b2 | 564 | |
cb4b721c FG |
565 | if let Some(backup_dir_path) = img.parent() { |
566 | let backup_dir_path = backup_dir_path.strip_prefix(self.base_path())?; | |
567 | if let Some(backup_dir_str) = backup_dir_path.to_str() { | |
568 | if BackupDir::from_str(backup_dir_str).is_err() { | |
569 | strange_paths_count += 1; | |
570 | } | |
571 | } | |
572 | } | |
573 | ||
efcac39d | 574 | match std::fs::File::open(&img) { |
e0762002 | 575 | Ok(file) => { |
788d82d9 | 576 | if let Ok(archive_type) = archive_type(&img) { |
e0762002 | 577 | if archive_type == ArchiveType::FixedIndex { |
788d82d9 | 578 | let index = FixedIndexReader::new(file).map_err(|e| { |
efcac39d | 579 | format_err!("can't read index '{}' - {}", img.to_string_lossy(), e) |
2f0b9235 | 580 | })?; |
788d82d9 | 581 | self.index_mark_used_chunks(index, &img, status, worker)?; |
e0762002 | 582 | } else if archive_type == ArchiveType::DynamicIndex { |
788d82d9 | 583 | let index = DynamicIndexReader::new(file).map_err(|e| { |
efcac39d | 584 | format_err!("can't read index '{}' - {}", img.to_string_lossy(), e) |
2f0b9235 | 585 | })?; |
788d82d9 | 586 | self.index_mark_used_chunks(index, &img, status, worker)?; |
e0762002 DM |
587 | } |
588 | } | |
589 | } | |
788d82d9 | 590 | Err(err) if err.kind() == io::ErrorKind::NotFound => (), // ignore vanished files |
efcac39d | 591 | Err(err) => bail!("can't open index {} - {}", img.to_string_lossy(), err), |
77703d95 | 592 | } |
8317873c | 593 | |
ea368a06 | 594 | let percentage = (i + 1) * 100 / image_count; |
8317873c | 595 | if percentage > last_percentage { |
c23192d3 | 596 | task_log!( |
f6b1d1cc | 597 | worker, |
7956877f | 598 | "marked {}% ({} of {} index files)", |
f6b1d1cc | 599 | percentage, |
ea368a06 | 600 | i + 1, |
f6b1d1cc WB |
601 | image_count, |
602 | ); | |
8317873c DM |
603 | last_percentage = percentage; |
604 | } | |
3d5c11e5 DM |
605 | } |
606 | ||
cb4b721c | 607 | if strange_paths_count > 0 { |
c23192d3 | 608 | task_log!( |
cb4b721c FG |
609 | worker, |
610 | "found (and marked) {} index files outside of expected directory scheme", | |
611 | strange_paths_count, | |
612 | ); | |
613 | } | |
614 | ||
615 | ||
3d5c11e5 | 616 | Ok(()) |
f2b99c34 DM |
617 | } |
618 | ||
619 | pub fn last_gc_status(&self) -> GarbageCollectionStatus { | |
620 | self.last_gc_status.lock().unwrap().clone() | |
621 | } | |
3d5c11e5 | 622 | |
8545480a | 623 | pub fn garbage_collection_running(&self) -> bool { |
a6bd6698 | 624 | !matches!(self.gc_mutex.try_lock(), Ok(_)) |
8545480a DM |
625 | } |
626 | ||
c8449217 | 627 | pub fn garbage_collection(&self, worker: &dyn WorkerTaskContext, upid: &UPID) -> Result<(), Error> { |
3d5c11e5 | 628 | |
a198d74f | 629 | if let Ok(ref mut _mutex) = self.gc_mutex.try_lock() { |
e95950e4 | 630 | |
c6772c92 TL |
631 | // avoids that we run GC if an old daemon process has still a |
632 | // running backup writer, which is not save as we have no "oldest | |
633 | // writer" information and thus no safe atime cutoff | |
43b13033 DM |
634 | let _exclusive_lock = self.chunk_store.try_exclusive_lock()?; |
635 | ||
6ef1b649 | 636 | let phase1_start_time = proxmox_time::epoch_i64(); |
49a92084 | 637 | let oldest_writer = self.chunk_store.oldest_writer().unwrap_or(phase1_start_time); |
11861a48 | 638 | |
64e53b28 | 639 | let mut gc_status = GarbageCollectionStatus::default(); |
f6b1d1cc WB |
640 | gc_status.upid = Some(upid.to_string()); |
641 | ||
c23192d3 | 642 | task_log!(worker, "Start GC phase1 (mark used chunks)"); |
f6b1d1cc WB |
643 | |
644 | self.mark_used_chunks(&mut gc_status, worker)?; | |
645 | ||
c23192d3 | 646 | task_log!(worker, "Start GC phase2 (sweep unused chunks)"); |
f6b1d1cc WB |
647 | self.chunk_store.sweep_unused_chunks( |
648 | oldest_writer, | |
649 | phase1_start_time, | |
650 | &mut gc_status, | |
651 | worker, | |
652 | )?; | |
653 | ||
c23192d3 | 654 | task_log!( |
f6b1d1cc WB |
655 | worker, |
656 | "Removed garbage: {}", | |
657 | HumanByte::from(gc_status.removed_bytes), | |
658 | ); | |
c23192d3 | 659 | task_log!(worker, "Removed chunks: {}", gc_status.removed_chunks); |
cf459b19 | 660 | if gc_status.pending_bytes > 0 { |
c23192d3 | 661 | task_log!( |
f6b1d1cc WB |
662 | worker, |
663 | "Pending removals: {} (in {} chunks)", | |
664 | HumanByte::from(gc_status.pending_bytes), | |
665 | gc_status.pending_chunks, | |
666 | ); | |
cf459b19 | 667 | } |
a9767cf7 | 668 | if gc_status.removed_bad > 0 { |
c23192d3 | 669 | task_log!(worker, "Removed bad chunks: {}", gc_status.removed_bad); |
a9767cf7 | 670 | } |
cf459b19 | 671 | |
b4fb2623 | 672 | if gc_status.still_bad > 0 { |
c23192d3 | 673 | task_log!(worker, "Leftover bad chunks: {}", gc_status.still_bad); |
b4fb2623 DM |
674 | } |
675 | ||
c23192d3 | 676 | task_log!( |
f6b1d1cc WB |
677 | worker, |
678 | "Original data usage: {}", | |
679 | HumanByte::from(gc_status.index_data_bytes), | |
680 | ); | |
868c5852 DM |
681 | |
682 | if gc_status.index_data_bytes > 0 { | |
49a92084 | 683 | let comp_per = (gc_status.disk_bytes as f64 * 100.)/gc_status.index_data_bytes as f64; |
c23192d3 | 684 | task_log!( |
f6b1d1cc WB |
685 | worker, |
686 | "On-Disk usage: {} ({:.2}%)", | |
687 | HumanByte::from(gc_status.disk_bytes), | |
688 | comp_per, | |
689 | ); | |
868c5852 DM |
690 | } |
691 | ||
c23192d3 | 692 | task_log!(worker, "On-Disk chunks: {}", gc_status.disk_chunks); |
868c5852 | 693 | |
d6373f35 DM |
694 | let deduplication_factor = if gc_status.disk_bytes > 0 { |
695 | (gc_status.index_data_bytes as f64)/(gc_status.disk_bytes as f64) | |
696 | } else { | |
697 | 1.0 | |
698 | }; | |
699 | ||
c23192d3 | 700 | task_log!(worker, "Deduplication factor: {:.2}", deduplication_factor); |
d6373f35 | 701 | |
868c5852 DM |
702 | if gc_status.disk_chunks > 0 { |
703 | let avg_chunk = gc_status.disk_bytes/(gc_status.disk_chunks as u64); | |
c23192d3 | 704 | task_log!(worker, "Average chunk size: {}", HumanByte::from(avg_chunk)); |
868c5852 | 705 | } |
64e53b28 | 706 | |
b683fd58 DC |
707 | if let Ok(serialized) = serde_json::to_string(&gc_status) { |
708 | let mut path = self.base_path(); | |
709 | path.push(".gc-status"); | |
710 | ||
21211748 | 711 | let backup_user = pbs_config::backup_user()?; |
b683fd58 DC |
712 | let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644); |
713 | // set the correct owner/group/permissions while saving file | |
714 | // owner(rw) = backup, group(r)= backup | |
715 | let options = CreateOptions::new() | |
716 | .perm(mode) | |
717 | .owner(backup_user.uid) | |
718 | .group(backup_user.gid); | |
719 | ||
720 | // ignore errors | |
e0a19d33 | 721 | let _ = replace_file(path, serialized.as_bytes(), options, false); |
b683fd58 DC |
722 | } |
723 | ||
f2b99c34 DM |
724 | *self.last_gc_status.lock().unwrap() = gc_status; |
725 | ||
64e53b28 | 726 | } else { |
d4b59ae0 | 727 | bail!("Start GC failed - (already running/locked)"); |
64e53b28 | 728 | } |
3d5c11e5 DM |
729 | |
730 | Ok(()) | |
731 | } | |
3b7ade9e | 732 | |
ccc3896f | 733 | pub fn try_shared_chunk_store_lock(&self) -> Result<ProcessLockSharedGuard, Error> { |
1cf5178a DM |
734 | self.chunk_store.try_shared_lock() |
735 | } | |
736 | ||
d48a9955 DM |
737 | pub fn chunk_path(&self, digest:&[u8; 32]) -> (PathBuf, String) { |
738 | self.chunk_store.chunk_path(digest) | |
739 | } | |
740 | ||
2585a8a4 DM |
741 | pub fn cond_touch_chunk(&self, digest: &[u8; 32], fail_if_not_exist: bool) -> Result<bool, Error> { |
742 | self.chunk_store.cond_touch_chunk(digest, fail_if_not_exist) | |
743 | } | |
744 | ||
f98ac774 | 745 | pub fn insert_chunk( |
3b7ade9e | 746 | &self, |
4ee8f53d DM |
747 | chunk: &DataBlob, |
748 | digest: &[u8; 32], | |
3b7ade9e | 749 | ) -> Result<(bool, u64), Error> { |
4ee8f53d | 750 | self.chunk_store.insert_chunk(chunk, digest) |
3b7ade9e | 751 | } |
60f9a6ea | 752 | |
39f18b30 | 753 | pub fn load_blob(&self, backup_dir: &BackupDir, filename: &str) -> Result<DataBlob, Error> { |
60f9a6ea DM |
754 | let mut path = self.base_path(); |
755 | path.push(backup_dir.relative_path()); | |
756 | path.push(filename); | |
757 | ||
6ef1b649 | 758 | proxmox_lang::try_block!({ |
39f18b30 DM |
759 | let mut file = std::fs::File::open(&path)?; |
760 | DataBlob::load_from_reader(&mut file) | |
761 | }).map_err(|err| format_err!("unable to load blob '{:?}' - {}", path, err)) | |
762 | } | |
e4439025 DM |
763 | |
764 | ||
7f394c80 DC |
765 | pub fn stat_chunk(&self, digest: &[u8; 32]) -> Result<std::fs::Metadata, Error> { |
766 | let (chunk_path, _digest_str) = self.chunk_store.chunk_path(digest); | |
767 | std::fs::metadata(chunk_path).map_err(Error::from) | |
768 | } | |
769 | ||
39f18b30 DM |
770 | pub fn load_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> { |
771 | ||
772 | let (chunk_path, digest_str) = self.chunk_store.chunk_path(digest); | |
773 | ||
6ef1b649 | 774 | proxmox_lang::try_block!({ |
39f18b30 DM |
775 | let mut file = std::fs::File::open(&chunk_path)?; |
776 | DataBlob::load_from_reader(&mut file) | |
777 | }).map_err(|err| format_err!( | |
778 | "store '{}', unable to load chunk '{}' - {}", | |
779 | self.name(), | |
780 | digest_str, | |
781 | err, | |
782 | )) | |
1a374fcf SR |
783 | } |
784 | ||
179145dc DC |
785 | /// Returns the filename to lock a manifest |
786 | /// | |
787 | /// Also creates the basedir. The lockfile is located in | |
788 | /// '/run/proxmox-backup/locks/{datastore}/{type}/{id}/{timestamp}.index.json.lck' | |
789 | fn manifest_lock_path( | |
790 | &self, | |
791 | backup_dir: &BackupDir, | |
792 | ) -> Result<String, Error> { | |
793 | let mut path = format!( | |
794 | "/run/proxmox-backup/locks/{}/{}/{}", | |
795 | self.name(), | |
796 | backup_dir.group().backup_type(), | |
797 | backup_dir.group().backup_id(), | |
798 | ); | |
799 | std::fs::create_dir_all(&path)?; | |
800 | use std::fmt::Write; | |
801 | write!(path, "/{}{}", backup_dir.backup_time_string(), &MANIFEST_LOCK_NAME)?; | |
802 | ||
803 | Ok(path) | |
804 | } | |
805 | ||
1a374fcf SR |
806 | fn lock_manifest( |
807 | &self, | |
808 | backup_dir: &BackupDir, | |
7526d864 | 809 | ) -> Result<BackupLockGuard, Error> { |
179145dc | 810 | let path = self.manifest_lock_path(backup_dir)?; |
1a374fcf SR |
811 | |
812 | // update_manifest should never take a long time, so if someone else has | |
813 | // the lock we can simply block a bit and should get it soon | |
7526d864 | 814 | open_backup_lockfile(&path, Some(Duration::from_secs(5)), true) |
1a374fcf SR |
815 | .map_err(|err| { |
816 | format_err!( | |
817 | "unable to acquire manifest lock {:?} - {}", &path, err | |
818 | ) | |
819 | }) | |
820 | } | |
e4439025 | 821 | |
1a374fcf | 822 | /// Load the manifest without a lock. Must not be written back. |
521a0acb WB |
823 | pub fn load_manifest( |
824 | &self, | |
825 | backup_dir: &BackupDir, | |
ff86ef00 | 826 | ) -> Result<(BackupManifest, u64), Error> { |
39f18b30 DM |
827 | let blob = self.load_blob(backup_dir, MANIFEST_BLOB_NAME)?; |
828 | let raw_size = blob.raw_size(); | |
60f9a6ea | 829 | let manifest = BackupManifest::try_from(blob)?; |
ff86ef00 | 830 | Ok((manifest, raw_size)) |
60f9a6ea | 831 | } |
e4439025 | 832 | |
1a374fcf SR |
833 | /// Update the manifest of the specified snapshot. Never write a manifest directly, |
834 | /// only use this method - anything else may break locking guarantees. | |
835 | pub fn update_manifest( | |
e4439025 DM |
836 | &self, |
837 | backup_dir: &BackupDir, | |
1a374fcf | 838 | update_fn: impl FnOnce(&mut BackupManifest), |
e4439025 | 839 | ) -> Result<(), Error> { |
1a374fcf SR |
840 | |
841 | let _guard = self.lock_manifest(backup_dir)?; | |
842 | let (mut manifest, _) = self.load_manifest(&backup_dir)?; | |
843 | ||
844 | update_fn(&mut manifest); | |
845 | ||
883aa6d5 | 846 | let manifest = serde_json::to_value(manifest)?; |
e4439025 DM |
847 | let manifest = serde_json::to_string_pretty(&manifest)?; |
848 | let blob = DataBlob::encode(manifest.as_bytes(), None, true)?; | |
849 | let raw_data = blob.raw_data(); | |
850 | ||
851 | let mut path = self.base_path(); | |
852 | path.push(backup_dir.relative_path()); | |
853 | path.push(MANIFEST_BLOB_NAME); | |
854 | ||
1a374fcf | 855 | // atomic replace invalidates flock - no other writes past this point! |
e0a19d33 | 856 | replace_file(&path, raw_data, CreateOptions::new(), false)?; |
e4439025 DM |
857 | |
858 | Ok(()) | |
859 | } | |
0698f78d | 860 | |
8292d3d2 DC |
861 | /// Updates the protection status of the specified snapshot. |
862 | pub fn update_protection( | |
863 | &self, | |
864 | backup_dir: &BackupDir, | |
865 | protection: bool | |
866 | ) -> Result<(), Error> { | |
867 | let full_path = self.snapshot_path(backup_dir); | |
868 | ||
869 | let _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or in use")?; | |
870 | ||
871 | let protected_path = backup_dir.protected_file(self.base_path()); | |
872 | if protection { | |
873 | std::fs::File::create(protected_path) | |
874 | .map_err(|err| format_err!("could not create protection file: {}", err))?; | |
875 | } else if let Err(err) = std::fs::remove_file(protected_path) { | |
876 | // ignore error for non-existing file | |
877 | if err.kind() != std::io::ErrorKind::NotFound { | |
878 | bail!("could not remove protection file: {}", err); | |
879 | } | |
880 | } | |
881 | ||
882 | Ok(()) | |
883 | } | |
884 | ||
0698f78d SR |
885 | pub fn verify_new(&self) -> bool { |
886 | self.verify_new | |
887 | } | |
4921a411 DC |
888 | |
889 | /// returns a list of chunks sorted by their inode number on disk | |
890 | /// chunks that could not be stat'ed are at the end of the list | |
891 | pub fn get_chunks_in_order<F, A>( | |
892 | &self, | |
893 | index: &Box<dyn IndexFile + Send>, | |
894 | skip_chunk: F, | |
895 | check_abort: A, | |
896 | ) -> Result<Vec<(usize, u64)>, Error> | |
897 | where | |
898 | F: Fn(&[u8; 32]) -> bool, | |
899 | A: Fn(usize) -> Result<(), Error>, | |
900 | { | |
901 | let index_count = index.index_count(); | |
902 | let mut chunk_list = Vec::with_capacity(index_count); | |
903 | use std::os::unix::fs::MetadataExt; | |
904 | for pos in 0..index_count { | |
905 | check_abort(pos)?; | |
906 | ||
907 | let info = index.chunk_info(pos).unwrap(); | |
908 | ||
909 | if skip_chunk(&info.digest) { | |
910 | continue; | |
911 | } | |
912 | ||
913 | let ino = match self.stat_chunk(&info.digest) { | |
914 | Err(_) => u64::MAX, // could not stat, move to end of list | |
915 | Ok(metadata) => metadata.ino(), | |
916 | }; | |
917 | ||
918 | chunk_list.push((pos, ino)); | |
919 | } | |
920 | ||
921 | // sorting by inode improves data locality, which makes it lots faster on spinners | |
922 | chunk_list.sort_unstable_by(|(_, ino_a), (_, ino_b)| ino_a.cmp(&ino_b)); | |
923 | ||
924 | Ok(chunk_list) | |
925 | } | |
529de6c7 | 926 | } |