]>
Commit | Line | Data |
---|---|---|
1 | use std::collections::{HashMap, HashSet}; | |
2 | use std::io::{self, Write}; | |
3 | use std::os::unix::io::AsRawFd; | |
4 | use std::path::{Path, PathBuf}; | |
5 | use std::sync::{Arc, Mutex}; | |
6 | ||
7 | use anyhow::{bail, format_err, Error}; | |
8 | use lazy_static::lazy_static; | |
9 | use nix::unistd::{unlinkat, UnlinkatFlags}; | |
10 | ||
11 | use proxmox_human_byte::HumanByte; | |
12 | use proxmox_schema::ApiType; | |
13 | ||
14 | use proxmox_sys::error::SysError; | |
15 | use proxmox_sys::fs::{file_read_optional_string, replace_file, CreateOptions}; | |
16 | use proxmox_sys::fs::{lock_dir_noblock, DirLockGuard}; | |
17 | use proxmox_sys::process_locker::ProcessLockSharedGuard; | |
18 | use proxmox_sys::WorkerTaskContext; | |
19 | use proxmox_sys::{task_log, task_warn}; | |
20 | ||
21 | use pbs_api_types::{ | |
22 | Authid, BackupNamespace, BackupType, ChunkOrder, DataStoreConfig, DatastoreFSyncLevel, | |
23 | DatastoreTuning, GarbageCollectionStatus, Operation, UPID, | |
24 | }; | |
25 | ||
26 | use crate::backup_info::{BackupDir, BackupGroup}; | |
27 | use crate::chunk_store::ChunkStore; | |
28 | use crate::dynamic_index::{DynamicIndexReader, DynamicIndexWriter}; | |
29 | use crate::fixed_index::{FixedIndexReader, FixedIndexWriter}; | |
30 | use crate::hierarchy::{ListGroups, ListGroupsType, ListNamespaces, ListNamespacesRecursive}; | |
31 | use crate::index::IndexFile; | |
32 | use crate::manifest::{archive_type, ArchiveType}; | |
33 | use crate::task_tracking::{self, update_active_operations}; | |
34 | use crate::DataBlob; | |
35 | ||
36 | lazy_static! { | |
37 | static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStoreImpl>>> = | |
38 | Mutex::new(HashMap::new()); | |
39 | } | |
40 | ||
41 | /// checks if auth_id is owner, or, if owner is a token, if | |
42 | /// auth_id is the user of the token | |
43 | pub fn check_backup_owner(owner: &Authid, auth_id: &Authid) -> Result<(), Error> { | |
44 | let correct_owner = | |
45 | owner == auth_id || (owner.is_token() && &Authid::from(owner.user().clone()) == auth_id); | |
46 | if !correct_owner { | |
47 | bail!("backup owner check failed ({} != {})", auth_id, owner); | |
48 | } | |
49 | Ok(()) | |
50 | } | |
51 | ||
52 | /// Datastore Management | |
53 | /// | |
54 | /// A Datastore can store severals backups, and provides the | |
55 | /// management interface for backup. | |
56 | pub struct DataStoreImpl { | |
57 | chunk_store: Arc<ChunkStore>, | |
58 | gc_mutex: Mutex<()>, | |
59 | last_gc_status: Mutex<GarbageCollectionStatus>, | |
60 | verify_new: bool, | |
61 | chunk_order: ChunkOrder, | |
62 | last_digest: Option<[u8; 32]>, | |
63 | sync_level: DatastoreFSyncLevel, | |
64 | } | |
65 | ||
66 | impl DataStoreImpl { | |
67 | // This one just panics on everything | |
68 | #[doc(hidden)] | |
69 | pub(crate) unsafe fn new_test() -> Arc<Self> { | |
70 | Arc::new(Self { | |
71 | chunk_store: Arc::new(unsafe { ChunkStore::panic_store() }), | |
72 | gc_mutex: Mutex::new(()), | |
73 | last_gc_status: Mutex::new(GarbageCollectionStatus::default()), | |
74 | verify_new: false, | |
75 | chunk_order: Default::default(), | |
76 | last_digest: None, | |
77 | sync_level: Default::default(), | |
78 | }) | |
79 | } | |
80 | } | |
81 | ||
82 | pub struct DataStore { | |
83 | inner: Arc<DataStoreImpl>, | |
84 | operation: Option<Operation>, | |
85 | } | |
86 | ||
87 | impl Clone for DataStore { | |
88 | fn clone(&self) -> Self { | |
89 | let mut new_operation = self.operation; | |
90 | if let Some(operation) = self.operation { | |
91 | if let Err(e) = update_active_operations(self.name(), operation, 1) { | |
92 | log::error!("could not update active operations - {}", e); | |
93 | new_operation = None; | |
94 | } | |
95 | } | |
96 | ||
97 | DataStore { | |
98 | inner: self.inner.clone(), | |
99 | operation: new_operation, | |
100 | } | |
101 | } | |
102 | } | |
103 | ||
104 | impl Drop for DataStore { | |
105 | fn drop(&mut self) { | |
106 | if let Some(operation) = self.operation { | |
107 | if let Err(e) = update_active_operations(self.name(), operation, -1) { | |
108 | log::error!("could not update active operations - {}", e); | |
109 | } | |
110 | } | |
111 | } | |
112 | } | |
113 | ||
114 | impl DataStore { | |
115 | // This one just panics on everything | |
116 | #[doc(hidden)] | |
117 | pub(crate) unsafe fn new_test() -> Arc<Self> { | |
118 | Arc::new(Self { | |
119 | inner: unsafe { DataStoreImpl::new_test() }, | |
120 | operation: None, | |
121 | }) | |
122 | } | |
123 | ||
124 | pub fn lookup_datastore( | |
125 | name: &str, | |
126 | operation: Option<Operation>, | |
127 | ) -> Result<Arc<DataStore>, Error> { | |
128 | // Avoid TOCTOU between checking maintenance mode and updating active operation counter, as | |
129 | // we use it to decide whether it is okay to delete the datastore. | |
130 | let config_lock = pbs_config::datastore::lock_config()?; | |
131 | ||
132 | // we could use the ConfigVersionCache's generation for staleness detection, but we load | |
133 | // the config anyway -> just use digest, additional benefit: manual changes get detected | |
134 | let (config, digest) = pbs_config::datastore::config()?; | |
135 | let config: DataStoreConfig = config.lookup("datastore", name)?; | |
136 | ||
137 | if let Some(maintenance_mode) = config.get_maintenance_mode() { | |
138 | if let Err(error) = maintenance_mode.check(operation) { | |
139 | bail!("datastore '{name}' is in {error}"); | |
140 | } | |
141 | } | |
142 | ||
143 | if let Some(operation) = operation { | |
144 | update_active_operations(name, operation, 1)?; | |
145 | } | |
146 | ||
147 | // Our operation is registered, unlock the config. | |
148 | drop(config_lock); | |
149 | ||
150 | let mut datastore_cache = DATASTORE_MAP.lock().unwrap(); | |
151 | let entry = datastore_cache.get(name); | |
152 | ||
153 | // reuse chunk store so that we keep using the same process locker instance! | |
154 | let chunk_store = if let Some(datastore) = &entry { | |
155 | let last_digest = datastore.last_digest.as_ref(); | |
156 | if let Some(true) = last_digest.map(|last_digest| last_digest == &digest) { | |
157 | return Ok(Arc::new(Self { | |
158 | inner: Arc::clone(datastore), | |
159 | operation, | |
160 | })); | |
161 | } | |
162 | Arc::clone(&datastore.chunk_store) | |
163 | } else { | |
164 | let tuning: DatastoreTuning = serde_json::from_value( | |
165 | DatastoreTuning::API_SCHEMA | |
166 | .parse_property_string(config.tuning.as_deref().unwrap_or(""))?, | |
167 | )?; | |
168 | Arc::new(ChunkStore::open( | |
169 | name, | |
170 | &config.path, | |
171 | tuning.sync_level.unwrap_or_default(), | |
172 | )?) | |
173 | }; | |
174 | ||
175 | let datastore = DataStore::with_store_and_config(chunk_store, config, Some(digest))?; | |
176 | ||
177 | let datastore = Arc::new(datastore); | |
178 | datastore_cache.insert(name.to_string(), datastore.clone()); | |
179 | ||
180 | Ok(Arc::new(Self { | |
181 | inner: datastore, | |
182 | operation, | |
183 | })) | |
184 | } | |
185 | ||
186 | /// removes all datastores that are not configured anymore | |
187 | pub fn remove_unused_datastores() -> Result<(), Error> { | |
188 | let (config, _digest) = pbs_config::datastore::config()?; | |
189 | ||
190 | let mut map = DATASTORE_MAP.lock().unwrap(); | |
191 | // removes all elements that are not in the config | |
192 | map.retain(|key, _| config.sections.contains_key(key)); | |
193 | Ok(()) | |
194 | } | |
195 | ||
196 | /// Open a raw database given a name and a path. | |
197 | /// | |
198 | /// # Safety | |
199 | /// See the safety section in `open_from_config` | |
200 | pub unsafe fn open_path( | |
201 | name: &str, | |
202 | path: impl AsRef<Path>, | |
203 | operation: Option<Operation>, | |
204 | ) -> Result<Arc<Self>, Error> { | |
205 | let path = path | |
206 | .as_ref() | |
207 | .to_str() | |
208 | .ok_or_else(|| format_err!("non-utf8 paths not supported"))? | |
209 | .to_owned(); | |
210 | unsafe { Self::open_from_config(DataStoreConfig::new(name.to_owned(), path), operation) } | |
211 | } | |
212 | ||
213 | /// Open a datastore given a raw configuration. | |
214 | /// | |
215 | /// # Safety | |
216 | /// There's no memory safety implication, but as this is opening a new ChunkStore it will | |
217 | /// create a new process locker instance, potentially on the same path as existing safely | |
218 | /// created ones. This is dangerous as dropping the reference of this and thus the underlying | |
219 | /// chunkstore's process locker will close all locks from our process on the config.path, | |
220 | /// breaking guarantees we need to uphold for safe long backup + GC interaction on newer/older | |
221 | /// process instances (from package update). | |
222 | unsafe fn open_from_config( | |
223 | config: DataStoreConfig, | |
224 | operation: Option<Operation>, | |
225 | ) -> Result<Arc<Self>, Error> { | |
226 | let name = config.name.clone(); | |
227 | ||
228 | let tuning: DatastoreTuning = serde_json::from_value( | |
229 | DatastoreTuning::API_SCHEMA | |
230 | .parse_property_string(config.tuning.as_deref().unwrap_or(""))?, | |
231 | )?; | |
232 | let chunk_store = | |
233 | ChunkStore::open(&name, &config.path, tuning.sync_level.unwrap_or_default())?; | |
234 | let inner = Arc::new(Self::with_store_and_config( | |
235 | Arc::new(chunk_store), | |
236 | config, | |
237 | None, | |
238 | )?); | |
239 | ||
240 | if let Some(operation) = operation { | |
241 | update_active_operations(&name, operation, 1)?; | |
242 | } | |
243 | ||
244 | Ok(Arc::new(Self { inner, operation })) | |
245 | } | |
246 | ||
247 | fn with_store_and_config( | |
248 | chunk_store: Arc<ChunkStore>, | |
249 | config: DataStoreConfig, | |
250 | last_digest: Option<[u8; 32]>, | |
251 | ) -> Result<DataStoreImpl, Error> { | |
252 | let mut gc_status_path = chunk_store.base_path(); | |
253 | gc_status_path.push(".gc-status"); | |
254 | ||
255 | let gc_status = if let Some(state) = file_read_optional_string(gc_status_path)? { | |
256 | match serde_json::from_str(&state) { | |
257 | Ok(state) => state, | |
258 | Err(err) => { | |
259 | log::error!("error reading gc-status: {}", err); | |
260 | GarbageCollectionStatus::default() | |
261 | } | |
262 | } | |
263 | } else { | |
264 | GarbageCollectionStatus::default() | |
265 | }; | |
266 | ||
267 | let tuning: DatastoreTuning = serde_json::from_value( | |
268 | DatastoreTuning::API_SCHEMA | |
269 | .parse_property_string(config.tuning.as_deref().unwrap_or(""))?, | |
270 | )?; | |
271 | ||
272 | Ok(DataStoreImpl { | |
273 | chunk_store, | |
274 | gc_mutex: Mutex::new(()), | |
275 | last_gc_status: Mutex::new(gc_status), | |
276 | verify_new: config.verify_new.unwrap_or(false), | |
277 | chunk_order: tuning.chunk_order.unwrap_or_default(), | |
278 | last_digest, | |
279 | sync_level: tuning.sync_level.unwrap_or_default(), | |
280 | }) | |
281 | } | |
282 | ||
283 | pub fn get_chunk_iterator( | |
284 | &self, | |
285 | ) -> Result< | |
286 | impl Iterator<Item = (Result<proxmox_sys::fs::ReadDirEntry, Error>, usize, bool)>, | |
287 | Error, | |
288 | > { | |
289 | self.inner.chunk_store.get_chunk_iterator() | |
290 | } | |
291 | ||
292 | pub fn create_fixed_writer<P: AsRef<Path>>( | |
293 | &self, | |
294 | filename: P, | |
295 | size: usize, | |
296 | chunk_size: usize, | |
297 | ) -> Result<FixedIndexWriter, Error> { | |
298 | let index = FixedIndexWriter::create( | |
299 | self.inner.chunk_store.clone(), | |
300 | filename.as_ref(), | |
301 | size, | |
302 | chunk_size, | |
303 | )?; | |
304 | ||
305 | Ok(index) | |
306 | } | |
307 | ||
308 | pub fn open_fixed_reader<P: AsRef<Path>>( | |
309 | &self, | |
310 | filename: P, | |
311 | ) -> Result<FixedIndexReader, Error> { | |
312 | let full_path = self.inner.chunk_store.relative_path(filename.as_ref()); | |
313 | ||
314 | let index = FixedIndexReader::open(&full_path)?; | |
315 | ||
316 | Ok(index) | |
317 | } | |
318 | ||
319 | pub fn create_dynamic_writer<P: AsRef<Path>>( | |
320 | &self, | |
321 | filename: P, | |
322 | ) -> Result<DynamicIndexWriter, Error> { | |
323 | let index = DynamicIndexWriter::create(self.inner.chunk_store.clone(), filename.as_ref())?; | |
324 | ||
325 | Ok(index) | |
326 | } | |
327 | ||
328 | pub fn open_dynamic_reader<P: AsRef<Path>>( | |
329 | &self, | |
330 | filename: P, | |
331 | ) -> Result<DynamicIndexReader, Error> { | |
332 | let full_path = self.inner.chunk_store.relative_path(filename.as_ref()); | |
333 | ||
334 | let index = DynamicIndexReader::open(&full_path)?; | |
335 | ||
336 | Ok(index) | |
337 | } | |
338 | ||
339 | pub fn open_index<P>(&self, filename: P) -> Result<Box<dyn IndexFile + Send>, Error> | |
340 | where | |
341 | P: AsRef<Path>, | |
342 | { | |
343 | let filename = filename.as_ref(); | |
344 | let out: Box<dyn IndexFile + Send> = match archive_type(filename)? { | |
345 | ArchiveType::DynamicIndex => Box::new(self.open_dynamic_reader(filename)?), | |
346 | ArchiveType::FixedIndex => Box::new(self.open_fixed_reader(filename)?), | |
347 | _ => bail!("cannot open index file of unknown type: {:?}", filename), | |
348 | }; | |
349 | Ok(out) | |
350 | } | |
351 | ||
352 | /// Fast index verification - only check if chunks exists | |
353 | pub fn fast_index_verification( | |
354 | &self, | |
355 | index: &dyn IndexFile, | |
356 | checked: &mut HashSet<[u8; 32]>, | |
357 | ) -> Result<(), Error> { | |
358 | for pos in 0..index.index_count() { | |
359 | let info = index.chunk_info(pos).unwrap(); | |
360 | if checked.contains(&info.digest) { | |
361 | continue; | |
362 | } | |
363 | ||
364 | self.stat_chunk(&info.digest).map_err(|err| { | |
365 | format_err!( | |
366 | "fast_index_verification error, stat_chunk {} failed - {}", | |
367 | hex::encode(info.digest), | |
368 | err, | |
369 | ) | |
370 | })?; | |
371 | ||
372 | checked.insert(info.digest); | |
373 | } | |
374 | ||
375 | Ok(()) | |
376 | } | |
377 | ||
378 | pub fn name(&self) -> &str { | |
379 | self.inner.chunk_store.name() | |
380 | } | |
381 | ||
382 | pub fn base_path(&self) -> PathBuf { | |
383 | self.inner.chunk_store.base_path() | |
384 | } | |
385 | ||
386 | /// Returns the absolute path for a backup namespace on this datastore | |
387 | pub fn namespace_path(&self, ns: &BackupNamespace) -> PathBuf { | |
388 | let mut path = self.base_path(); | |
389 | path.reserve(ns.path_len()); | |
390 | for part in ns.components() { | |
391 | path.push("ns"); | |
392 | path.push(part); | |
393 | } | |
394 | path | |
395 | } | |
396 | ||
397 | /// Returns the absolute path for a backup_type | |
398 | pub fn type_path(&self, ns: &BackupNamespace, backup_type: BackupType) -> PathBuf { | |
399 | let mut full_path = self.namespace_path(ns); | |
400 | full_path.push(backup_type.to_string()); | |
401 | full_path | |
402 | } | |
403 | ||
404 | /// Returns the absolute path for a backup_group | |
405 | pub fn group_path( | |
406 | &self, | |
407 | ns: &BackupNamespace, | |
408 | backup_group: &pbs_api_types::BackupGroup, | |
409 | ) -> PathBuf { | |
410 | let mut full_path = self.namespace_path(ns); | |
411 | full_path.push(backup_group.to_string()); | |
412 | full_path | |
413 | } | |
414 | ||
415 | /// Returns the absolute path for backup_dir | |
416 | pub fn snapshot_path( | |
417 | &self, | |
418 | ns: &BackupNamespace, | |
419 | backup_dir: &pbs_api_types::BackupDir, | |
420 | ) -> PathBuf { | |
421 | let mut full_path = self.namespace_path(ns); | |
422 | full_path.push(backup_dir.to_string()); | |
423 | full_path | |
424 | } | |
425 | ||
426 | /// Create a backup namespace. | |
427 | pub fn create_namespace( | |
428 | self: &Arc<Self>, | |
429 | parent: &BackupNamespace, | |
430 | name: String, | |
431 | ) -> Result<BackupNamespace, Error> { | |
432 | if !self.namespace_exists(parent) { | |
433 | bail!("cannot create new namespace, parent {parent} doesn't already exists"); | |
434 | } | |
435 | ||
436 | // construct ns before mkdir to enforce max-depth and name validity | |
437 | let ns = BackupNamespace::from_parent_ns(parent, name)?; | |
438 | ||
439 | let mut ns_full_path = self.base_path(); | |
440 | ns_full_path.push(ns.path()); | |
441 | ||
442 | std::fs::create_dir_all(ns_full_path)?; | |
443 | ||
444 | Ok(ns) | |
445 | } | |
446 | ||
447 | /// Returns if the given namespace exists on the datastore | |
448 | pub fn namespace_exists(&self, ns: &BackupNamespace) -> bool { | |
449 | let mut path = self.base_path(); | |
450 | path.push(ns.path()); | |
451 | path.exists() | |
452 | } | |
453 | ||
454 | /// Remove all backup groups of a single namespace level but not the namespace itself. | |
455 | /// | |
456 | /// Does *not* descends into child-namespaces and doesn't remoes the namespace itself either. | |
457 | /// | |
458 | /// Returns true if all the groups were removed, and false if some were protected. | |
459 | pub fn remove_namespace_groups(self: &Arc<Self>, ns: &BackupNamespace) -> Result<bool, Error> { | |
460 | // FIXME: locking? The single groups/snapshots are already protected, so may not be | |
461 | // necessary (depends on what we all allow to do with namespaces) | |
462 | log::info!("removing all groups in namespace {}:/{ns}", self.name()); | |
463 | ||
464 | let mut removed_all_groups = true; | |
465 | ||
466 | for group in self.iter_backup_groups(ns.to_owned())? { | |
467 | let removed_group = group?.destroy()?; | |
468 | removed_all_groups = removed_all_groups && removed_group; | |
469 | } | |
470 | ||
471 | let base_file = std::fs::File::open(self.base_path())?; | |
472 | let base_fd = base_file.as_raw_fd(); | |
473 | for ty in BackupType::iter() { | |
474 | let mut ty_dir = ns.path(); | |
475 | ty_dir.push(ty.to_string()); | |
476 | // best effort only, but we probably should log the error | |
477 | if let Err(err) = unlinkat(Some(base_fd), &ty_dir, UnlinkatFlags::RemoveDir) { | |
478 | if err != nix::errno::Errno::ENOENT { | |
479 | log::error!("failed to remove backup type {ty} in {ns} - {err}"); | |
480 | } | |
481 | } | |
482 | } | |
483 | ||
484 | Ok(removed_all_groups) | |
485 | } | |
486 | ||
487 | /// Remove a complete backup namespace optionally including all it's, and child namespaces', | |
488 | /// groups. If `removed_groups` is false this only prunes empty namespaces. | |
489 | /// | |
490 | /// Returns true if everything requested, and false if some groups were protected or if some | |
491 | /// namespaces weren't empty even though all groups were deleted (race with new backup) | |
492 | pub fn remove_namespace_recursive( | |
493 | self: &Arc<Self>, | |
494 | ns: &BackupNamespace, | |
495 | delete_groups: bool, | |
496 | ) -> Result<bool, Error> { | |
497 | let store = self.name(); | |
498 | let mut removed_all_requested = true; | |
499 | if delete_groups { | |
500 | log::info!("removing whole namespace recursively below {store}:/{ns}",); | |
501 | for ns in self.recursive_iter_backup_ns(ns.to_owned())? { | |
502 | let removed_ns_groups = self.remove_namespace_groups(&ns?)?; | |
503 | removed_all_requested = removed_all_requested && removed_ns_groups; | |
504 | } | |
505 | } else { | |
506 | log::info!("pruning empty namespace recursively below {store}:/{ns}"); | |
507 | } | |
508 | ||
509 | // now try to delete the actual namespaces, bottom up so that we can use safe rmdir that | |
510 | // will choke if a new backup/group appeared in the meantime (but not on an new empty NS) | |
511 | let mut children = self | |
512 | .recursive_iter_backup_ns(ns.to_owned())? | |
513 | .collect::<Result<Vec<BackupNamespace>, Error>>()?; | |
514 | ||
515 | children.sort_by_key(|b| std::cmp::Reverse(b.depth())); | |
516 | ||
517 | let base_file = std::fs::File::open(self.base_path())?; | |
518 | let base_fd = base_file.as_raw_fd(); | |
519 | ||
520 | for ns in children.iter() { | |
521 | let mut ns_dir = ns.path(); | |
522 | ns_dir.push("ns"); | |
523 | let _ = unlinkat(Some(base_fd), &ns_dir, UnlinkatFlags::RemoveDir); | |
524 | ||
525 | if !ns.is_root() { | |
526 | match unlinkat(Some(base_fd), &ns.path(), UnlinkatFlags::RemoveDir) { | |
527 | Ok(()) => log::debug!("removed namespace {ns}"), | |
528 | Err(nix::errno::Errno::ENOENT) => { | |
529 | log::debug!("namespace {ns} already removed") | |
530 | } | |
531 | Err(nix::errno::Errno::ENOTEMPTY) if !delete_groups => { | |
532 | removed_all_requested = false; | |
533 | log::debug!("skip removal of non-empty namespace {ns}") | |
534 | } | |
535 | Err(err) => { | |
536 | removed_all_requested = false; | |
537 | log::warn!("failed to remove namespace {ns} - {err}") | |
538 | } | |
539 | } | |
540 | } | |
541 | } | |
542 | ||
543 | Ok(removed_all_requested) | |
544 | } | |
545 | ||
546 | /// Remove a complete backup group including all snapshots. | |
547 | /// | |
548 | /// Returns true if all snapshots were removed, and false if some were protected | |
549 | pub fn remove_backup_group( | |
550 | self: &Arc<Self>, | |
551 | ns: &BackupNamespace, | |
552 | backup_group: &pbs_api_types::BackupGroup, | |
553 | ) -> Result<bool, Error> { | |
554 | let backup_group = self.backup_group(ns.clone(), backup_group.clone()); | |
555 | ||
556 | backup_group.destroy() | |
557 | } | |
558 | ||
559 | /// Remove a backup directory including all content | |
560 | pub fn remove_backup_dir( | |
561 | self: &Arc<Self>, | |
562 | ns: &BackupNamespace, | |
563 | backup_dir: &pbs_api_types::BackupDir, | |
564 | force: bool, | |
565 | ) -> Result<(), Error> { | |
566 | let backup_dir = self.backup_dir(ns.clone(), backup_dir.clone())?; | |
567 | ||
568 | backup_dir.destroy(force) | |
569 | } | |
570 | ||
571 | /// Returns the time of the last successful backup | |
572 | /// | |
573 | /// Or None if there is no backup in the group (or the group dir does not exist). | |
574 | pub fn last_successful_backup( | |
575 | self: &Arc<Self>, | |
576 | ns: &BackupNamespace, | |
577 | backup_group: &pbs_api_types::BackupGroup, | |
578 | ) -> Result<Option<i64>, Error> { | |
579 | let backup_group = self.backup_group(ns.clone(), backup_group.clone()); | |
580 | ||
581 | let group_path = backup_group.full_group_path(); | |
582 | ||
583 | if group_path.exists() { | |
584 | backup_group.last_successful_backup() | |
585 | } else { | |
586 | Ok(None) | |
587 | } | |
588 | } | |
589 | ||
590 | /// Return the path of the 'owner' file. | |
591 | fn owner_path(&self, ns: &BackupNamespace, group: &pbs_api_types::BackupGroup) -> PathBuf { | |
592 | self.group_path(ns, group).join("owner") | |
593 | } | |
594 | ||
595 | /// Returns the backup owner. | |
596 | /// | |
597 | /// The backup owner is the entity who first created the backup group. | |
598 | pub fn get_owner( | |
599 | &self, | |
600 | ns: &BackupNamespace, | |
601 | backup_group: &pbs_api_types::BackupGroup, | |
602 | ) -> Result<Authid, Error> { | |
603 | let full_path = self.owner_path(ns, backup_group); | |
604 | let owner = proxmox_sys::fs::file_read_firstline(full_path)?; | |
605 | owner | |
606 | .trim_end() // remove trailing newline | |
607 | .parse() | |
608 | .map_err(|err| { | |
609 | format_err!("parsing owner for {backup_group} failed: {err}") | |
610 | }) | |
611 | } | |
612 | ||
613 | pub fn owns_backup( | |
614 | &self, | |
615 | ns: &BackupNamespace, | |
616 | backup_group: &pbs_api_types::BackupGroup, | |
617 | auth_id: &Authid, | |
618 | ) -> Result<bool, Error> { | |
619 | let owner = self.get_owner(ns, backup_group)?; | |
620 | ||
621 | Ok(check_backup_owner(&owner, auth_id).is_ok()) | |
622 | } | |
623 | ||
624 | /// Set the backup owner. | |
625 | pub fn set_owner( | |
626 | &self, | |
627 | ns: &BackupNamespace, | |
628 | backup_group: &pbs_api_types::BackupGroup, | |
629 | auth_id: &Authid, | |
630 | force: bool, | |
631 | ) -> Result<(), Error> { | |
632 | let path = self.owner_path(ns, backup_group); | |
633 | ||
634 | let mut open_options = std::fs::OpenOptions::new(); | |
635 | open_options.write(true); | |
636 | open_options.truncate(true); | |
637 | ||
638 | if force { | |
639 | open_options.create(true); | |
640 | } else { | |
641 | open_options.create_new(true); | |
642 | } | |
643 | ||
644 | let mut file = open_options | |
645 | .open(&path) | |
646 | .map_err(|err| format_err!("unable to create owner file {:?} - {}", path, err))?; | |
647 | ||
648 | writeln!(file, "{}", auth_id) | |
649 | .map_err(|err| format_err!("unable to write owner file {:?} - {}", path, err))?; | |
650 | ||
651 | Ok(()) | |
652 | } | |
653 | ||
654 | /// Create (if it does not already exists) and lock a backup group | |
655 | /// | |
656 | /// And set the owner to 'userid'. If the group already exists, it returns the | |
657 | /// current owner (instead of setting the owner). | |
658 | /// | |
659 | /// This also acquires an exclusive lock on the directory and returns the lock guard. | |
660 | pub fn create_locked_backup_group( | |
661 | &self, | |
662 | ns: &BackupNamespace, | |
663 | backup_group: &pbs_api_types::BackupGroup, | |
664 | auth_id: &Authid, | |
665 | ) -> Result<(Authid, DirLockGuard), Error> { | |
666 | // create intermediate path first: | |
667 | let mut full_path = self.base_path(); | |
668 | for ns in ns.components() { | |
669 | full_path.push("ns"); | |
670 | full_path.push(ns); | |
671 | } | |
672 | full_path.push(backup_group.ty.as_str()); | |
673 | std::fs::create_dir_all(&full_path)?; | |
674 | ||
675 | full_path.push(&backup_group.id); | |
676 | ||
677 | // create the last component now | |
678 | match std::fs::create_dir(&full_path) { | |
679 | Ok(_) => { | |
680 | let guard = lock_dir_noblock( | |
681 | &full_path, | |
682 | "backup group", | |
683 | "another backup is already running", | |
684 | )?; | |
685 | self.set_owner(ns, backup_group, auth_id, false)?; | |
686 | let owner = self.get_owner(ns, backup_group)?; // just to be sure | |
687 | Ok((owner, guard)) | |
688 | } | |
689 | Err(ref err) if err.kind() == io::ErrorKind::AlreadyExists => { | |
690 | let guard = lock_dir_noblock( | |
691 | &full_path, | |
692 | "backup group", | |
693 | "another backup is already running", | |
694 | )?; | |
695 | let owner = self.get_owner(ns, backup_group)?; // just to be sure | |
696 | Ok((owner, guard)) | |
697 | } | |
698 | Err(err) => bail!("unable to create backup group {:?} - {}", full_path, err), | |
699 | } | |
700 | } | |
701 | ||
702 | /// Creates a new backup snapshot inside a BackupGroup | |
703 | /// | |
704 | /// The BackupGroup directory needs to exist. | |
705 | pub fn create_locked_backup_dir( | |
706 | &self, | |
707 | ns: &BackupNamespace, | |
708 | backup_dir: &pbs_api_types::BackupDir, | |
709 | ) -> Result<(PathBuf, bool, DirLockGuard), Error> { | |
710 | let full_path = self.snapshot_path(ns, backup_dir); | |
711 | let relative_path = full_path.strip_prefix(self.base_path()).map_err(|err| { | |
712 | format_err!( | |
713 | "failed to produce correct path for backup {backup_dir} in namespace {ns}: {err}" | |
714 | ) | |
715 | })?; | |
716 | ||
717 | let lock = || { | |
718 | lock_dir_noblock( | |
719 | &full_path, | |
720 | "snapshot", | |
721 | "internal error - tried creating snapshot that's already in use", | |
722 | ) | |
723 | }; | |
724 | ||
725 | match std::fs::create_dir(&full_path) { | |
726 | Ok(_) => Ok((relative_path.to_owned(), true, lock()?)), | |
727 | Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => { | |
728 | Ok((relative_path.to_owned(), false, lock()?)) | |
729 | } | |
730 | Err(e) => Err(e.into()), | |
731 | } | |
732 | } | |
733 | ||
734 | /// Get a streaming iter over single-level backup namespaces of a datatstore | |
735 | /// | |
736 | /// The iterated item is still a Result that can contain errors from rather unexptected FS or | |
737 | /// parsing errors. | |
738 | pub fn iter_backup_ns( | |
739 | self: &Arc<DataStore>, | |
740 | ns: BackupNamespace, | |
741 | ) -> Result<ListNamespaces, Error> { | |
742 | ListNamespaces::new(Arc::clone(self), ns) | |
743 | } | |
744 | ||
745 | /// Get a streaming iter over single-level backup namespaces of a datatstore, filtered by Ok | |
746 | /// | |
747 | /// The iterated item's result is already unwrapped, if it contained an error it will be | |
748 | /// logged. Can be useful in iterator chain commands | |
749 | pub fn iter_backup_ns_ok( | |
750 | self: &Arc<DataStore>, | |
751 | ns: BackupNamespace, | |
752 | ) -> Result<impl Iterator<Item = BackupNamespace> + 'static, Error> { | |
753 | let this = Arc::clone(self); | |
754 | Ok( | |
755 | ListNamespaces::new(Arc::clone(self), ns)?.filter_map(move |ns| match ns { | |
756 | Ok(ns) => Some(ns), | |
757 | Err(err) => { | |
758 | log::error!("list groups error on datastore {} - {}", this.name(), err); | |
759 | None | |
760 | } | |
761 | }), | |
762 | ) | |
763 | } | |
764 | ||
765 | /// Get a streaming iter over single-level backup namespaces of a datatstore | |
766 | /// | |
767 | /// The iterated item is still a Result that can contain errors from rather unexptected FS or | |
768 | /// parsing errors. | |
769 | pub fn recursive_iter_backup_ns( | |
770 | self: &Arc<DataStore>, | |
771 | ns: BackupNamespace, | |
772 | ) -> Result<ListNamespacesRecursive, Error> { | |
773 | ListNamespacesRecursive::new(Arc::clone(self), ns) | |
774 | } | |
775 | ||
776 | /// Get a streaming iter over single-level backup namespaces of a datatstore, filtered by Ok | |
777 | /// | |
778 | /// The iterated item's result is already unwrapped, if it contained an error it will be | |
779 | /// logged. Can be useful in iterator chain commands | |
780 | pub fn recursive_iter_backup_ns_ok( | |
781 | self: &Arc<DataStore>, | |
782 | ns: BackupNamespace, | |
783 | max_depth: Option<usize>, | |
784 | ) -> Result<impl Iterator<Item = BackupNamespace> + 'static, Error> { | |
785 | let this = Arc::clone(self); | |
786 | Ok(if let Some(depth) = max_depth { | |
787 | ListNamespacesRecursive::new_max_depth(Arc::clone(self), ns, depth)? | |
788 | } else { | |
789 | ListNamespacesRecursive::new(Arc::clone(self), ns)? | |
790 | } | |
791 | .filter_map(move |ns| match ns { | |
792 | Ok(ns) => Some(ns), | |
793 | Err(err) => { | |
794 | log::error!("list groups error on datastore {} - {}", this.name(), err); | |
795 | None | |
796 | } | |
797 | })) | |
798 | } | |
799 | ||
800 | /// Get a streaming iter over top-level backup groups of a datatstore of a particular type. | |
801 | /// | |
802 | /// The iterated item is still a Result that can contain errors from rather unexptected FS or | |
803 | /// parsing errors. | |
804 | pub fn iter_backup_type( | |
805 | self: &Arc<DataStore>, | |
806 | ns: BackupNamespace, | |
807 | ty: BackupType, | |
808 | ) -> Result<ListGroupsType, Error> { | |
809 | ListGroupsType::new(Arc::clone(self), ns, ty) | |
810 | } | |
811 | ||
812 | /// Get a streaming iter over top-level backup groups of a datatstore of a particular type, | |
813 | /// filtered by `Ok` results | |
814 | /// | |
815 | /// The iterated item's result is already unwrapped, if it contained an error it will be | |
816 | /// logged. Can be useful in iterator chain commands | |
817 | pub fn iter_backup_type_ok( | |
818 | self: &Arc<DataStore>, | |
819 | ns: BackupNamespace, | |
820 | ty: BackupType, | |
821 | ) -> Result<impl Iterator<Item = BackupGroup> + 'static, Error> { | |
822 | Ok(self.iter_backup_type(ns, ty)?.ok()) | |
823 | } | |
824 | ||
825 | /// Get a streaming iter over top-level backup groups of a datatstore | |
826 | /// | |
827 | /// The iterated item is still a Result that can contain errors from rather unexptected FS or | |
828 | /// parsing errors. | |
829 | pub fn iter_backup_groups( | |
830 | self: &Arc<DataStore>, | |
831 | ns: BackupNamespace, | |
832 | ) -> Result<ListGroups, Error> { | |
833 | ListGroups::new(Arc::clone(self), ns) | |
834 | } | |
835 | ||
836 | /// Get a streaming iter over top-level backup groups of a datatstore, filtered by Ok results | |
837 | /// | |
838 | /// The iterated item's result is already unwrapped, if it contained an error it will be | |
839 | /// logged. Can be useful in iterator chain commands | |
840 | pub fn iter_backup_groups_ok( | |
841 | self: &Arc<DataStore>, | |
842 | ns: BackupNamespace, | |
843 | ) -> Result<impl Iterator<Item = BackupGroup> + 'static, Error> { | |
844 | Ok(self.iter_backup_groups(ns)?.ok()) | |
845 | } | |
846 | ||
847 | /// Get a in-memory vector for all top-level backup groups of a datatstore | |
848 | /// | |
849 | /// NOTE: using the iterator directly is most often more efficient w.r.t. memory usage | |
850 | pub fn list_backup_groups( | |
851 | self: &Arc<DataStore>, | |
852 | ns: BackupNamespace, | |
853 | ) -> Result<Vec<BackupGroup>, Error> { | |
854 | ListGroups::new(Arc::clone(self), ns)?.collect() | |
855 | } | |
856 | ||
857 | pub fn list_images(&self) -> Result<Vec<PathBuf>, Error> { | |
858 | let base = self.base_path(); | |
859 | ||
860 | let mut list = vec![]; | |
861 | ||
862 | use walkdir::WalkDir; | |
863 | ||
864 | let walker = WalkDir::new(base).into_iter(); | |
865 | ||
866 | // make sure we skip .chunks (and other hidden files to keep it simple) | |
867 | fn is_hidden(entry: &walkdir::DirEntry) -> bool { | |
868 | entry | |
869 | .file_name() | |
870 | .to_str() | |
871 | .map(|s| s.starts_with('.')) | |
872 | .unwrap_or(false) | |
873 | } | |
874 | let handle_entry_err = |err: walkdir::Error| { | |
875 | // first, extract the actual IO error and the affected path | |
876 | let (inner, path) = match (err.io_error(), err.path()) { | |
877 | (None, _) => return Ok(()), // not an IO-error | |
878 | (Some(inner), Some(path)) => (inner, path), | |
879 | (Some(inner), None) => bail!("unexpected error on datastore traversal: {inner}"), | |
880 | }; | |
881 | if inner.kind() == io::ErrorKind::PermissionDenied { | |
882 | if err.depth() <= 1 && path.ends_with("lost+found") { | |
883 | // allow skipping of (root-only) ext4 fsck-directory on EPERM .. | |
884 | return Ok(()); | |
885 | } | |
886 | // .. but do not ignore EPERM in general, otherwise we might prune too many chunks. | |
887 | // E.g., if users messed up with owner/perms on a rsync | |
888 | bail!("cannot continue garbage-collection safely, permission denied on: {path:?}"); | |
889 | } else if inner.kind() == io::ErrorKind::NotFound { | |
890 | log::info!("ignoring vanished file: {path:?}"); | |
891 | return Ok(()); | |
892 | } else { | |
893 | bail!("unexpected error on datastore traversal: {inner} - {path:?}"); | |
894 | } | |
895 | }; | |
896 | for entry in walker.filter_entry(|e| !is_hidden(e)) { | |
897 | let path = match entry { | |
898 | Ok(entry) => entry.into_path(), | |
899 | Err(err) => { | |
900 | handle_entry_err(err)?; | |
901 | continue; | |
902 | } | |
903 | }; | |
904 | if let Ok(archive_type) = archive_type(&path) { | |
905 | if archive_type == ArchiveType::FixedIndex | |
906 | || archive_type == ArchiveType::DynamicIndex | |
907 | { | |
908 | list.push(path); | |
909 | } | |
910 | } | |
911 | } | |
912 | ||
913 | Ok(list) | |
914 | } | |
915 | ||
916 | // mark chunks used by ``index`` as used | |
917 | fn index_mark_used_chunks<I: IndexFile>( | |
918 | &self, | |
919 | index: I, | |
920 | file_name: &Path, // only used for error reporting | |
921 | status: &mut GarbageCollectionStatus, | |
922 | worker: &dyn WorkerTaskContext, | |
923 | ) -> Result<(), Error> { | |
924 | status.index_file_count += 1; | |
925 | status.index_data_bytes += index.index_bytes(); | |
926 | ||
927 | for pos in 0..index.index_count() { | |
928 | worker.check_abort()?; | |
929 | worker.fail_on_shutdown()?; | |
930 | let digest = index.index_digest(pos).unwrap(); | |
931 | if !self.inner.chunk_store.cond_touch_chunk(digest, false)? { | |
932 | let hex = hex::encode(digest); | |
933 | task_warn!( | |
934 | worker, | |
935 | "warning: unable to access non-existent chunk {hex}, required by {file_name:?}" | |
936 | ); | |
937 | ||
938 | // touch any corresponding .bad files to keep them around, meaning if a chunk is | |
939 | // rewritten correctly they will be removed automatically, as well as if no index | |
940 | // file requires the chunk anymore (won't get to this loop then) | |
941 | for i in 0..=9 { | |
942 | let bad_ext = format!("{}.bad", i); | |
943 | let mut bad_path = PathBuf::new(); | |
944 | bad_path.push(self.chunk_path(digest).0); | |
945 | bad_path.set_extension(bad_ext); | |
946 | self.inner.chunk_store.cond_touch_path(&bad_path, false)?; | |
947 | } | |
948 | } | |
949 | } | |
950 | Ok(()) | |
951 | } | |
952 | ||
953 | fn mark_used_chunks( | |
954 | &self, | |
955 | status: &mut GarbageCollectionStatus, | |
956 | worker: &dyn WorkerTaskContext, | |
957 | ) -> Result<(), Error> { | |
958 | let image_list = self.list_images()?; | |
959 | let image_count = image_list.len(); | |
960 | ||
961 | let mut last_percentage: usize = 0; | |
962 | ||
963 | let mut strange_paths_count: u64 = 0; | |
964 | ||
965 | for (i, img) in image_list.into_iter().enumerate() { | |
966 | worker.check_abort()?; | |
967 | worker.fail_on_shutdown()?; | |
968 | ||
969 | if let Some(backup_dir_path) = img.parent() { | |
970 | let backup_dir_path = backup_dir_path.strip_prefix(self.base_path())?; | |
971 | if let Some(backup_dir_str) = backup_dir_path.to_str() { | |
972 | if pbs_api_types::parse_ns_and_snapshot(backup_dir_str).is_err() { | |
973 | strange_paths_count += 1; | |
974 | } | |
975 | } | |
976 | } | |
977 | ||
978 | match std::fs::File::open(&img) { | |
979 | Ok(file) => { | |
980 | if let Ok(archive_type) = archive_type(&img) { | |
981 | if archive_type == ArchiveType::FixedIndex { | |
982 | let index = FixedIndexReader::new(file).map_err(|e| { | |
983 | format_err!("can't read index '{}' - {}", img.to_string_lossy(), e) | |
984 | })?; | |
985 | self.index_mark_used_chunks(index, &img, status, worker)?; | |
986 | } else if archive_type == ArchiveType::DynamicIndex { | |
987 | let index = DynamicIndexReader::new(file).map_err(|e| { | |
988 | format_err!("can't read index '{}' - {}", img.to_string_lossy(), e) | |
989 | })?; | |
990 | self.index_mark_used_chunks(index, &img, status, worker)?; | |
991 | } | |
992 | } | |
993 | } | |
994 | Err(err) if err.kind() == io::ErrorKind::NotFound => (), // ignore vanished files | |
995 | Err(err) => bail!("can't open index {} - {}", img.to_string_lossy(), err), | |
996 | } | |
997 | ||
998 | let percentage = (i + 1) * 100 / image_count; | |
999 | if percentage > last_percentage { | |
1000 | task_log!( | |
1001 | worker, | |
1002 | "marked {}% ({} of {} index files)", | |
1003 | percentage, | |
1004 | i + 1, | |
1005 | image_count, | |
1006 | ); | |
1007 | last_percentage = percentage; | |
1008 | } | |
1009 | } | |
1010 | ||
1011 | if strange_paths_count > 0 { | |
1012 | task_log!( | |
1013 | worker, | |
1014 | "found (and marked) {} index files outside of expected directory scheme", | |
1015 | strange_paths_count, | |
1016 | ); | |
1017 | } | |
1018 | ||
1019 | Ok(()) | |
1020 | } | |
1021 | ||
1022 | pub fn last_gc_status(&self) -> GarbageCollectionStatus { | |
1023 | self.inner.last_gc_status.lock().unwrap().clone() | |
1024 | } | |
1025 | ||
1026 | pub fn garbage_collection_running(&self) -> bool { | |
1027 | !matches!(self.inner.gc_mutex.try_lock(), Ok(_)) | |
1028 | } | |
1029 | ||
1030 | pub fn garbage_collection( | |
1031 | &self, | |
1032 | worker: &dyn WorkerTaskContext, | |
1033 | upid: &UPID, | |
1034 | ) -> Result<(), Error> { | |
1035 | if let Ok(ref mut _mutex) = self.inner.gc_mutex.try_lock() { | |
1036 | // avoids that we run GC if an old daemon process has still a | |
1037 | // running backup writer, which is not save as we have no "oldest | |
1038 | // writer" information and thus no safe atime cutoff | |
1039 | let _exclusive_lock = self.inner.chunk_store.try_exclusive_lock()?; | |
1040 | ||
1041 | let phase1_start_time = proxmox_time::epoch_i64(); | |
1042 | let oldest_writer = self | |
1043 | .inner | |
1044 | .chunk_store | |
1045 | .oldest_writer() | |
1046 | .unwrap_or(phase1_start_time); | |
1047 | ||
1048 | let mut gc_status = GarbageCollectionStatus { | |
1049 | upid: Some(upid.to_string()), | |
1050 | ..Default::default() | |
1051 | }; | |
1052 | ||
1053 | task_log!(worker, "Start GC phase1 (mark used chunks)"); | |
1054 | ||
1055 | self.mark_used_chunks(&mut gc_status, worker)?; | |
1056 | ||
1057 | task_log!(worker, "Start GC phase2 (sweep unused chunks)"); | |
1058 | self.inner.chunk_store.sweep_unused_chunks( | |
1059 | oldest_writer, | |
1060 | phase1_start_time, | |
1061 | &mut gc_status, | |
1062 | worker, | |
1063 | )?; | |
1064 | ||
1065 | task_log!( | |
1066 | worker, | |
1067 | "Removed garbage: {}", | |
1068 | HumanByte::from(gc_status.removed_bytes), | |
1069 | ); | |
1070 | task_log!(worker, "Removed chunks: {}", gc_status.removed_chunks); | |
1071 | if gc_status.pending_bytes > 0 { | |
1072 | task_log!( | |
1073 | worker, | |
1074 | "Pending removals: {} (in {} chunks)", | |
1075 | HumanByte::from(gc_status.pending_bytes), | |
1076 | gc_status.pending_chunks, | |
1077 | ); | |
1078 | } | |
1079 | if gc_status.removed_bad > 0 { | |
1080 | task_log!(worker, "Removed bad chunks: {}", gc_status.removed_bad); | |
1081 | } | |
1082 | ||
1083 | if gc_status.still_bad > 0 { | |
1084 | task_log!(worker, "Leftover bad chunks: {}", gc_status.still_bad); | |
1085 | } | |
1086 | ||
1087 | task_log!( | |
1088 | worker, | |
1089 | "Original data usage: {}", | |
1090 | HumanByte::from(gc_status.index_data_bytes), | |
1091 | ); | |
1092 | ||
1093 | if gc_status.index_data_bytes > 0 { | |
1094 | let comp_per = | |
1095 | (gc_status.disk_bytes as f64 * 100.) / gc_status.index_data_bytes as f64; | |
1096 | task_log!( | |
1097 | worker, | |
1098 | "On-Disk usage: {} ({:.2}%)", | |
1099 | HumanByte::from(gc_status.disk_bytes), | |
1100 | comp_per, | |
1101 | ); | |
1102 | } | |
1103 | ||
1104 | task_log!(worker, "On-Disk chunks: {}", gc_status.disk_chunks); | |
1105 | ||
1106 | let deduplication_factor = if gc_status.disk_bytes > 0 { | |
1107 | (gc_status.index_data_bytes as f64) / (gc_status.disk_bytes as f64) | |
1108 | } else { | |
1109 | 1.0 | |
1110 | }; | |
1111 | ||
1112 | task_log!(worker, "Deduplication factor: {:.2}", deduplication_factor); | |
1113 | ||
1114 | if gc_status.disk_chunks > 0 { | |
1115 | let avg_chunk = gc_status.disk_bytes / (gc_status.disk_chunks as u64); | |
1116 | task_log!(worker, "Average chunk size: {}", HumanByte::from(avg_chunk)); | |
1117 | } | |
1118 | ||
1119 | if let Ok(serialized) = serde_json::to_string(&gc_status) { | |
1120 | let mut path = self.base_path(); | |
1121 | path.push(".gc-status"); | |
1122 | ||
1123 | let backup_user = pbs_config::backup_user()?; | |
1124 | let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644); | |
1125 | // set the correct owner/group/permissions while saving file | |
1126 | // owner(rw) = backup, group(r)= backup | |
1127 | let options = CreateOptions::new() | |
1128 | .perm(mode) | |
1129 | .owner(backup_user.uid) | |
1130 | .group(backup_user.gid); | |
1131 | ||
1132 | // ignore errors | |
1133 | let _ = replace_file(path, serialized.as_bytes(), options, false); | |
1134 | } | |
1135 | ||
1136 | *self.inner.last_gc_status.lock().unwrap() = gc_status; | |
1137 | } else { | |
1138 | bail!("Start GC failed - (already running/locked)"); | |
1139 | } | |
1140 | ||
1141 | Ok(()) | |
1142 | } | |
1143 | ||
1144 | pub fn try_shared_chunk_store_lock(&self) -> Result<ProcessLockSharedGuard, Error> { | |
1145 | self.inner.chunk_store.try_shared_lock() | |
1146 | } | |
1147 | ||
1148 | pub fn chunk_path(&self, digest: &[u8; 32]) -> (PathBuf, String) { | |
1149 | self.inner.chunk_store.chunk_path(digest) | |
1150 | } | |
1151 | ||
1152 | pub fn cond_touch_chunk(&self, digest: &[u8; 32], assert_exists: bool) -> Result<bool, Error> { | |
1153 | self.inner | |
1154 | .chunk_store | |
1155 | .cond_touch_chunk(digest, assert_exists) | |
1156 | } | |
1157 | ||
1158 | pub fn insert_chunk(&self, chunk: &DataBlob, digest: &[u8; 32]) -> Result<(bool, u64), Error> { | |
1159 | self.inner.chunk_store.insert_chunk(chunk, digest) | |
1160 | } | |
1161 | ||
1162 | pub fn stat_chunk(&self, digest: &[u8; 32]) -> Result<std::fs::Metadata, Error> { | |
1163 | let (chunk_path, _digest_str) = self.inner.chunk_store.chunk_path(digest); | |
1164 | std::fs::metadata(chunk_path).map_err(Error::from) | |
1165 | } | |
1166 | ||
1167 | pub fn load_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> { | |
1168 | let (chunk_path, digest_str) = self.inner.chunk_store.chunk_path(digest); | |
1169 | ||
1170 | proxmox_lang::try_block!({ | |
1171 | let mut file = std::fs::File::open(&chunk_path)?; | |
1172 | DataBlob::load_from_reader(&mut file) | |
1173 | }) | |
1174 | .map_err(|err| { | |
1175 | format_err!( | |
1176 | "store '{}', unable to load chunk '{}' - {}", | |
1177 | self.name(), | |
1178 | digest_str, | |
1179 | err, | |
1180 | ) | |
1181 | }) | |
1182 | } | |
1183 | ||
1184 | /// Updates the protection status of the specified snapshot. | |
1185 | pub fn update_protection(&self, backup_dir: &BackupDir, protection: bool) -> Result<(), Error> { | |
1186 | let full_path = backup_dir.full_path(); | |
1187 | ||
1188 | if !full_path.exists() { | |
1189 | bail!("snapshot {} does not exist!", backup_dir.dir()); | |
1190 | } | |
1191 | ||
1192 | let _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or in use")?; | |
1193 | ||
1194 | let protected_path = backup_dir.protected_file(); | |
1195 | if protection { | |
1196 | std::fs::File::create(protected_path) | |
1197 | .map_err(|err| format_err!("could not create protection file: {}", err))?; | |
1198 | } else if let Err(err) = std::fs::remove_file(protected_path) { | |
1199 | // ignore error for non-existing file | |
1200 | if err.kind() != std::io::ErrorKind::NotFound { | |
1201 | bail!("could not remove protection file: {}", err); | |
1202 | } | |
1203 | } | |
1204 | ||
1205 | Ok(()) | |
1206 | } | |
1207 | ||
1208 | pub fn verify_new(&self) -> bool { | |
1209 | self.inner.verify_new | |
1210 | } | |
1211 | ||
1212 | /// returns a list of chunks sorted by their inode number on disk chunks that couldn't get | |
1213 | /// stat'ed are placed at the end of the list | |
1214 | pub fn get_chunks_in_order<F, A>( | |
1215 | &self, | |
1216 | index: &(dyn IndexFile + Send), | |
1217 | skip_chunk: F, | |
1218 | check_abort: A, | |
1219 | ) -> Result<Vec<(usize, u64)>, Error> | |
1220 | where | |
1221 | F: Fn(&[u8; 32]) -> bool, | |
1222 | A: Fn(usize) -> Result<(), Error>, | |
1223 | { | |
1224 | let index_count = index.index_count(); | |
1225 | let mut chunk_list = Vec::with_capacity(index_count); | |
1226 | use std::os::unix::fs::MetadataExt; | |
1227 | for pos in 0..index_count { | |
1228 | check_abort(pos)?; | |
1229 | ||
1230 | let info = index.chunk_info(pos).unwrap(); | |
1231 | ||
1232 | if skip_chunk(&info.digest) { | |
1233 | continue; | |
1234 | } | |
1235 | ||
1236 | let ino = match self.inner.chunk_order { | |
1237 | ChunkOrder::Inode => { | |
1238 | match self.stat_chunk(&info.digest) { | |
1239 | Err(_) => u64::MAX, // could not stat, move to end of list | |
1240 | Ok(metadata) => metadata.ino(), | |
1241 | } | |
1242 | } | |
1243 | ChunkOrder::None => 0, | |
1244 | }; | |
1245 | ||
1246 | chunk_list.push((pos, ino)); | |
1247 | } | |
1248 | ||
1249 | match self.inner.chunk_order { | |
1250 | // sorting by inode improves data locality, which makes it lots faster on spinners | |
1251 | ChunkOrder::Inode => { | |
1252 | chunk_list.sort_unstable_by(|(_, ino_a), (_, ino_b)| ino_a.cmp(ino_b)) | |
1253 | } | |
1254 | ChunkOrder::None => {} | |
1255 | } | |
1256 | ||
1257 | Ok(chunk_list) | |
1258 | } | |
1259 | ||
1260 | /// Open a backup group from this datastore. | |
1261 | pub fn backup_group( | |
1262 | self: &Arc<Self>, | |
1263 | ns: BackupNamespace, | |
1264 | group: pbs_api_types::BackupGroup, | |
1265 | ) -> BackupGroup { | |
1266 | BackupGroup::new(Arc::clone(self), ns, group) | |
1267 | } | |
1268 | ||
1269 | /// Open a backup group from this datastore. | |
1270 | pub fn backup_group_from_parts<T>( | |
1271 | self: &Arc<Self>, | |
1272 | ns: BackupNamespace, | |
1273 | ty: BackupType, | |
1274 | id: T, | |
1275 | ) -> BackupGroup | |
1276 | where | |
1277 | T: Into<String>, | |
1278 | { | |
1279 | self.backup_group(ns, (ty, id.into()).into()) | |
1280 | } | |
1281 | ||
1282 | /* | |
1283 | /// Open a backup group from this datastore by backup group path such as `vm/100`. | |
1284 | /// | |
1285 | /// Convenience method for `store.backup_group(path.parse()?)` | |
1286 | pub fn backup_group_from_path(self: &Arc<Self>, path: &str) -> Result<BackupGroup, Error> { | |
1287 | todo!("split out the namespace"); | |
1288 | } | |
1289 | */ | |
1290 | ||
1291 | /// Open a snapshot (backup directory) from this datastore. | |
1292 | pub fn backup_dir( | |
1293 | self: &Arc<Self>, | |
1294 | ns: BackupNamespace, | |
1295 | dir: pbs_api_types::BackupDir, | |
1296 | ) -> Result<BackupDir, Error> { | |
1297 | BackupDir::with_group(self.backup_group(ns, dir.group), dir.time) | |
1298 | } | |
1299 | ||
1300 | /// Open a snapshot (backup directory) from this datastore. | |
1301 | pub fn backup_dir_from_parts<T>( | |
1302 | self: &Arc<Self>, | |
1303 | ns: BackupNamespace, | |
1304 | ty: BackupType, | |
1305 | id: T, | |
1306 | time: i64, | |
1307 | ) -> Result<BackupDir, Error> | |
1308 | where | |
1309 | T: Into<String>, | |
1310 | { | |
1311 | self.backup_dir(ns, (ty, id.into(), time).into()) | |
1312 | } | |
1313 | ||
1314 | /// Open a snapshot (backup directory) from this datastore with a cached rfc3339 time string. | |
1315 | pub fn backup_dir_with_rfc3339<T: Into<String>>( | |
1316 | self: &Arc<Self>, | |
1317 | group: BackupGroup, | |
1318 | time_string: T, | |
1319 | ) -> Result<BackupDir, Error> { | |
1320 | BackupDir::with_rfc3339(group, time_string.into()) | |
1321 | } | |
1322 | ||
1323 | /* | |
1324 | /// Open a snapshot (backup directory) from this datastore by a snapshot path. | |
1325 | pub fn backup_dir_from_path(self: &Arc<Self>, path: &str) -> Result<BackupDir, Error> { | |
1326 | todo!("split out the namespace"); | |
1327 | } | |
1328 | */ | |
1329 | ||
1330 | /// Syncs the filesystem of the datastore if 'sync_level' is set to | |
1331 | /// [`DatastoreFSyncLevel::Filesystem`]. Uses syncfs(2). | |
1332 | pub fn try_ensure_sync_level(&self) -> Result<(), Error> { | |
1333 | if self.inner.sync_level != DatastoreFSyncLevel::Filesystem { | |
1334 | return Ok(()); | |
1335 | } | |
1336 | let file = std::fs::File::open(self.base_path())?; | |
1337 | let fd = file.as_raw_fd(); | |
1338 | log::info!("syncing filesystem"); | |
1339 | if unsafe { libc::syncfs(fd) } < 0 { | |
1340 | bail!("error during syncfs: {}", std::io::Error::last_os_error()); | |
1341 | } | |
1342 | Ok(()) | |
1343 | } | |
1344 | ||
1345 | /// Destroy a datastore. This requires that there are no active operations on the datastore. | |
1346 | /// | |
1347 | /// This is a synchronous operation and should be run in a worker-thread. | |
1348 | pub fn destroy( | |
1349 | name: &str, | |
1350 | destroy_data: bool, | |
1351 | worker: &dyn WorkerTaskContext, | |
1352 | ) -> Result<(), Error> { | |
1353 | let config_lock = pbs_config::datastore::lock_config()?; | |
1354 | ||
1355 | let (mut config, _digest) = pbs_config::datastore::config()?; | |
1356 | let mut datastore_config: DataStoreConfig = config.lookup("datastore", name)?; | |
1357 | ||
1358 | datastore_config.maintenance_mode = Some("type=delete".to_string()); | |
1359 | config.set_data(name, "datastore", &datastore_config)?; | |
1360 | pbs_config::datastore::save_config(&config)?; | |
1361 | drop(config_lock); | |
1362 | ||
1363 | let (operations, _lock) = task_tracking::get_active_operations_locked(name)?; | |
1364 | ||
1365 | if operations.read != 0 || operations.write != 0 { | |
1366 | bail!("datastore is currently in use"); | |
1367 | } | |
1368 | ||
1369 | let base = PathBuf::from(&datastore_config.path); | |
1370 | ||
1371 | let mut ok = true; | |
1372 | if destroy_data { | |
1373 | let remove = |subdir, ok: &mut bool| { | |
1374 | if let Err(err) = std::fs::remove_dir_all(base.join(subdir)) { | |
1375 | if err.kind() != io::ErrorKind::NotFound { | |
1376 | task_warn!(worker, "failed to remove {subdir:?} subdirectory: {err}"); | |
1377 | *ok = false; | |
1378 | } | |
1379 | } | |
1380 | }; | |
1381 | ||
1382 | task_log!(worker, "Deleting datastore data..."); | |
1383 | remove("ns", &mut ok); // ns first | |
1384 | remove("ct", &mut ok); | |
1385 | remove("vm", &mut ok); | |
1386 | remove("host", &mut ok); | |
1387 | ||
1388 | if ok { | |
1389 | if let Err(err) = std::fs::remove_file(base.join(".gc-status")) { | |
1390 | if err.kind() != io::ErrorKind::NotFound { | |
1391 | task_warn!(worker, "failed to remove .gc-status file: {err}"); | |
1392 | ok = false; | |
1393 | } | |
1394 | } | |
1395 | } | |
1396 | ||
1397 | // chunks get removed last and only if the backups were successfully deleted | |
1398 | if ok { | |
1399 | remove(".chunks", &mut ok); | |
1400 | } | |
1401 | } | |
1402 | ||
1403 | // now the config | |
1404 | if ok { | |
1405 | task_log!(worker, "Removing datastore from config..."); | |
1406 | let _lock = pbs_config::datastore::lock_config()?; | |
1407 | let _ = config.sections.remove(name); | |
1408 | pbs_config::datastore::save_config(&config)?; | |
1409 | } | |
1410 | ||
1411 | // finally the lock & toplevel directory | |
1412 | if destroy_data { | |
1413 | if ok { | |
1414 | if let Err(err) = std::fs::remove_file(base.join(".lock")) { | |
1415 | if err.kind() != io::ErrorKind::NotFound { | |
1416 | task_warn!(worker, "failed to remove .lock file: {err}"); | |
1417 | ok = false; | |
1418 | } | |
1419 | } | |
1420 | } | |
1421 | ||
1422 | if ok { | |
1423 | task_log!(worker, "Finished deleting data."); | |
1424 | ||
1425 | match std::fs::remove_dir(base) { | |
1426 | Ok(()) => task_log!(worker, "Removed empty datastore directory."), | |
1427 | Err(err) if err.kind() == io::ErrorKind::NotFound => { | |
1428 | // weird, but ok | |
1429 | } | |
1430 | Err(err) if err.is_errno(nix::errno::Errno::EBUSY) => { | |
1431 | task_warn!( | |
1432 | worker, | |
1433 | "Cannot delete datastore directory (is it a mount point?)." | |
1434 | ) | |
1435 | } | |
1436 | Err(err) if err.is_errno(nix::errno::Errno::ENOTEMPTY) => { | |
1437 | task_warn!(worker, "Datastore directory not empty, not deleting.") | |
1438 | } | |
1439 | Err(err) => { | |
1440 | task_warn!(worker, "Failed to remove datastore directory: {err}"); | |
1441 | } | |
1442 | } | |
1443 | } else { | |
1444 | task_log!(worker, "There were errors deleting data."); | |
1445 | } | |
1446 | } | |
1447 | ||
1448 | Ok(()) | |
1449 | } | |
1450 | } |