]> git.proxmox.com Git - proxmox-backup.git/blame_incremental - pbs-datastore/src/datastore.rs
api: custom certificate upload: make key optional
[proxmox-backup.git] / pbs-datastore / src / datastore.rs
... / ...
CommitLineData
1use std::collections::{HashMap, HashSet};
2use std::io::{self, Write};
3use std::os::unix::io::AsRawFd;
4use std::path::{Path, PathBuf};
5use std::sync::{Arc, Mutex};
6
7use anyhow::{bail, format_err, Error};
8use lazy_static::lazy_static;
9use nix::unistd::{unlinkat, UnlinkatFlags};
10
11use proxmox_human_byte::HumanByte;
12use proxmox_schema::ApiType;
13
14use proxmox_sys::error::SysError;
15use proxmox_sys::fs::{file_read_optional_string, replace_file, CreateOptions};
16use proxmox_sys::fs::{lock_dir_noblock, DirLockGuard};
17use proxmox_sys::process_locker::ProcessLockSharedGuard;
18use proxmox_sys::WorkerTaskContext;
19use proxmox_sys::{task_log, task_warn};
20
21use pbs_api_types::{
22 Authid, BackupNamespace, BackupType, ChunkOrder, DataStoreConfig, DatastoreFSyncLevel,
23 DatastoreTuning, GarbageCollectionStatus, Operation, UPID,
24};
25
26use crate::backup_info::{BackupDir, BackupGroup};
27use crate::chunk_store::ChunkStore;
28use crate::dynamic_index::{DynamicIndexReader, DynamicIndexWriter};
29use crate::fixed_index::{FixedIndexReader, FixedIndexWriter};
30use crate::hierarchy::{ListGroups, ListGroupsType, ListNamespaces, ListNamespacesRecursive};
31use crate::index::IndexFile;
32use crate::manifest::{archive_type, ArchiveType};
33use crate::task_tracking::{self, update_active_operations};
34use crate::DataBlob;
35
36lazy_static! {
37 static ref DATASTORE_MAP: Mutex<HashMap<String, Arc<DataStoreImpl>>> =
38 Mutex::new(HashMap::new());
39}
40
41/// checks if auth_id is owner, or, if owner is a token, if
42/// auth_id is the user of the token
43pub fn check_backup_owner(owner: &Authid, auth_id: &Authid) -> Result<(), Error> {
44 let correct_owner =
45 owner == auth_id || (owner.is_token() && &Authid::from(owner.user().clone()) == auth_id);
46 if !correct_owner {
47 bail!("backup owner check failed ({} != {})", auth_id, owner);
48 }
49 Ok(())
50}
51
52/// Datastore Management
53///
54/// A Datastore can store severals backups, and provides the
55/// management interface for backup.
56pub struct DataStoreImpl {
57 chunk_store: Arc<ChunkStore>,
58 gc_mutex: Mutex<()>,
59 last_gc_status: Mutex<GarbageCollectionStatus>,
60 verify_new: bool,
61 chunk_order: ChunkOrder,
62 last_digest: Option<[u8; 32]>,
63 sync_level: DatastoreFSyncLevel,
64}
65
66impl DataStoreImpl {
67 // This one just panics on everything
68 #[doc(hidden)]
69 pub(crate) unsafe fn new_test() -> Arc<Self> {
70 Arc::new(Self {
71 chunk_store: Arc::new(unsafe { ChunkStore::panic_store() }),
72 gc_mutex: Mutex::new(()),
73 last_gc_status: Mutex::new(GarbageCollectionStatus::default()),
74 verify_new: false,
75 chunk_order: Default::default(),
76 last_digest: None,
77 sync_level: Default::default(),
78 })
79 }
80}
81
82pub struct DataStore {
83 inner: Arc<DataStoreImpl>,
84 operation: Option<Operation>,
85}
86
87impl Clone for DataStore {
88 fn clone(&self) -> Self {
89 let mut new_operation = self.operation;
90 if let Some(operation) = self.operation {
91 if let Err(e) = update_active_operations(self.name(), operation, 1) {
92 log::error!("could not update active operations - {}", e);
93 new_operation = None;
94 }
95 }
96
97 DataStore {
98 inner: self.inner.clone(),
99 operation: new_operation,
100 }
101 }
102}
103
104impl Drop for DataStore {
105 fn drop(&mut self) {
106 if let Some(operation) = self.operation {
107 if let Err(e) = update_active_operations(self.name(), operation, -1) {
108 log::error!("could not update active operations - {}", e);
109 }
110 }
111 }
112}
113
114impl DataStore {
115 // This one just panics on everything
116 #[doc(hidden)]
117 pub(crate) unsafe fn new_test() -> Arc<Self> {
118 Arc::new(Self {
119 inner: unsafe { DataStoreImpl::new_test() },
120 operation: None,
121 })
122 }
123
124 pub fn lookup_datastore(
125 name: &str,
126 operation: Option<Operation>,
127 ) -> Result<Arc<DataStore>, Error> {
128 // Avoid TOCTOU between checking maintenance mode and updating active operation counter, as
129 // we use it to decide whether it is okay to delete the datastore.
130 let config_lock = pbs_config::datastore::lock_config()?;
131
132 // we could use the ConfigVersionCache's generation for staleness detection, but we load
133 // the config anyway -> just use digest, additional benefit: manual changes get detected
134 let (config, digest) = pbs_config::datastore::config()?;
135 let config: DataStoreConfig = config.lookup("datastore", name)?;
136
137 if let Some(maintenance_mode) = config.get_maintenance_mode() {
138 if let Err(error) = maintenance_mode.check(operation) {
139 bail!("datastore '{name}' is in {error}");
140 }
141 }
142
143 if let Some(operation) = operation {
144 update_active_operations(name, operation, 1)?;
145 }
146
147 // Our operation is registered, unlock the config.
148 drop(config_lock);
149
150 let mut datastore_cache = DATASTORE_MAP.lock().unwrap();
151 let entry = datastore_cache.get(name);
152
153 // reuse chunk store so that we keep using the same process locker instance!
154 let chunk_store = if let Some(datastore) = &entry {
155 let last_digest = datastore.last_digest.as_ref();
156 if let Some(true) = last_digest.map(|last_digest| last_digest == &digest) {
157 return Ok(Arc::new(Self {
158 inner: Arc::clone(datastore),
159 operation,
160 }));
161 }
162 Arc::clone(&datastore.chunk_store)
163 } else {
164 let tuning: DatastoreTuning = serde_json::from_value(
165 DatastoreTuning::API_SCHEMA
166 .parse_property_string(config.tuning.as_deref().unwrap_or(""))?,
167 )?;
168 Arc::new(ChunkStore::open(
169 name,
170 &config.path,
171 tuning.sync_level.unwrap_or_default(),
172 )?)
173 };
174
175 let datastore = DataStore::with_store_and_config(chunk_store, config, Some(digest))?;
176
177 let datastore = Arc::new(datastore);
178 datastore_cache.insert(name.to_string(), datastore.clone());
179
180 Ok(Arc::new(Self {
181 inner: datastore,
182 operation,
183 }))
184 }
185
186 /// removes all datastores that are not configured anymore
187 pub fn remove_unused_datastores() -> Result<(), Error> {
188 let (config, _digest) = pbs_config::datastore::config()?;
189
190 let mut map = DATASTORE_MAP.lock().unwrap();
191 // removes all elements that are not in the config
192 map.retain(|key, _| config.sections.contains_key(key));
193 Ok(())
194 }
195
196 /// Open a raw database given a name and a path.
197 ///
198 /// # Safety
199 /// See the safety section in `open_from_config`
200 pub unsafe fn open_path(
201 name: &str,
202 path: impl AsRef<Path>,
203 operation: Option<Operation>,
204 ) -> Result<Arc<Self>, Error> {
205 let path = path
206 .as_ref()
207 .to_str()
208 .ok_or_else(|| format_err!("non-utf8 paths not supported"))?
209 .to_owned();
210 unsafe { Self::open_from_config(DataStoreConfig::new(name.to_owned(), path), operation) }
211 }
212
213 /// Open a datastore given a raw configuration.
214 ///
215 /// # Safety
216 /// There's no memory safety implication, but as this is opening a new ChunkStore it will
217 /// create a new process locker instance, potentially on the same path as existing safely
218 /// created ones. This is dangerous as dropping the reference of this and thus the underlying
219 /// chunkstore's process locker will close all locks from our process on the config.path,
220 /// breaking guarantees we need to uphold for safe long backup + GC interaction on newer/older
221 /// process instances (from package update).
222 unsafe fn open_from_config(
223 config: DataStoreConfig,
224 operation: Option<Operation>,
225 ) -> Result<Arc<Self>, Error> {
226 let name = config.name.clone();
227
228 let tuning: DatastoreTuning = serde_json::from_value(
229 DatastoreTuning::API_SCHEMA
230 .parse_property_string(config.tuning.as_deref().unwrap_or(""))?,
231 )?;
232 let chunk_store =
233 ChunkStore::open(&name, &config.path, tuning.sync_level.unwrap_or_default())?;
234 let inner = Arc::new(Self::with_store_and_config(
235 Arc::new(chunk_store),
236 config,
237 None,
238 )?);
239
240 if let Some(operation) = operation {
241 update_active_operations(&name, operation, 1)?;
242 }
243
244 Ok(Arc::new(Self { inner, operation }))
245 }
246
247 fn with_store_and_config(
248 chunk_store: Arc<ChunkStore>,
249 config: DataStoreConfig,
250 last_digest: Option<[u8; 32]>,
251 ) -> Result<DataStoreImpl, Error> {
252 let mut gc_status_path = chunk_store.base_path();
253 gc_status_path.push(".gc-status");
254
255 let gc_status = if let Some(state) = file_read_optional_string(gc_status_path)? {
256 match serde_json::from_str(&state) {
257 Ok(state) => state,
258 Err(err) => {
259 log::error!("error reading gc-status: {}", err);
260 GarbageCollectionStatus::default()
261 }
262 }
263 } else {
264 GarbageCollectionStatus::default()
265 };
266
267 let tuning: DatastoreTuning = serde_json::from_value(
268 DatastoreTuning::API_SCHEMA
269 .parse_property_string(config.tuning.as_deref().unwrap_or(""))?,
270 )?;
271
272 Ok(DataStoreImpl {
273 chunk_store,
274 gc_mutex: Mutex::new(()),
275 last_gc_status: Mutex::new(gc_status),
276 verify_new: config.verify_new.unwrap_or(false),
277 chunk_order: tuning.chunk_order.unwrap_or_default(),
278 last_digest,
279 sync_level: tuning.sync_level.unwrap_or_default(),
280 })
281 }
282
283 pub fn get_chunk_iterator(
284 &self,
285 ) -> Result<
286 impl Iterator<Item = (Result<proxmox_sys::fs::ReadDirEntry, Error>, usize, bool)>,
287 Error,
288 > {
289 self.inner.chunk_store.get_chunk_iterator()
290 }
291
292 pub fn create_fixed_writer<P: AsRef<Path>>(
293 &self,
294 filename: P,
295 size: usize,
296 chunk_size: usize,
297 ) -> Result<FixedIndexWriter, Error> {
298 let index = FixedIndexWriter::create(
299 self.inner.chunk_store.clone(),
300 filename.as_ref(),
301 size,
302 chunk_size,
303 )?;
304
305 Ok(index)
306 }
307
308 pub fn open_fixed_reader<P: AsRef<Path>>(
309 &self,
310 filename: P,
311 ) -> Result<FixedIndexReader, Error> {
312 let full_path = self.inner.chunk_store.relative_path(filename.as_ref());
313
314 let index = FixedIndexReader::open(&full_path)?;
315
316 Ok(index)
317 }
318
319 pub fn create_dynamic_writer<P: AsRef<Path>>(
320 &self,
321 filename: P,
322 ) -> Result<DynamicIndexWriter, Error> {
323 let index = DynamicIndexWriter::create(self.inner.chunk_store.clone(), filename.as_ref())?;
324
325 Ok(index)
326 }
327
328 pub fn open_dynamic_reader<P: AsRef<Path>>(
329 &self,
330 filename: P,
331 ) -> Result<DynamicIndexReader, Error> {
332 let full_path = self.inner.chunk_store.relative_path(filename.as_ref());
333
334 let index = DynamicIndexReader::open(&full_path)?;
335
336 Ok(index)
337 }
338
339 pub fn open_index<P>(&self, filename: P) -> Result<Box<dyn IndexFile + Send>, Error>
340 where
341 P: AsRef<Path>,
342 {
343 let filename = filename.as_ref();
344 let out: Box<dyn IndexFile + Send> = match archive_type(filename)? {
345 ArchiveType::DynamicIndex => Box::new(self.open_dynamic_reader(filename)?),
346 ArchiveType::FixedIndex => Box::new(self.open_fixed_reader(filename)?),
347 _ => bail!("cannot open index file of unknown type: {:?}", filename),
348 };
349 Ok(out)
350 }
351
352 /// Fast index verification - only check if chunks exists
353 pub fn fast_index_verification(
354 &self,
355 index: &dyn IndexFile,
356 checked: &mut HashSet<[u8; 32]>,
357 ) -> Result<(), Error> {
358 for pos in 0..index.index_count() {
359 let info = index.chunk_info(pos).unwrap();
360 if checked.contains(&info.digest) {
361 continue;
362 }
363
364 self.stat_chunk(&info.digest).map_err(|err| {
365 format_err!(
366 "fast_index_verification error, stat_chunk {} failed - {}",
367 hex::encode(info.digest),
368 err,
369 )
370 })?;
371
372 checked.insert(info.digest);
373 }
374
375 Ok(())
376 }
377
378 pub fn name(&self) -> &str {
379 self.inner.chunk_store.name()
380 }
381
382 pub fn base_path(&self) -> PathBuf {
383 self.inner.chunk_store.base_path()
384 }
385
386 /// Returns the absolute path for a backup namespace on this datastore
387 pub fn namespace_path(&self, ns: &BackupNamespace) -> PathBuf {
388 let mut path = self.base_path();
389 path.reserve(ns.path_len());
390 for part in ns.components() {
391 path.push("ns");
392 path.push(part);
393 }
394 path
395 }
396
397 /// Returns the absolute path for a backup_type
398 pub fn type_path(&self, ns: &BackupNamespace, backup_type: BackupType) -> PathBuf {
399 let mut full_path = self.namespace_path(ns);
400 full_path.push(backup_type.to_string());
401 full_path
402 }
403
404 /// Returns the absolute path for a backup_group
405 pub fn group_path(
406 &self,
407 ns: &BackupNamespace,
408 backup_group: &pbs_api_types::BackupGroup,
409 ) -> PathBuf {
410 let mut full_path = self.namespace_path(ns);
411 full_path.push(backup_group.to_string());
412 full_path
413 }
414
415 /// Returns the absolute path for backup_dir
416 pub fn snapshot_path(
417 &self,
418 ns: &BackupNamespace,
419 backup_dir: &pbs_api_types::BackupDir,
420 ) -> PathBuf {
421 let mut full_path = self.namespace_path(ns);
422 full_path.push(backup_dir.to_string());
423 full_path
424 }
425
426 /// Create a backup namespace.
427 pub fn create_namespace(
428 self: &Arc<Self>,
429 parent: &BackupNamespace,
430 name: String,
431 ) -> Result<BackupNamespace, Error> {
432 if !self.namespace_exists(parent) {
433 bail!("cannot create new namespace, parent {parent} doesn't already exists");
434 }
435
436 // construct ns before mkdir to enforce max-depth and name validity
437 let ns = BackupNamespace::from_parent_ns(parent, name)?;
438
439 let mut ns_full_path = self.base_path();
440 ns_full_path.push(ns.path());
441
442 std::fs::create_dir_all(ns_full_path)?;
443
444 Ok(ns)
445 }
446
447 /// Returns if the given namespace exists on the datastore
448 pub fn namespace_exists(&self, ns: &BackupNamespace) -> bool {
449 let mut path = self.base_path();
450 path.push(ns.path());
451 path.exists()
452 }
453
454 /// Remove all backup groups of a single namespace level but not the namespace itself.
455 ///
456 /// Does *not* descends into child-namespaces and doesn't remoes the namespace itself either.
457 ///
458 /// Returns true if all the groups were removed, and false if some were protected.
459 pub fn remove_namespace_groups(self: &Arc<Self>, ns: &BackupNamespace) -> Result<bool, Error> {
460 // FIXME: locking? The single groups/snapshots are already protected, so may not be
461 // necessary (depends on what we all allow to do with namespaces)
462 log::info!("removing all groups in namespace {}:/{ns}", self.name());
463
464 let mut removed_all_groups = true;
465
466 for group in self.iter_backup_groups(ns.to_owned())? {
467 let removed_group = group?.destroy()?;
468 removed_all_groups = removed_all_groups && removed_group;
469 }
470
471 let base_file = std::fs::File::open(self.base_path())?;
472 let base_fd = base_file.as_raw_fd();
473 for ty in BackupType::iter() {
474 let mut ty_dir = ns.path();
475 ty_dir.push(ty.to_string());
476 // best effort only, but we probably should log the error
477 if let Err(err) = unlinkat(Some(base_fd), &ty_dir, UnlinkatFlags::RemoveDir) {
478 if err != nix::errno::Errno::ENOENT {
479 log::error!("failed to remove backup type {ty} in {ns} - {err}");
480 }
481 }
482 }
483
484 Ok(removed_all_groups)
485 }
486
487 /// Remove a complete backup namespace optionally including all it's, and child namespaces',
488 /// groups. If `removed_groups` is false this only prunes empty namespaces.
489 ///
490 /// Returns true if everything requested, and false if some groups were protected or if some
491 /// namespaces weren't empty even though all groups were deleted (race with new backup)
492 pub fn remove_namespace_recursive(
493 self: &Arc<Self>,
494 ns: &BackupNamespace,
495 delete_groups: bool,
496 ) -> Result<bool, Error> {
497 let store = self.name();
498 let mut removed_all_requested = true;
499 if delete_groups {
500 log::info!("removing whole namespace recursively below {store}:/{ns}",);
501 for ns in self.recursive_iter_backup_ns(ns.to_owned())? {
502 let removed_ns_groups = self.remove_namespace_groups(&ns?)?;
503 removed_all_requested = removed_all_requested && removed_ns_groups;
504 }
505 } else {
506 log::info!("pruning empty namespace recursively below {store}:/{ns}");
507 }
508
509 // now try to delete the actual namespaces, bottom up so that we can use safe rmdir that
510 // will choke if a new backup/group appeared in the meantime (but not on an new empty NS)
511 let mut children = self
512 .recursive_iter_backup_ns(ns.to_owned())?
513 .collect::<Result<Vec<BackupNamespace>, Error>>()?;
514
515 children.sort_by_key(|b| std::cmp::Reverse(b.depth()));
516
517 let base_file = std::fs::File::open(self.base_path())?;
518 let base_fd = base_file.as_raw_fd();
519
520 for ns in children.iter() {
521 let mut ns_dir = ns.path();
522 ns_dir.push("ns");
523 let _ = unlinkat(Some(base_fd), &ns_dir, UnlinkatFlags::RemoveDir);
524
525 if !ns.is_root() {
526 match unlinkat(Some(base_fd), &ns.path(), UnlinkatFlags::RemoveDir) {
527 Ok(()) => log::debug!("removed namespace {ns}"),
528 Err(nix::errno::Errno::ENOENT) => {
529 log::debug!("namespace {ns} already removed")
530 }
531 Err(nix::errno::Errno::ENOTEMPTY) if !delete_groups => {
532 removed_all_requested = false;
533 log::debug!("skip removal of non-empty namespace {ns}")
534 }
535 Err(err) => {
536 removed_all_requested = false;
537 log::warn!("failed to remove namespace {ns} - {err}")
538 }
539 }
540 }
541 }
542
543 Ok(removed_all_requested)
544 }
545
546 /// Remove a complete backup group including all snapshots.
547 ///
548 /// Returns true if all snapshots were removed, and false if some were protected
549 pub fn remove_backup_group(
550 self: &Arc<Self>,
551 ns: &BackupNamespace,
552 backup_group: &pbs_api_types::BackupGroup,
553 ) -> Result<bool, Error> {
554 let backup_group = self.backup_group(ns.clone(), backup_group.clone());
555
556 backup_group.destroy()
557 }
558
559 /// Remove a backup directory including all content
560 pub fn remove_backup_dir(
561 self: &Arc<Self>,
562 ns: &BackupNamespace,
563 backup_dir: &pbs_api_types::BackupDir,
564 force: bool,
565 ) -> Result<(), Error> {
566 let backup_dir = self.backup_dir(ns.clone(), backup_dir.clone())?;
567
568 backup_dir.destroy(force)
569 }
570
571 /// Returns the time of the last successful backup
572 ///
573 /// Or None if there is no backup in the group (or the group dir does not exist).
574 pub fn last_successful_backup(
575 self: &Arc<Self>,
576 ns: &BackupNamespace,
577 backup_group: &pbs_api_types::BackupGroup,
578 ) -> Result<Option<i64>, Error> {
579 let backup_group = self.backup_group(ns.clone(), backup_group.clone());
580
581 let group_path = backup_group.full_group_path();
582
583 if group_path.exists() {
584 backup_group.last_successful_backup()
585 } else {
586 Ok(None)
587 }
588 }
589
590 /// Return the path of the 'owner' file.
591 fn owner_path(&self, ns: &BackupNamespace, group: &pbs_api_types::BackupGroup) -> PathBuf {
592 self.group_path(ns, group).join("owner")
593 }
594
595 /// Returns the backup owner.
596 ///
597 /// The backup owner is the entity who first created the backup group.
598 pub fn get_owner(
599 &self,
600 ns: &BackupNamespace,
601 backup_group: &pbs_api_types::BackupGroup,
602 ) -> Result<Authid, Error> {
603 let full_path = self.owner_path(ns, backup_group);
604 let owner = proxmox_sys::fs::file_read_firstline(full_path)?;
605 owner
606 .trim_end() // remove trailing newline
607 .parse()
608 .map_err(|err| {
609 format_err!("parsing owner for {backup_group} failed: {err}")
610 })
611 }
612
613 pub fn owns_backup(
614 &self,
615 ns: &BackupNamespace,
616 backup_group: &pbs_api_types::BackupGroup,
617 auth_id: &Authid,
618 ) -> Result<bool, Error> {
619 let owner = self.get_owner(ns, backup_group)?;
620
621 Ok(check_backup_owner(&owner, auth_id).is_ok())
622 }
623
624 /// Set the backup owner.
625 pub fn set_owner(
626 &self,
627 ns: &BackupNamespace,
628 backup_group: &pbs_api_types::BackupGroup,
629 auth_id: &Authid,
630 force: bool,
631 ) -> Result<(), Error> {
632 let path = self.owner_path(ns, backup_group);
633
634 let mut open_options = std::fs::OpenOptions::new();
635 open_options.write(true);
636 open_options.truncate(true);
637
638 if force {
639 open_options.create(true);
640 } else {
641 open_options.create_new(true);
642 }
643
644 let mut file = open_options
645 .open(&path)
646 .map_err(|err| format_err!("unable to create owner file {:?} - {}", path, err))?;
647
648 writeln!(file, "{}", auth_id)
649 .map_err(|err| format_err!("unable to write owner file {:?} - {}", path, err))?;
650
651 Ok(())
652 }
653
654 /// Create (if it does not already exists) and lock a backup group
655 ///
656 /// And set the owner to 'userid'. If the group already exists, it returns the
657 /// current owner (instead of setting the owner).
658 ///
659 /// This also acquires an exclusive lock on the directory and returns the lock guard.
660 pub fn create_locked_backup_group(
661 &self,
662 ns: &BackupNamespace,
663 backup_group: &pbs_api_types::BackupGroup,
664 auth_id: &Authid,
665 ) -> Result<(Authid, DirLockGuard), Error> {
666 // create intermediate path first:
667 let mut full_path = self.base_path();
668 for ns in ns.components() {
669 full_path.push("ns");
670 full_path.push(ns);
671 }
672 full_path.push(backup_group.ty.as_str());
673 std::fs::create_dir_all(&full_path)?;
674
675 full_path.push(&backup_group.id);
676
677 // create the last component now
678 match std::fs::create_dir(&full_path) {
679 Ok(_) => {
680 let guard = lock_dir_noblock(
681 &full_path,
682 "backup group",
683 "another backup is already running",
684 )?;
685 self.set_owner(ns, backup_group, auth_id, false)?;
686 let owner = self.get_owner(ns, backup_group)?; // just to be sure
687 Ok((owner, guard))
688 }
689 Err(ref err) if err.kind() == io::ErrorKind::AlreadyExists => {
690 let guard = lock_dir_noblock(
691 &full_path,
692 "backup group",
693 "another backup is already running",
694 )?;
695 let owner = self.get_owner(ns, backup_group)?; // just to be sure
696 Ok((owner, guard))
697 }
698 Err(err) => bail!("unable to create backup group {:?} - {}", full_path, err),
699 }
700 }
701
702 /// Creates a new backup snapshot inside a BackupGroup
703 ///
704 /// The BackupGroup directory needs to exist.
705 pub fn create_locked_backup_dir(
706 &self,
707 ns: &BackupNamespace,
708 backup_dir: &pbs_api_types::BackupDir,
709 ) -> Result<(PathBuf, bool, DirLockGuard), Error> {
710 let full_path = self.snapshot_path(ns, backup_dir);
711 let relative_path = full_path.strip_prefix(self.base_path()).map_err(|err| {
712 format_err!(
713 "failed to produce correct path for backup {backup_dir} in namespace {ns}: {err}"
714 )
715 })?;
716
717 let lock = || {
718 lock_dir_noblock(
719 &full_path,
720 "snapshot",
721 "internal error - tried creating snapshot that's already in use",
722 )
723 };
724
725 match std::fs::create_dir(&full_path) {
726 Ok(_) => Ok((relative_path.to_owned(), true, lock()?)),
727 Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => {
728 Ok((relative_path.to_owned(), false, lock()?))
729 }
730 Err(e) => Err(e.into()),
731 }
732 }
733
734 /// Get a streaming iter over single-level backup namespaces of a datatstore
735 ///
736 /// The iterated item is still a Result that can contain errors from rather unexptected FS or
737 /// parsing errors.
738 pub fn iter_backup_ns(
739 self: &Arc<DataStore>,
740 ns: BackupNamespace,
741 ) -> Result<ListNamespaces, Error> {
742 ListNamespaces::new(Arc::clone(self), ns)
743 }
744
745 /// Get a streaming iter over single-level backup namespaces of a datatstore, filtered by Ok
746 ///
747 /// The iterated item's result is already unwrapped, if it contained an error it will be
748 /// logged. Can be useful in iterator chain commands
749 pub fn iter_backup_ns_ok(
750 self: &Arc<DataStore>,
751 ns: BackupNamespace,
752 ) -> Result<impl Iterator<Item = BackupNamespace> + 'static, Error> {
753 let this = Arc::clone(self);
754 Ok(
755 ListNamespaces::new(Arc::clone(self), ns)?.filter_map(move |ns| match ns {
756 Ok(ns) => Some(ns),
757 Err(err) => {
758 log::error!("list groups error on datastore {} - {}", this.name(), err);
759 None
760 }
761 }),
762 )
763 }
764
765 /// Get a streaming iter over single-level backup namespaces of a datatstore
766 ///
767 /// The iterated item is still a Result that can contain errors from rather unexptected FS or
768 /// parsing errors.
769 pub fn recursive_iter_backup_ns(
770 self: &Arc<DataStore>,
771 ns: BackupNamespace,
772 ) -> Result<ListNamespacesRecursive, Error> {
773 ListNamespacesRecursive::new(Arc::clone(self), ns)
774 }
775
776 /// Get a streaming iter over single-level backup namespaces of a datatstore, filtered by Ok
777 ///
778 /// The iterated item's result is already unwrapped, if it contained an error it will be
779 /// logged. Can be useful in iterator chain commands
780 pub fn recursive_iter_backup_ns_ok(
781 self: &Arc<DataStore>,
782 ns: BackupNamespace,
783 max_depth: Option<usize>,
784 ) -> Result<impl Iterator<Item = BackupNamespace> + 'static, Error> {
785 let this = Arc::clone(self);
786 Ok(if let Some(depth) = max_depth {
787 ListNamespacesRecursive::new_max_depth(Arc::clone(self), ns, depth)?
788 } else {
789 ListNamespacesRecursive::new(Arc::clone(self), ns)?
790 }
791 .filter_map(move |ns| match ns {
792 Ok(ns) => Some(ns),
793 Err(err) => {
794 log::error!("list groups error on datastore {} - {}", this.name(), err);
795 None
796 }
797 }))
798 }
799
800 /// Get a streaming iter over top-level backup groups of a datatstore of a particular type.
801 ///
802 /// The iterated item is still a Result that can contain errors from rather unexptected FS or
803 /// parsing errors.
804 pub fn iter_backup_type(
805 self: &Arc<DataStore>,
806 ns: BackupNamespace,
807 ty: BackupType,
808 ) -> Result<ListGroupsType, Error> {
809 ListGroupsType::new(Arc::clone(self), ns, ty)
810 }
811
812 /// Get a streaming iter over top-level backup groups of a datatstore of a particular type,
813 /// filtered by `Ok` results
814 ///
815 /// The iterated item's result is already unwrapped, if it contained an error it will be
816 /// logged. Can be useful in iterator chain commands
817 pub fn iter_backup_type_ok(
818 self: &Arc<DataStore>,
819 ns: BackupNamespace,
820 ty: BackupType,
821 ) -> Result<impl Iterator<Item = BackupGroup> + 'static, Error> {
822 Ok(self.iter_backup_type(ns, ty)?.ok())
823 }
824
825 /// Get a streaming iter over top-level backup groups of a datatstore
826 ///
827 /// The iterated item is still a Result that can contain errors from rather unexptected FS or
828 /// parsing errors.
829 pub fn iter_backup_groups(
830 self: &Arc<DataStore>,
831 ns: BackupNamespace,
832 ) -> Result<ListGroups, Error> {
833 ListGroups::new(Arc::clone(self), ns)
834 }
835
836 /// Get a streaming iter over top-level backup groups of a datatstore, filtered by Ok results
837 ///
838 /// The iterated item's result is already unwrapped, if it contained an error it will be
839 /// logged. Can be useful in iterator chain commands
840 pub fn iter_backup_groups_ok(
841 self: &Arc<DataStore>,
842 ns: BackupNamespace,
843 ) -> Result<impl Iterator<Item = BackupGroup> + 'static, Error> {
844 Ok(self.iter_backup_groups(ns)?.ok())
845 }
846
847 /// Get a in-memory vector for all top-level backup groups of a datatstore
848 ///
849 /// NOTE: using the iterator directly is most often more efficient w.r.t. memory usage
850 pub fn list_backup_groups(
851 self: &Arc<DataStore>,
852 ns: BackupNamespace,
853 ) -> Result<Vec<BackupGroup>, Error> {
854 ListGroups::new(Arc::clone(self), ns)?.collect()
855 }
856
857 pub fn list_images(&self) -> Result<Vec<PathBuf>, Error> {
858 let base = self.base_path();
859
860 let mut list = vec![];
861
862 use walkdir::WalkDir;
863
864 let walker = WalkDir::new(base).into_iter();
865
866 // make sure we skip .chunks (and other hidden files to keep it simple)
867 fn is_hidden(entry: &walkdir::DirEntry) -> bool {
868 entry
869 .file_name()
870 .to_str()
871 .map(|s| s.starts_with('.'))
872 .unwrap_or(false)
873 }
874 let handle_entry_err = |err: walkdir::Error| {
875 // first, extract the actual IO error and the affected path
876 let (inner, path) = match (err.io_error(), err.path()) {
877 (None, _) => return Ok(()), // not an IO-error
878 (Some(inner), Some(path)) => (inner, path),
879 (Some(inner), None) => bail!("unexpected error on datastore traversal: {inner}"),
880 };
881 if inner.kind() == io::ErrorKind::PermissionDenied {
882 if err.depth() <= 1 && path.ends_with("lost+found") {
883 // allow skipping of (root-only) ext4 fsck-directory on EPERM ..
884 return Ok(());
885 }
886 // .. but do not ignore EPERM in general, otherwise we might prune too many chunks.
887 // E.g., if users messed up with owner/perms on a rsync
888 bail!("cannot continue garbage-collection safely, permission denied on: {path:?}");
889 } else if inner.kind() == io::ErrorKind::NotFound {
890 log::info!("ignoring vanished file: {path:?}");
891 return Ok(());
892 } else {
893 bail!("unexpected error on datastore traversal: {inner} - {path:?}");
894 }
895 };
896 for entry in walker.filter_entry(|e| !is_hidden(e)) {
897 let path = match entry {
898 Ok(entry) => entry.into_path(),
899 Err(err) => {
900 handle_entry_err(err)?;
901 continue;
902 }
903 };
904 if let Ok(archive_type) = archive_type(&path) {
905 if archive_type == ArchiveType::FixedIndex
906 || archive_type == ArchiveType::DynamicIndex
907 {
908 list.push(path);
909 }
910 }
911 }
912
913 Ok(list)
914 }
915
916 // mark chunks used by ``index`` as used
917 fn index_mark_used_chunks<I: IndexFile>(
918 &self,
919 index: I,
920 file_name: &Path, // only used for error reporting
921 status: &mut GarbageCollectionStatus,
922 worker: &dyn WorkerTaskContext,
923 ) -> Result<(), Error> {
924 status.index_file_count += 1;
925 status.index_data_bytes += index.index_bytes();
926
927 for pos in 0..index.index_count() {
928 worker.check_abort()?;
929 worker.fail_on_shutdown()?;
930 let digest = index.index_digest(pos).unwrap();
931 if !self.inner.chunk_store.cond_touch_chunk(digest, false)? {
932 let hex = hex::encode(digest);
933 task_warn!(
934 worker,
935 "warning: unable to access non-existent chunk {hex}, required by {file_name:?}"
936 );
937
938 // touch any corresponding .bad files to keep them around, meaning if a chunk is
939 // rewritten correctly they will be removed automatically, as well as if no index
940 // file requires the chunk anymore (won't get to this loop then)
941 for i in 0..=9 {
942 let bad_ext = format!("{}.bad", i);
943 let mut bad_path = PathBuf::new();
944 bad_path.push(self.chunk_path(digest).0);
945 bad_path.set_extension(bad_ext);
946 self.inner.chunk_store.cond_touch_path(&bad_path, false)?;
947 }
948 }
949 }
950 Ok(())
951 }
952
953 fn mark_used_chunks(
954 &self,
955 status: &mut GarbageCollectionStatus,
956 worker: &dyn WorkerTaskContext,
957 ) -> Result<(), Error> {
958 let image_list = self.list_images()?;
959 let image_count = image_list.len();
960
961 let mut last_percentage: usize = 0;
962
963 let mut strange_paths_count: u64 = 0;
964
965 for (i, img) in image_list.into_iter().enumerate() {
966 worker.check_abort()?;
967 worker.fail_on_shutdown()?;
968
969 if let Some(backup_dir_path) = img.parent() {
970 let backup_dir_path = backup_dir_path.strip_prefix(self.base_path())?;
971 if let Some(backup_dir_str) = backup_dir_path.to_str() {
972 if pbs_api_types::parse_ns_and_snapshot(backup_dir_str).is_err() {
973 strange_paths_count += 1;
974 }
975 }
976 }
977
978 match std::fs::File::open(&img) {
979 Ok(file) => {
980 if let Ok(archive_type) = archive_type(&img) {
981 if archive_type == ArchiveType::FixedIndex {
982 let index = FixedIndexReader::new(file).map_err(|e| {
983 format_err!("can't read index '{}' - {}", img.to_string_lossy(), e)
984 })?;
985 self.index_mark_used_chunks(index, &img, status, worker)?;
986 } else if archive_type == ArchiveType::DynamicIndex {
987 let index = DynamicIndexReader::new(file).map_err(|e| {
988 format_err!("can't read index '{}' - {}", img.to_string_lossy(), e)
989 })?;
990 self.index_mark_used_chunks(index, &img, status, worker)?;
991 }
992 }
993 }
994 Err(err) if err.kind() == io::ErrorKind::NotFound => (), // ignore vanished files
995 Err(err) => bail!("can't open index {} - {}", img.to_string_lossy(), err),
996 }
997
998 let percentage = (i + 1) * 100 / image_count;
999 if percentage > last_percentage {
1000 task_log!(
1001 worker,
1002 "marked {}% ({} of {} index files)",
1003 percentage,
1004 i + 1,
1005 image_count,
1006 );
1007 last_percentage = percentage;
1008 }
1009 }
1010
1011 if strange_paths_count > 0 {
1012 task_log!(
1013 worker,
1014 "found (and marked) {} index files outside of expected directory scheme",
1015 strange_paths_count,
1016 );
1017 }
1018
1019 Ok(())
1020 }
1021
1022 pub fn last_gc_status(&self) -> GarbageCollectionStatus {
1023 self.inner.last_gc_status.lock().unwrap().clone()
1024 }
1025
1026 pub fn garbage_collection_running(&self) -> bool {
1027 !matches!(self.inner.gc_mutex.try_lock(), Ok(_))
1028 }
1029
1030 pub fn garbage_collection(
1031 &self,
1032 worker: &dyn WorkerTaskContext,
1033 upid: &UPID,
1034 ) -> Result<(), Error> {
1035 if let Ok(ref mut _mutex) = self.inner.gc_mutex.try_lock() {
1036 // avoids that we run GC if an old daemon process has still a
1037 // running backup writer, which is not save as we have no "oldest
1038 // writer" information and thus no safe atime cutoff
1039 let _exclusive_lock = self.inner.chunk_store.try_exclusive_lock()?;
1040
1041 let phase1_start_time = proxmox_time::epoch_i64();
1042 let oldest_writer = self
1043 .inner
1044 .chunk_store
1045 .oldest_writer()
1046 .unwrap_or(phase1_start_time);
1047
1048 let mut gc_status = GarbageCollectionStatus {
1049 upid: Some(upid.to_string()),
1050 ..Default::default()
1051 };
1052
1053 task_log!(worker, "Start GC phase1 (mark used chunks)");
1054
1055 self.mark_used_chunks(&mut gc_status, worker)?;
1056
1057 task_log!(worker, "Start GC phase2 (sweep unused chunks)");
1058 self.inner.chunk_store.sweep_unused_chunks(
1059 oldest_writer,
1060 phase1_start_time,
1061 &mut gc_status,
1062 worker,
1063 )?;
1064
1065 task_log!(
1066 worker,
1067 "Removed garbage: {}",
1068 HumanByte::from(gc_status.removed_bytes),
1069 );
1070 task_log!(worker, "Removed chunks: {}", gc_status.removed_chunks);
1071 if gc_status.pending_bytes > 0 {
1072 task_log!(
1073 worker,
1074 "Pending removals: {} (in {} chunks)",
1075 HumanByte::from(gc_status.pending_bytes),
1076 gc_status.pending_chunks,
1077 );
1078 }
1079 if gc_status.removed_bad > 0 {
1080 task_log!(worker, "Removed bad chunks: {}", gc_status.removed_bad);
1081 }
1082
1083 if gc_status.still_bad > 0 {
1084 task_log!(worker, "Leftover bad chunks: {}", gc_status.still_bad);
1085 }
1086
1087 task_log!(
1088 worker,
1089 "Original data usage: {}",
1090 HumanByte::from(gc_status.index_data_bytes),
1091 );
1092
1093 if gc_status.index_data_bytes > 0 {
1094 let comp_per =
1095 (gc_status.disk_bytes as f64 * 100.) / gc_status.index_data_bytes as f64;
1096 task_log!(
1097 worker,
1098 "On-Disk usage: {} ({:.2}%)",
1099 HumanByte::from(gc_status.disk_bytes),
1100 comp_per,
1101 );
1102 }
1103
1104 task_log!(worker, "On-Disk chunks: {}", gc_status.disk_chunks);
1105
1106 let deduplication_factor = if gc_status.disk_bytes > 0 {
1107 (gc_status.index_data_bytes as f64) / (gc_status.disk_bytes as f64)
1108 } else {
1109 1.0
1110 };
1111
1112 task_log!(worker, "Deduplication factor: {:.2}", deduplication_factor);
1113
1114 if gc_status.disk_chunks > 0 {
1115 let avg_chunk = gc_status.disk_bytes / (gc_status.disk_chunks as u64);
1116 task_log!(worker, "Average chunk size: {}", HumanByte::from(avg_chunk));
1117 }
1118
1119 if let Ok(serialized) = serde_json::to_string(&gc_status) {
1120 let mut path = self.base_path();
1121 path.push(".gc-status");
1122
1123 let backup_user = pbs_config::backup_user()?;
1124 let mode = nix::sys::stat::Mode::from_bits_truncate(0o0644);
1125 // set the correct owner/group/permissions while saving file
1126 // owner(rw) = backup, group(r)= backup
1127 let options = CreateOptions::new()
1128 .perm(mode)
1129 .owner(backup_user.uid)
1130 .group(backup_user.gid);
1131
1132 // ignore errors
1133 let _ = replace_file(path, serialized.as_bytes(), options, false);
1134 }
1135
1136 *self.inner.last_gc_status.lock().unwrap() = gc_status;
1137 } else {
1138 bail!("Start GC failed - (already running/locked)");
1139 }
1140
1141 Ok(())
1142 }
1143
1144 pub fn try_shared_chunk_store_lock(&self) -> Result<ProcessLockSharedGuard, Error> {
1145 self.inner.chunk_store.try_shared_lock()
1146 }
1147
1148 pub fn chunk_path(&self, digest: &[u8; 32]) -> (PathBuf, String) {
1149 self.inner.chunk_store.chunk_path(digest)
1150 }
1151
1152 pub fn cond_touch_chunk(&self, digest: &[u8; 32], assert_exists: bool) -> Result<bool, Error> {
1153 self.inner
1154 .chunk_store
1155 .cond_touch_chunk(digest, assert_exists)
1156 }
1157
1158 pub fn insert_chunk(&self, chunk: &DataBlob, digest: &[u8; 32]) -> Result<(bool, u64), Error> {
1159 self.inner.chunk_store.insert_chunk(chunk, digest)
1160 }
1161
1162 pub fn stat_chunk(&self, digest: &[u8; 32]) -> Result<std::fs::Metadata, Error> {
1163 let (chunk_path, _digest_str) = self.inner.chunk_store.chunk_path(digest);
1164 std::fs::metadata(chunk_path).map_err(Error::from)
1165 }
1166
1167 pub fn load_chunk(&self, digest: &[u8; 32]) -> Result<DataBlob, Error> {
1168 let (chunk_path, digest_str) = self.inner.chunk_store.chunk_path(digest);
1169
1170 proxmox_lang::try_block!({
1171 let mut file = std::fs::File::open(&chunk_path)?;
1172 DataBlob::load_from_reader(&mut file)
1173 })
1174 .map_err(|err| {
1175 format_err!(
1176 "store '{}', unable to load chunk '{}' - {}",
1177 self.name(),
1178 digest_str,
1179 err,
1180 )
1181 })
1182 }
1183
1184 /// Updates the protection status of the specified snapshot.
1185 pub fn update_protection(&self, backup_dir: &BackupDir, protection: bool) -> Result<(), Error> {
1186 let full_path = backup_dir.full_path();
1187
1188 if !full_path.exists() {
1189 bail!("snapshot {} does not exist!", backup_dir.dir());
1190 }
1191
1192 let _guard = lock_dir_noblock(&full_path, "snapshot", "possibly running or in use")?;
1193
1194 let protected_path = backup_dir.protected_file();
1195 if protection {
1196 std::fs::File::create(protected_path)
1197 .map_err(|err| format_err!("could not create protection file: {}", err))?;
1198 } else if let Err(err) = std::fs::remove_file(protected_path) {
1199 // ignore error for non-existing file
1200 if err.kind() != std::io::ErrorKind::NotFound {
1201 bail!("could not remove protection file: {}", err);
1202 }
1203 }
1204
1205 Ok(())
1206 }
1207
1208 pub fn verify_new(&self) -> bool {
1209 self.inner.verify_new
1210 }
1211
1212 /// returns a list of chunks sorted by their inode number on disk chunks that couldn't get
1213 /// stat'ed are placed at the end of the list
1214 pub fn get_chunks_in_order<F, A>(
1215 &self,
1216 index: &(dyn IndexFile + Send),
1217 skip_chunk: F,
1218 check_abort: A,
1219 ) -> Result<Vec<(usize, u64)>, Error>
1220 where
1221 F: Fn(&[u8; 32]) -> bool,
1222 A: Fn(usize) -> Result<(), Error>,
1223 {
1224 let index_count = index.index_count();
1225 let mut chunk_list = Vec::with_capacity(index_count);
1226 use std::os::unix::fs::MetadataExt;
1227 for pos in 0..index_count {
1228 check_abort(pos)?;
1229
1230 let info = index.chunk_info(pos).unwrap();
1231
1232 if skip_chunk(&info.digest) {
1233 continue;
1234 }
1235
1236 let ino = match self.inner.chunk_order {
1237 ChunkOrder::Inode => {
1238 match self.stat_chunk(&info.digest) {
1239 Err(_) => u64::MAX, // could not stat, move to end of list
1240 Ok(metadata) => metadata.ino(),
1241 }
1242 }
1243 ChunkOrder::None => 0,
1244 };
1245
1246 chunk_list.push((pos, ino));
1247 }
1248
1249 match self.inner.chunk_order {
1250 // sorting by inode improves data locality, which makes it lots faster on spinners
1251 ChunkOrder::Inode => {
1252 chunk_list.sort_unstable_by(|(_, ino_a), (_, ino_b)| ino_a.cmp(ino_b))
1253 }
1254 ChunkOrder::None => {}
1255 }
1256
1257 Ok(chunk_list)
1258 }
1259
1260 /// Open a backup group from this datastore.
1261 pub fn backup_group(
1262 self: &Arc<Self>,
1263 ns: BackupNamespace,
1264 group: pbs_api_types::BackupGroup,
1265 ) -> BackupGroup {
1266 BackupGroup::new(Arc::clone(self), ns, group)
1267 }
1268
1269 /// Open a backup group from this datastore.
1270 pub fn backup_group_from_parts<T>(
1271 self: &Arc<Self>,
1272 ns: BackupNamespace,
1273 ty: BackupType,
1274 id: T,
1275 ) -> BackupGroup
1276 where
1277 T: Into<String>,
1278 {
1279 self.backup_group(ns, (ty, id.into()).into())
1280 }
1281
1282 /*
1283 /// Open a backup group from this datastore by backup group path such as `vm/100`.
1284 ///
1285 /// Convenience method for `store.backup_group(path.parse()?)`
1286 pub fn backup_group_from_path(self: &Arc<Self>, path: &str) -> Result<BackupGroup, Error> {
1287 todo!("split out the namespace");
1288 }
1289 */
1290
1291 /// Open a snapshot (backup directory) from this datastore.
1292 pub fn backup_dir(
1293 self: &Arc<Self>,
1294 ns: BackupNamespace,
1295 dir: pbs_api_types::BackupDir,
1296 ) -> Result<BackupDir, Error> {
1297 BackupDir::with_group(self.backup_group(ns, dir.group), dir.time)
1298 }
1299
1300 /// Open a snapshot (backup directory) from this datastore.
1301 pub fn backup_dir_from_parts<T>(
1302 self: &Arc<Self>,
1303 ns: BackupNamespace,
1304 ty: BackupType,
1305 id: T,
1306 time: i64,
1307 ) -> Result<BackupDir, Error>
1308 where
1309 T: Into<String>,
1310 {
1311 self.backup_dir(ns, (ty, id.into(), time).into())
1312 }
1313
1314 /// Open a snapshot (backup directory) from this datastore with a cached rfc3339 time string.
1315 pub fn backup_dir_with_rfc3339<T: Into<String>>(
1316 self: &Arc<Self>,
1317 group: BackupGroup,
1318 time_string: T,
1319 ) -> Result<BackupDir, Error> {
1320 BackupDir::with_rfc3339(group, time_string.into())
1321 }
1322
1323 /*
1324 /// Open a snapshot (backup directory) from this datastore by a snapshot path.
1325 pub fn backup_dir_from_path(self: &Arc<Self>, path: &str) -> Result<BackupDir, Error> {
1326 todo!("split out the namespace");
1327 }
1328 */
1329
1330 /// Syncs the filesystem of the datastore if 'sync_level' is set to
1331 /// [`DatastoreFSyncLevel::Filesystem`]. Uses syncfs(2).
1332 pub fn try_ensure_sync_level(&self) -> Result<(), Error> {
1333 if self.inner.sync_level != DatastoreFSyncLevel::Filesystem {
1334 return Ok(());
1335 }
1336 let file = std::fs::File::open(self.base_path())?;
1337 let fd = file.as_raw_fd();
1338 log::info!("syncing filesystem");
1339 if unsafe { libc::syncfs(fd) } < 0 {
1340 bail!("error during syncfs: {}", std::io::Error::last_os_error());
1341 }
1342 Ok(())
1343 }
1344
1345 /// Destroy a datastore. This requires that there are no active operations on the datastore.
1346 ///
1347 /// This is a synchronous operation and should be run in a worker-thread.
1348 pub fn destroy(
1349 name: &str,
1350 destroy_data: bool,
1351 worker: &dyn WorkerTaskContext,
1352 ) -> Result<(), Error> {
1353 let config_lock = pbs_config::datastore::lock_config()?;
1354
1355 let (mut config, _digest) = pbs_config::datastore::config()?;
1356 let mut datastore_config: DataStoreConfig = config.lookup("datastore", name)?;
1357
1358 datastore_config.maintenance_mode = Some("type=delete".to_string());
1359 config.set_data(name, "datastore", &datastore_config)?;
1360 pbs_config::datastore::save_config(&config)?;
1361 drop(config_lock);
1362
1363 let (operations, _lock) = task_tracking::get_active_operations_locked(name)?;
1364
1365 if operations.read != 0 || operations.write != 0 {
1366 bail!("datastore is currently in use");
1367 }
1368
1369 let base = PathBuf::from(&datastore_config.path);
1370
1371 let mut ok = true;
1372 if destroy_data {
1373 let remove = |subdir, ok: &mut bool| {
1374 if let Err(err) = std::fs::remove_dir_all(base.join(subdir)) {
1375 if err.kind() != io::ErrorKind::NotFound {
1376 task_warn!(worker, "failed to remove {subdir:?} subdirectory: {err}");
1377 *ok = false;
1378 }
1379 }
1380 };
1381
1382 task_log!(worker, "Deleting datastore data...");
1383 remove("ns", &mut ok); // ns first
1384 remove("ct", &mut ok);
1385 remove("vm", &mut ok);
1386 remove("host", &mut ok);
1387
1388 if ok {
1389 if let Err(err) = std::fs::remove_file(base.join(".gc-status")) {
1390 if err.kind() != io::ErrorKind::NotFound {
1391 task_warn!(worker, "failed to remove .gc-status file: {err}");
1392 ok = false;
1393 }
1394 }
1395 }
1396
1397 // chunks get removed last and only if the backups were successfully deleted
1398 if ok {
1399 remove(".chunks", &mut ok);
1400 }
1401 }
1402
1403 // now the config
1404 if ok {
1405 task_log!(worker, "Removing datastore from config...");
1406 let _lock = pbs_config::datastore::lock_config()?;
1407 let _ = config.sections.remove(name);
1408 pbs_config::datastore::save_config(&config)?;
1409 }
1410
1411 // finally the lock & toplevel directory
1412 if destroy_data {
1413 if ok {
1414 if let Err(err) = std::fs::remove_file(base.join(".lock")) {
1415 if err.kind() != io::ErrorKind::NotFound {
1416 task_warn!(worker, "failed to remove .lock file: {err}");
1417 ok = false;
1418 }
1419 }
1420 }
1421
1422 if ok {
1423 task_log!(worker, "Finished deleting data.");
1424
1425 match std::fs::remove_dir(base) {
1426 Ok(()) => task_log!(worker, "Removed empty datastore directory."),
1427 Err(err) if err.kind() == io::ErrorKind::NotFound => {
1428 // weird, but ok
1429 }
1430 Err(err) if err.is_errno(nix::errno::Errno::EBUSY) => {
1431 task_warn!(
1432 worker,
1433 "Cannot delete datastore directory (is it a mount point?)."
1434 )
1435 }
1436 Err(err) if err.is_errno(nix::errno::Errno::ENOTEMPTY) => {
1437 task_warn!(worker, "Datastore directory not empty, not deleting.")
1438 }
1439 Err(err) => {
1440 task_warn!(worker, "Failed to remove datastore directory: {err}");
1441 }
1442 }
1443 } else {
1444 task_log!(worker, "There were errors deleting data.");
1445 }
1446 }
1447
1448 Ok(())
1449 }
1450}