1 //! Backup media Inventory
3 //! The Inventory persistently stores the list of known backup
4 //! media. A backup media is identified by its 'MediaId', which is the
5 //! MediaLabel/MediaSetLabel combination.
9 //! The inventory itself has several methods to update single entries,
10 //! but all of them can be considered atomic.
14 //! To add/modify media assigned to a pool, we always do
15 //! lock_media_pool(). For unassigned media, we call
16 //! lock_unassigned_media_pool().
20 //! To add/remove media from a media set, or to modify catalogs we
21 //! always do lock_media_set(). Also, we acquire this lock during
22 //! restore, to make sure it is not reused for backups.
25 use std
::collections
::{BTreeMap, HashMap}
;
26 use std
::path
::{Path, PathBuf}
;
27 use std
::time
::Duration
;
29 use anyhow
::{bail, Error}
;
30 use serde
::{Deserialize, Serialize}
;
33 use proxmox_sys
::fs
::{file_get_json, replace_file, CreateOptions}
;
34 use proxmox_uuid
::Uuid
;
36 use pbs_api_types
::{Fingerprint, MediaLocation, MediaSetPolicy, MediaStatus, RetentionPolicy}
;
37 use pbs_config
::BackupLockGuard
;
40 use pbs_config
::open_backup_lockfile
;
43 fn open_backup_lockfile
<P
: AsRef
<std
::path
::Path
>>(
45 _timeout
: Option
<std
::time
::Duration
>,
47 ) -> Result
<pbs_config
::BackupLockGuard
, anyhow
::Error
> {
48 Ok(unsafe { pbs_config::create_mocked_lock() }
)
52 changer
::OnlineStatusMap
,
53 file_formats
::{MediaLabel, MediaSetLabel}
,
54 MediaCatalog
, MediaSet
, TAPE_STATUS_DIR
,
57 /// Unique Media Identifier
59 /// This combines the label and media set label.
60 #[derive(Debug, Serialize, Deserialize, Clone)]
62 pub label
: MediaLabel
,
63 #[serde(skip_serializing_if = "Option::is_none")]
64 pub media_set_label
: Option
<MediaSetLabel
>,
68 pub fn pool(&self) -> Option
<String
> {
69 if let Some(set
) = &self.media_set_label
{
70 return Some(set
.pool
.to_owned());
72 self.label
.pool
.to_owned()
74 pub(crate) fn get_encryption_fp(&self) -> Option
<(Fingerprint
, Uuid
)> {
75 let label
= self.clone().media_set_label?
;
76 label
.encryption_key_fingerprint
.map(|fp
| (fp
, label
.uuid
))
80 #[derive(Serialize, Deserialize)]
81 struct MediaStateEntry
{
83 #[serde(skip_serializing_if = "Option::is_none")]
84 location
: Option
<MediaLocation
>,
85 #[serde(skip_serializing_if = "Option::is_none")]
86 status
: Option
<MediaStatus
>,
90 pub struct Inventory
{
91 map
: BTreeMap
<Uuid
, MediaStateEntry
>,
93 inventory_path
: PathBuf
,
94 lockfile_path
: PathBuf
,
97 media_set_start_times
: HashMap
<Uuid
, i64>,
101 pub const MEDIA_INVENTORY_FILENAME
: &'
static str = "inventory.json";
102 pub const MEDIA_INVENTORY_LOCKFILE
: &'
static str = ".inventory.lck";
104 /// Create empty instance, no data loaded
105 pub fn new
<P
: AsRef
<Path
>>(base_path
: P
) -> Self {
106 let mut inventory_path
= base_path
.as_ref().to_owned();
107 inventory_path
.push(Self::MEDIA_INVENTORY_FILENAME
);
109 let mut lockfile_path
= base_path
.as_ref().to_owned();
110 lockfile_path
.push(Self::MEDIA_INVENTORY_LOCKFILE
);
113 map
: BTreeMap
::new(),
114 media_set_start_times
: HashMap
::new(),
120 pub fn load
<P
: AsRef
<Path
>>(base_path
: P
) -> Result
<Self, Error
> {
121 let mut me
= Self::new(base_path
);
126 /// Reload the database
127 pub fn reload(&mut self) -> Result
<(), Error
> {
128 self.map
= self.load_media_db()?
;
129 self.update_helpers();
133 fn update_helpers(&mut self) {
134 // recompute media_set_start_times
136 let mut set_start_times
= HashMap
::new();
138 for entry
in self.map
.values() {
139 let set
= match &entry
.id
.media_set_label
{
144 set_start_times
.insert(set
.uuid
.clone(), set
.ctime
);
148 self.media_set_start_times
= set_start_times
;
151 /// Lock the database
152 fn lock(&self) -> Result
<BackupLockGuard
, Error
> {
153 open_backup_lockfile(&self.lockfile_path
, None
, true)
156 fn load_media_db(&self) -> Result
<BTreeMap
<Uuid
, MediaStateEntry
>, Error
> {
157 let data
= file_get_json(&self.inventory_path
, Some(json
!([])))?
;
158 let media_list
: Vec
<MediaStateEntry
> = serde_json
::from_value(data
)?
;
160 let mut map
= BTreeMap
::new();
161 for entry
in media_list
.into_iter() {
162 map
.insert(entry
.id
.label
.uuid
.clone(), entry
);
168 fn replace_file(&self) -> Result
<(), Error
> {
169 let list
: Vec
<&MediaStateEntry
> = self.map
.values().collect();
170 let raw
= serde_json
::to_string_pretty(&serde_json
::to_value(list
)?
)?
;
172 let mode
= nix
::sys
::stat
::Mode
::from_bits_truncate(0o0640);
174 let options
= if cfg
!(test
) {
175 // We cannot use chown inside test environment (no permissions)
176 CreateOptions
::new().perm(mode
)
178 let backup_user
= pbs_config
::backup_user()?
;
181 .owner(backup_user
.uid
)
182 .group(backup_user
.gid
)
185 replace_file(&self.inventory_path
, raw
.as_bytes(), options
, true)?
;
190 /// Stores a single MediaID persistently
191 pub fn store(&mut self, mut media_id
: MediaId
, clear_media_status
: bool
) -> Result
<(), Error
> {
192 let _lock
= self.lock()?
;
193 self.map
= self.load_media_db()?
;
195 let uuid
= media_id
.label
.uuid
.clone();
197 if let Some(previous
) = self.map
.remove(&media_id
.label
.uuid
) {
198 // do not overwrite unsaved pool assignments
199 if media_id
.media_set_label
.is_none() {
200 if let Some(ref set
) = previous
.id
.media_set_label
{
201 if set
.unassigned() {
202 media_id
.media_set_label
= Some(set
.clone());
206 let entry
= MediaStateEntry
{
208 location
: previous
.location
,
209 status
: if clear_media_status
{
215 self.map
.insert(uuid
, entry
);
217 let entry
= MediaStateEntry
{
222 self.map
.insert(uuid
, entry
);
225 self.update_helpers();
226 self.replace_file()?
;
230 /// Remove a single media persistently
231 pub fn remove_media(&mut self, uuid
: &Uuid
) -> Result
<(), Error
> {
232 let _lock
= self.lock()?
;
233 self.map
= self.load_media_db()?
;
234 self.map
.remove(uuid
);
235 self.update_helpers();
236 self.replace_file()?
;
241 pub fn lookup_media(&self, uuid
: &Uuid
) -> Option
<&MediaId
> {
242 self.map
.get(uuid
).map(|entry
| &entry
.id
)
245 /// List all media Uuids
246 pub fn media_list(&self) -> Vec
<&Uuid
> {
247 self.map
.keys().collect()
250 /// find media by label_text
251 pub fn find_media_by_label_text(&self, label_text
: &str) -> Result
<Option
<&MediaId
>, Error
> {
252 let ids
: Vec
<_
> = self
255 .filter_map(|entry
| {
256 if entry
.id
.label
.label_text
== label_text
{
266 1 => Ok(Some(ids
[0])),
267 count
=> bail
!("There are '{count}' tapes with the label '{label_text}'"),
271 /// Lookup media pool
273 /// Returns (pool, is_empty)
274 pub fn lookup_media_pool(&self, uuid
: &Uuid
) -> Option
<(&str, bool
)> {
275 let media_id
= &self.map
.get(uuid
)?
.id
;
276 match (&media_id
.label
.pool
, &media_id
.media_set_label
) {
277 (_
, Some(media_set
)) => Some((media_set
.pool
.as_str(), media_set
.unassigned())),
278 (Some(pool
), None
) => Some((pool
.as_str(), true)),
279 (None
, None
) => None
,
283 /// List all media assigned to the pool
284 pub fn list_pool_media(&self, pool
: &str) -> Vec
<MediaId
> {
285 let mut list
= Vec
::new();
287 for entry
in self.map
.values() {
288 if entry
.id
.pool().as_deref() == Some(pool
) {
289 match entry
.id
.media_set_label
{
290 Some(ref set
) if set
.unassigned() => list
.push(MediaId
{
291 label
: entry
.id
.label
.clone(),
292 media_set_label
: None
,
295 list
.push(entry
.id
.clone());
304 /// List all used media
305 pub fn list_used_media(&self) -> Vec
<MediaId
> {
308 .filter_map(|entry
| match entry
.id
.media_set_label
{
309 Some(ref set
) if !set
.unassigned() => Some(entry
.id
.clone()),
315 /// List media not assigned to any pool
316 pub fn list_unassigned_media(&self) -> Vec
<MediaId
> {
319 .filter_map(|entry
| match entry
.id
.pool() {
320 None
=> Some(entry
.id
.clone()),
326 pub fn media_set_start_time(&self, media_set_uuid
: &Uuid
) -> Option
<i64> {
327 self.media_set_start_times
.get(media_set_uuid
).copied()
330 /// Lookup media set pool
331 pub fn lookup_media_set_pool(&self, media_set_uuid
: &Uuid
) -> Result
<String
, Error
> {
332 let mut last_pool
= None
;
334 for entry
in self.map
.values() {
335 match entry
.id
.media_set_label
{
337 Some(MediaSetLabel { ref uuid, .. }
) => {
338 if uuid
!= media_set_uuid
{
341 if let Some((pool
, _
)) = self.lookup_media_pool(&entry
.id
.label
.uuid
) {
342 if let Some(last_pool
) = last_pool
{
343 if last_pool
!= pool
{
344 bail
!("detected media set with inconsistent pool assignment - internal error");
347 last_pool
= Some(pool
);
355 Some(pool
) => Ok(pool
.to_string()),
357 "media set {} is incomplete - unable to lookup pool",
363 /// Compute a single media sets
364 pub fn compute_media_set_members(&self, media_set_uuid
: &Uuid
) -> Result
<MediaSet
, Error
> {
365 let mut set
= MediaSet
::with_data(media_set_uuid
.clone(), Vec
::new());
367 for entry
in self.map
.values() {
368 match entry
.id
.media_set_label
{
373 if uuid
!= media_set_uuid
{
376 set
.insert_media(entry
.id
.label
.uuid
.clone(), seq_nr
)?
;
384 /// Compute all media sets
385 pub fn compute_media_set_list(&self) -> Result
<HashMap
<Uuid
, MediaSet
>, Error
> {
386 let mut set_map
: HashMap
<Uuid
, MediaSet
> = HashMap
::new();
388 for entry
in self.map
.values() {
389 match entry
.id
.media_set_label
{
396 .or_insert_with(|| MediaSet
::with_data(uuid
.clone(), Vec
::new()));
398 set
.insert_media(entry
.id
.label
.uuid
.clone(), seq_nr
)?
;
406 /// Returns the latest media set for a pool
407 pub fn latest_media_set(&self, pool
: &str) -> Option
<Uuid
> {
408 let mut last_set
: Option
<(Uuid
, i64)> = None
;
413 .filter_map(|entry
| entry
.id
.media_set_label
.as_ref())
414 .filter(|set
| set
.pool
== pool
&& !set
.unassigned());
416 for set
in set_list
{
419 last_set
= Some((set
.uuid
.clone(), set
.ctime
));
421 Some((_
, last_ctime
)) => {
422 if set
.ctime
> last_ctime
{
423 last_set
= Some((set
.uuid
.clone(), set
.ctime
));
429 let (uuid
, ctime
) = match last_set
{
431 Some((uuid
, ctime
)) => (uuid
, ctime
),
434 // consistency check - must be the only set with that ctime
438 .filter_map(|entry
| entry
.id
.media_set_label
.as_ref())
439 .filter(|set
| set
.pool
== pool
&& !set
.unassigned());
441 for set
in set_list
{
442 if set
.uuid
!= uuid
&& set
.ctime
>= ctime
{
445 "latest_media_set: found set with equal ctime ({}, {})",
455 // Test if there is a media set (in the same pool) newer than this one.
456 // Return the ctime of the nearest media set
457 fn media_set_next_start_time(&self, media_set_uuid
: &Uuid
) -> Option
<i64> {
458 let (pool
, ctime
) = match self
461 .filter_map(|entry
| entry
.id
.media_set_label
.as_ref())
463 if &set
.uuid
== media_set_uuid
{
464 Some((set
.pool
.clone(), set
.ctime
))
469 Some((pool
, ctime
)) => (pool
, ctime
),
476 .filter_map(|entry
| entry
.id
.media_set_label
.as_ref())
477 .filter(|set
| (&set
.uuid
!= media_set_uuid
) && (set
.pool
== pool
));
479 let mut next_ctime
= None
;
481 for set
in set_list
{
482 if set
.ctime
> ctime
{
485 next_ctime
= Some(set
.ctime
);
487 Some(last_next_ctime
) => {
488 if set
.ctime
< last_next_ctime
{
489 next_ctime
= Some(set
.ctime
);
499 pub fn media_expire_time(
502 media_set_policy
: &MediaSetPolicy
,
503 retention_policy
: &RetentionPolicy
,
505 if let RetentionPolicy
::KeepForever
= retention_policy
{
509 let set
= match media
.media_set_label
{
510 None
=> return i64::MAX
,
511 Some(ref set
) => set
,
514 let set_start_time
= match self.media_set_start_time(&set
.uuid
) {
516 // missing information, use ctime from this
517 // set (always greater than ctime from seq_nr 0)
523 let max_use_time
= match self.media_set_next_start_time(&set
.uuid
) {
524 Some(next_start_time
) => match media_set_policy
{
525 MediaSetPolicy
::AlwaysCreate
=> set_start_time
,
526 _
=> next_start_time
,
528 None
=> match media_set_policy
{
529 MediaSetPolicy
::ContinueCurrent
=> {
532 MediaSetPolicy
::AlwaysCreate
=> set_start_time
,
533 MediaSetPolicy
::CreateAt(ref event
) => {
534 match event
.compute_next_event(set_start_time
) {
535 Ok(Some(next
)) => next
,
536 Ok(None
) | Err(_
) => return i64::MAX
,
542 match retention_policy
{
543 RetentionPolicy
::KeepForever
=> i64::MAX
,
544 RetentionPolicy
::OverwriteAlways
=> max_use_time
,
545 RetentionPolicy
::ProtectFor(time_span
) => {
546 let seconds
= f64::from(time_span
.clone()) as i64;
547 max_use_time
+ seconds
552 /// Generate a human readable name for the media set
554 /// The template can include strftime time format specifications.
555 pub fn generate_media_set_name(
557 media_set_uuid
: &Uuid
,
558 template
: Option
<String
>,
559 ) -> Result
<String
, Error
> {
560 if let Some(ctime
) = self.media_set_start_time(media_set_uuid
) {
561 let mut template
= template
.unwrap_or_else(|| String
::from("%c"));
562 template
= template
.replace("%id%", &media_set_uuid
.to_string());
563 Ok(proxmox_time
::strftime_local(&template
, ctime
)?
)
565 // We don't know the set start time, so we cannot use the template
566 Ok(media_set_uuid
.to_string())
570 // Helpers to simplify testing
572 /// Generate and insert a new free tape (test helper)
573 pub fn generate_free_tape(&mut self, label_text
: &str, ctime
: i64) -> Uuid
{
574 let label
= MediaLabel
{
575 label_text
: label_text
.to_string(),
576 uuid
: Uuid
::generate(),
580 let uuid
= label
.uuid
.clone();
585 media_set_label
: None
,
594 /// Generate and insert a new tape assigned to a specific pool
596 pub fn generate_assigned_tape(&mut self, label_text
: &str, pool
: &str, ctime
: i64) -> Uuid
{
597 let label
= MediaLabel
{
598 label_text
: label_text
.to_string(),
599 uuid
: Uuid
::generate(),
601 pool
: Some(pool
.to_string()),
604 let uuid
= label
.uuid
.clone();
609 media_set_label
: None
,
618 /// Generate and insert a used tape (test helper)
619 pub fn generate_used_tape(&mut self, label_text
: &str, set
: MediaSetLabel
, ctime
: i64) -> Uuid
{
620 let label
= MediaLabel
{
621 label_text
: label_text
.to_string(),
622 uuid
: Uuid
::generate(),
624 pool
: Some(set
.pool
.clone()),
626 let uuid
= label
.uuid
.clone();
631 media_set_label
: Some(set
),
641 // Status/location handling
643 /// Returns status and location with reasonable defaults.
645 /// Default status is 'MediaStatus::Unknown'.
646 /// Default location is 'MediaLocation::Offline'.
647 pub fn status_and_location(&self, uuid
: &Uuid
) -> (MediaStatus
, MediaLocation
) {
648 match self.map
.get(uuid
) {
650 // no info stored - assume media is writable/offline
651 (MediaStatus
::Unknown
, MediaLocation
::Offline
)
654 let location
= entry
.location
.clone().unwrap_or(MediaLocation
::Offline
);
655 let status
= entry
.status
.unwrap_or(MediaStatus
::Unknown
);
661 // Lock database, reload database, set status, store database
662 fn set_media_status(&mut self, uuid
: &Uuid
, status
: Option
<MediaStatus
>) -> Result
<(), Error
> {
663 let _lock
= self.lock()?
;
664 self.map
= self.load_media_db()?
;
665 if let Some(entry
) = self.map
.get_mut(uuid
) {
666 entry
.status
= status
;
667 self.update_helpers();
668 self.replace_file()?
;
671 bail
!("no such media '{}'", uuid
);
675 /// Lock database, reload database, set status to Full, store database
676 pub fn set_media_status_full(&mut self, uuid
: &Uuid
) -> Result
<(), Error
> {
677 self.set_media_status(uuid
, Some(MediaStatus
::Full
))
680 /// Lock database, reload database, set status to Damaged, store database
681 pub fn set_media_status_damaged(&mut self, uuid
: &Uuid
) -> Result
<(), Error
> {
682 self.set_media_status(uuid
, Some(MediaStatus
::Damaged
))
685 /// Lock database, reload database, set status to Retired, store database
686 pub fn set_media_status_retired(&mut self, uuid
: &Uuid
) -> Result
<(), Error
> {
687 self.set_media_status(uuid
, Some(MediaStatus
::Retired
))
690 /// Lock database, reload database, set status to None, store database
691 pub fn clear_media_status(&mut self, uuid
: &Uuid
) -> Result
<(), Error
> {
692 self.set_media_status(uuid
, None
)
695 // Lock database, reload database, set location, store database
696 fn set_media_location(
699 location
: Option
<MediaLocation
>,
700 ) -> Result
<(), Error
> {
701 let _lock
= self.lock()?
;
702 self.map
= self.load_media_db()?
;
703 if let Some(entry
) = self.map
.get_mut(uuid
) {
704 entry
.location
= location
;
705 self.update_helpers();
706 self.replace_file()?
;
709 bail
!("no such media '{}'", uuid
);
713 /// Lock database, reload database, set location to vault, store database
714 pub fn set_media_location_vault(&mut self, uuid
: &Uuid
, vault
: &str) -> Result
<(), Error
> {
715 self.set_media_location(uuid
, Some(MediaLocation
::Vault(vault
.to_string())))
718 /// Lock database, reload database, set location to offline, store database
719 pub fn set_media_location_offline(&mut self, uuid
: &Uuid
) -> Result
<(), Error
> {
720 self.set_media_location(uuid
, Some(MediaLocation
::Offline
))
723 /// Update online status
724 pub fn update_online_status(&mut self, online_map
: &OnlineStatusMap
) -> Result
<(), Error
> {
725 let _lock
= self.lock()?
;
726 self.map
= self.load_media_db()?
;
728 for (uuid
, entry
) in self.map
.iter_mut() {
729 if let Some(changer_name
) = online_map
.lookup_changer(uuid
) {
730 entry
.location
= Some(MediaLocation
::Online(changer_name
.to_string()));
731 } else if let Some(MediaLocation
::Online(ref changer_name
)) = entry
.location
{
732 match online_map
.online_map(changer_name
) {
734 // no such changer device
735 entry
.location
= Some(MediaLocation
::Offline
);
738 // got no info - do nothing
741 // media changer changed
742 entry
.location
= Some(MediaLocation
::Offline
);
748 self.update_helpers();
749 self.replace_file()?
;
755 /// Lock a media pool
756 pub fn lock_media_pool
<P
: AsRef
<Path
>>(base_path
: P
, name
: &str) -> Result
<BackupLockGuard
, Error
> {
757 let mut path
= base_path
.as_ref().to_owned();
758 path
.push(format
!(".pool-{}", name
));
759 path
.set_extension("lck");
761 open_backup_lockfile(&path
, None
, true)
764 /// Lock for media not assigned to any pool
765 pub fn lock_unassigned_media_pool
<P
: AsRef
<Path
>>(base_path
: P
) -> Result
<BackupLockGuard
, Error
> {
766 // lock artificial "__UNASSIGNED__" pool to avoid races
767 lock_media_pool(base_path
, "__UNASSIGNED__")
772 /// Timeout is 10 seconds by default
773 pub fn lock_media_set
<P
: AsRef
<Path
>>(
775 media_set_uuid
: &Uuid
,
776 timeout
: Option
<Duration
>,
777 ) -> Result
<BackupLockGuard
, Error
> {
778 let mut path
= base_path
.as_ref().to_owned();
779 path
.push(format
!(".media-set-{}", media_set_uuid
));
780 path
.set_extension("lck");
782 open_backup_lockfile(&path
, timeout
, true)
785 // shell completion helper
787 /// List of known media uuids
788 pub fn complete_media_uuid(_arg
: &str, _param
: &HashMap
<String
, String
>) -> Vec
<String
> {
789 let inventory
= match Inventory
::load(TAPE_STATUS_DIR
) {
790 Ok(inventory
) => inventory
,
791 Err(_
) => return Vec
::new(),
794 inventory
.map
.keys().map(|uuid
| uuid
.to_string()).collect()
797 /// List of known media sets
798 pub fn complete_media_set_uuid(_arg
: &str, _param
: &HashMap
<String
, String
>) -> Vec
<String
> {
799 let inventory
= match Inventory
::load(TAPE_STATUS_DIR
) {
800 Ok(inventory
) => inventory
,
801 Err(_
) => return Vec
::new(),
807 .filter_map(|entry
| entry
.id
.media_set_label
.as_ref())
808 .map(|set
| set
.uuid
.to_string())
812 /// List of known media labels (barcodes)
813 pub fn complete_media_label_text(_arg
: &str, _param
: &HashMap
<String
, String
>) -> Vec
<String
> {
814 let inventory
= match Inventory
::load(TAPE_STATUS_DIR
) {
815 Ok(inventory
) => inventory
,
816 Err(_
) => return Vec
::new(),
822 .map(|entry
| entry
.id
.label
.label_text
.clone())
826 pub fn complete_media_set_snapshots(_arg
: &str, param
: &HashMap
<String
, String
>) -> Vec
<String
> {
827 let media_set_uuid
: Uuid
= match param
.get("media-set").and_then(|s
| s
.parse().ok()) {
829 None
=> return Vec
::new(),
831 let inventory
= match Inventory
::load(TAPE_STATUS_DIR
) {
832 Ok(inventory
) => inventory
,
833 Err(_
) => return Vec
::new(),
836 let mut res
= Vec
::new();
841 .filter(|media
| match &media
.media_set_label
{
842 Some(label
) => label
.uuid
== media_set_uuid
,
846 for media_id
in media_ids
{
847 let catalog
= match MediaCatalog
::open(TAPE_STATUS_DIR
, &media_id
, false, false) {
848 Ok(catalog
) => catalog
,
852 for (store
, content
) in catalog
.content() {
853 for snapshot
in content
.snapshot_index
.keys() {
854 res
.push(format
!("{}:{}", store
, snapshot
));