]> git.proxmox.com Git - proxmox-backup.git/blob - src/tape/inventory.rs
tape: add Inventory.media_list() helper
[proxmox-backup.git] / src / tape / inventory.rs
1 //! Backup media Inventory
2 //!
3 //! The Inventory persistently stores the list of known backup
4 //! media. A backup media is identified by its 'MediaId', which is the
5 //! MediaLabel/MediaSetLabel combination.
6
7 use std::collections::{HashMap, BTreeMap};
8 use std::path::{Path, PathBuf};
9 use std::os::unix::io::AsRawFd;
10
11 use anyhow::{bail, Error};
12 use serde::{Serialize, Deserialize};
13 use serde_json::json;
14
15 use proxmox::tools::{
16 Uuid,
17 fs::{
18 open_file_locked,
19 replace_file,
20 fchown,
21 file_get_json,
22 CreateOptions,
23 },
24 };
25
26 use crate::{
27 tools::systemd::time::compute_next_event,
28 api2::types::{
29 MediaSetPolicy,
30 RetentionPolicy,
31 MediaStatus,
32 MediaLocation,
33 },
34 tape::{
35 TAPE_STATUS_DIR,
36 MediaSet,
37 file_formats::{
38 MediaLabel,
39 MediaSetLabel,
40 },
41 changer::OnlineStatusMap,
42 },
43 };
44
45 /// Unique Media Identifier
46 ///
47 /// This combines the label and media set label.
48 #[derive(Debug,Serialize,Deserialize,Clone)]
49 pub struct MediaId {
50 pub label: MediaLabel,
51 #[serde(skip_serializing_if="Option::is_none")]
52 pub media_set_label: Option<MediaSetLabel>,
53 }
54
55
56 #[derive(Serialize,Deserialize)]
57 struct MediaStateEntry {
58 id: MediaId,
59 #[serde(skip_serializing_if="Option::is_none")]
60 location: Option<MediaLocation>,
61 #[serde(skip_serializing_if="Option::is_none")]
62 status: Option<MediaStatus>,
63 }
64
65 /// Media Inventory
66 pub struct Inventory {
67 map: BTreeMap<Uuid, MediaStateEntry>,
68
69 inventory_path: PathBuf,
70 lockfile_path: PathBuf,
71
72 // helpers
73 media_set_start_times: HashMap<Uuid, i64>
74 }
75
76 impl Inventory {
77
78 pub const MEDIA_INVENTORY_FILENAME: &'static str = "inventory.json";
79 pub const MEDIA_INVENTORY_LOCKFILE: &'static str = ".inventory.lck";
80
81 fn new(base_path: &Path) -> Self {
82
83 let mut inventory_path = base_path.to_owned();
84 inventory_path.push(Self::MEDIA_INVENTORY_FILENAME);
85
86 let mut lockfile_path = base_path.to_owned();
87 lockfile_path.push(Self::MEDIA_INVENTORY_LOCKFILE);
88
89 Self {
90 map: BTreeMap::new(),
91 media_set_start_times: HashMap::new(),
92 inventory_path,
93 lockfile_path,
94 }
95 }
96
97 pub fn load(base_path: &Path) -> Result<Self, Error> {
98 let mut me = Self::new(base_path);
99 me.reload()?;
100 Ok(me)
101 }
102
103 /// Reload the database
104 pub fn reload(&mut self) -> Result<(), Error> {
105 self.map = Self::load_media_db(&self.inventory_path)?;
106 self.update_helpers();
107 Ok(())
108 }
109
110 fn update_helpers(&mut self) {
111
112 // recompute media_set_start_times
113
114 let mut set_start_times = HashMap::new();
115
116 for entry in self.map.values() {
117 let set = match &entry.id.media_set_label {
118 None => continue,
119 Some(set) => set,
120 };
121 if set.seq_nr == 0 {
122 set_start_times.insert(set.uuid.clone(), set.ctime);
123 }
124 }
125
126 self.media_set_start_times = set_start_times;
127 }
128
129 /// Lock the database
130 pub fn lock(&self) -> Result<std::fs::File, Error> {
131 let file = open_file_locked(&self.lockfile_path, std::time::Duration::new(10, 0), true)?;
132 if cfg!(test) {
133 // We cannot use chown inside test environment (no permissions)
134 return Ok(file);
135 }
136
137 let backup_user = crate::backup::backup_user()?;
138 fchown(file.as_raw_fd(), Some(backup_user.uid), Some(backup_user.gid))?;
139
140 Ok(file)
141 }
142
143 fn load_media_db(path: &Path) -> Result<BTreeMap<Uuid, MediaStateEntry>, Error> {
144
145 let data = file_get_json(path, Some(json!([])))?;
146 let media_list: Vec<MediaStateEntry> = serde_json::from_value(data)?;
147
148 let mut map = BTreeMap::new();
149 for entry in media_list.into_iter() {
150 map.insert(entry.id.label.uuid.clone(), entry);
151 }
152
153 Ok(map)
154 }
155
156 fn replace_file(&self) -> Result<(), Error> {
157 let list: Vec<&MediaStateEntry> = self.map.values().collect();
158 let raw = serde_json::to_string_pretty(&serde_json::to_value(list)?)?;
159
160 let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640);
161
162 let options = if cfg!(test) {
163 // We cannot use chown inside test environment (no permissions)
164 CreateOptions::new().perm(mode)
165 } else {
166 let backup_user = crate::backup::backup_user()?;
167 CreateOptions::new()
168 .perm(mode)
169 .owner(backup_user.uid)
170 .group(backup_user.gid)
171 };
172
173 replace_file(&self.inventory_path, raw.as_bytes(), options)?;
174
175 Ok(())
176 }
177
178 /// Stores a single MediaID persistently
179 pub fn store(
180 &mut self,
181 mut media_id: MediaId,
182 clear_media_status: bool,
183 ) -> Result<(), Error> {
184 let _lock = self.lock()?;
185 self.map = Self::load_media_db(&self.inventory_path)?;
186
187 let uuid = media_id.label.uuid.clone();
188
189 if let Some(previous) = self.map.remove(&media_id.label.uuid) {
190 // do not overwrite unsaved pool assignments
191 if media_id.media_set_label.is_none() {
192 if let Some(ref set) = previous.id.media_set_label {
193 if set.uuid.as_ref() == [0u8;16] {
194 media_id.media_set_label = Some(set.clone());
195 }
196 }
197 }
198 let entry = MediaStateEntry {
199 id: media_id,
200 location: previous.location,
201 status: if clear_media_status {
202 None
203 } else {
204 previous.status
205 },
206 };
207 self.map.insert(uuid, entry);
208 } else {
209 let entry = MediaStateEntry { id: media_id, location: None, status: None };
210 self.map.insert(uuid, entry);
211 }
212
213 self.update_helpers();
214 self.replace_file()?;
215 Ok(())
216 }
217
218 /// Remove a single media persistently
219 pub fn remove_media(&mut self, uuid: &Uuid) -> Result<(), Error> {
220 let _lock = self.lock()?;
221 self.map = Self::load_media_db(&self.inventory_path)?;
222 self.map.remove(uuid);
223 self.update_helpers();
224 self.replace_file()?;
225 Ok(())
226 }
227
228 /// Lookup media
229 pub fn lookup_media(&self, uuid: &Uuid) -> Option<&MediaId> {
230 self.map.get(uuid).map(|entry| &entry.id)
231 }
232
233 /// List all media Uuids
234 pub fn media_list(&self) -> Vec<&Uuid> {
235 self.map.keys().collect()
236 }
237
238 /// find media by label_text
239 pub fn find_media_by_label_text(&self, label_text: &str) -> Option<&MediaId> {
240 self.map.values().find_map(|entry| {
241 if entry.id.label.label_text == label_text {
242 Some(&entry.id)
243 } else {
244 None
245 }
246 })
247 }
248
249 /// Lookup media pool
250 ///
251 /// Returns (pool, is_empty)
252 pub fn lookup_media_pool(&self, uuid: &Uuid) -> Option<(&str, bool)> {
253 match self.map.get(uuid) {
254 None => None,
255 Some(entry) => {
256 match entry.id.media_set_label {
257 None => None, // not assigned to any pool
258 Some(ref set) => {
259 let is_empty = set.uuid.as_ref() == [0u8;16];
260 Some((&set.pool, is_empty))
261 }
262 }
263 }
264 }
265 }
266
267 /// List all media assigned to the pool
268 pub fn list_pool_media(&self, pool: &str) -> Vec<MediaId> {
269 let mut list = Vec::new();
270
271 for entry in self.map.values() {
272 match entry.id.media_set_label {
273 None => continue, // not assigned to any pool
274 Some(ref set) => {
275 if set.pool != pool {
276 continue; // belong to another pool
277 }
278
279 if set.uuid.as_ref() == [0u8;16] { // should we do this??
280 list.push(MediaId {
281 label: entry.id.label.clone(),
282 media_set_label: None,
283 })
284 } else {
285 list.push(entry.id.clone());
286 }
287 }
288 }
289 }
290
291 list
292 }
293
294 /// List all used media
295 pub fn list_used_media(&self) -> Vec<MediaId> {
296 let mut list = Vec::new();
297
298 for entry in self.map.values() {
299 match entry.id.media_set_label {
300 None => continue, // not assigned to any pool
301 Some(ref set) => {
302 if set.uuid.as_ref() != [0u8;16] {
303 list.push(entry.id.clone());
304 }
305 }
306 }
307 }
308
309 list
310 }
311
312 /// List media not assigned to any pool
313 pub fn list_unassigned_media(&self) -> Vec<MediaId> {
314 self.map.values().filter_map(|entry|
315 if entry.id.media_set_label.is_none() {
316 Some(entry.id.clone())
317 } else {
318 None
319 }
320 ).collect()
321 }
322
323 pub fn media_set_start_time(&self, media_set_uuid: &Uuid) -> Option<i64> {
324 self.media_set_start_times.get(media_set_uuid).copied()
325 }
326
327 /// Lookup media set pool
328 pub fn lookup_media_set_pool(&self, media_set_uuid: &Uuid) -> Result<String, Error> {
329
330 let mut last_pool = None;
331
332 for entry in self.map.values() {
333 match entry.id.media_set_label {
334 None => continue,
335 Some(MediaSetLabel { ref uuid, .. }) => {
336 if uuid != media_set_uuid {
337 continue;
338 }
339 if let Some((pool, _)) = self.lookup_media_pool(&entry.id.label.uuid) {
340 if let Some(last_pool) = last_pool {
341 if last_pool != pool {
342 bail!("detected media set with inconsistent pool assignment - internal error");
343 }
344 } else {
345 last_pool = Some(pool);
346 }
347 }
348 }
349 }
350 }
351
352 match last_pool {
353 Some(pool) => Ok(pool.to_string()),
354 None => bail!("media set {} is incomplete - unable to lookup pool", media_set_uuid),
355 }
356 }
357
358 /// Compute a single media sets
359 pub fn compute_media_set_members(&self, media_set_uuid: &Uuid) -> Result<MediaSet, Error> {
360
361 let mut set = MediaSet::with_data(media_set_uuid.clone(), Vec::new());
362
363 for entry in self.map.values() {
364 match entry.id.media_set_label {
365 None => continue,
366 Some(MediaSetLabel { seq_nr, ref uuid, .. }) => {
367 if uuid != media_set_uuid {
368 continue;
369 }
370 set.insert_media(entry.id.label.uuid.clone(), seq_nr)?;
371 }
372 }
373 }
374
375 Ok(set)
376 }
377
378 /// Compute all media sets
379 pub fn compute_media_set_list(&self) -> Result<HashMap<Uuid, MediaSet>, Error> {
380
381 let mut set_map: HashMap<Uuid, MediaSet> = HashMap::new();
382
383 for entry in self.map.values() {
384 match entry.id.media_set_label {
385 None => continue,
386 Some(MediaSetLabel { seq_nr, ref uuid, .. }) => {
387
388 let set = set_map.entry(uuid.clone()).or_insert_with(|| {
389 MediaSet::with_data(uuid.clone(), Vec::new())
390 });
391
392 set.insert_media(entry.id.label.uuid.clone(), seq_nr)?;
393 }
394 }
395 }
396
397 Ok(set_map)
398 }
399
400 /// Returns the latest media set for a pool
401 pub fn latest_media_set(&self, pool: &str) -> Option<Uuid> {
402
403 let mut last_set: Option<(Uuid, i64)> = None;
404
405 let set_list = self.map.values()
406 .filter_map(|entry| entry.id.media_set_label.as_ref())
407 .filter(|set| set.pool == pool && set.uuid.as_ref() != [0u8;16]);
408
409 for set in set_list {
410 match last_set {
411 None => {
412 last_set = Some((set.uuid.clone(), set.ctime));
413 }
414 Some((_, last_ctime)) => {
415 if set.ctime > last_ctime {
416 last_set = Some((set.uuid.clone(), set.ctime));
417 }
418 }
419 }
420 }
421
422 let (uuid, ctime) = match last_set {
423 None => return None,
424 Some((uuid, ctime)) => (uuid, ctime),
425 };
426
427 // consistency check - must be the only set with that ctime
428 let set_list = self.map.values()
429 .filter_map(|entry| entry.id.media_set_label.as_ref())
430 .filter(|set| set.pool == pool && set.uuid.as_ref() != [0u8;16]);
431
432 for set in set_list {
433 if set.uuid != uuid && set.ctime >= ctime { // should not happen
434 eprintln!("latest_media_set: found set with equal ctime ({}, {})", set.uuid, uuid);
435 return None;
436 }
437 }
438
439 Some(uuid)
440 }
441
442 // Test if there is a media set (in the same pool) newer than this one.
443 // Return the ctime of the nearest media set
444 fn media_set_next_start_time(&self, media_set_uuid: &Uuid) -> Option<i64> {
445
446 let (pool, ctime) = match self.map.values()
447 .filter_map(|entry| entry.id.media_set_label.as_ref())
448 .find_map(|set| {
449 if &set.uuid == media_set_uuid {
450 Some((set.pool.clone(), set.ctime))
451 } else {
452 None
453 }
454 }) {
455 Some((pool, ctime)) => (pool, ctime),
456 None => return None,
457 };
458
459 let set_list = self.map.values()
460 .filter_map(|entry| entry.id.media_set_label.as_ref())
461 .filter(|set| (&set.uuid != media_set_uuid) && (set.pool == pool));
462
463 let mut next_ctime = None;
464
465 for set in set_list {
466 if set.ctime > ctime {
467 match next_ctime {
468 None => {
469 next_ctime = Some(set.ctime);
470 }
471 Some(last_next_ctime) => {
472 if set.ctime < last_next_ctime {
473 next_ctime = Some(set.ctime);
474 }
475 }
476 }
477 }
478 }
479
480 next_ctime
481 }
482
483 pub fn media_expire_time(
484 &self,
485 media: &MediaId,
486 media_set_policy: &MediaSetPolicy,
487 retention_policy: &RetentionPolicy,
488 ) -> i64 {
489
490 if let RetentionPolicy::KeepForever = retention_policy {
491 return i64::MAX;
492 }
493
494 let set = match media.media_set_label {
495 None => return i64::MAX,
496 Some(ref set) => set,
497 };
498
499 let set_start_time = match self.media_set_start_time(&set.uuid) {
500 None => {
501 // missing information, use ctime from this
502 // set (always greater than ctime from seq_nr 0)
503 set.ctime
504 }
505 Some(time) => time,
506 };
507
508 let max_use_time = match media_set_policy {
509 MediaSetPolicy::ContinueCurrent => {
510 match self.media_set_next_start_time(&set.uuid) {
511 Some(next_start_time) => next_start_time,
512 None => return i64::MAX,
513 }
514 }
515 MediaSetPolicy::AlwaysCreate => {
516 set_start_time + 1
517 }
518 MediaSetPolicy::CreateAt(ref event) => {
519 match compute_next_event(event, set_start_time, false) {
520 Ok(Some(next)) => next,
521 Ok(None) | Err(_) => return i64::MAX,
522 }
523 }
524 };
525
526 match retention_policy {
527 RetentionPolicy::KeepForever => i64::MAX,
528 RetentionPolicy::OverwriteAlways => max_use_time,
529 RetentionPolicy::ProtectFor(time_span) => {
530 let seconds = f64::from(time_span.clone()) as i64;
531 max_use_time + seconds
532 }
533 }
534 }
535
536 /// Generate a human readable name for the media set
537 ///
538 /// The template can include strftime time format specifications.
539 pub fn generate_media_set_name(
540 &self,
541 media_set_uuid: &Uuid,
542 template: Option<String>,
543 ) -> Result<String, Error> {
544
545 if let Some(ctime) = self.media_set_start_time(media_set_uuid) {
546 let mut template = template.unwrap_or_else(|| String::from("%c"));
547 template = template.replace("%id%", &media_set_uuid.to_string());
548 proxmox::tools::time::strftime_local(&template, ctime)
549 } else {
550 // We don't know the set start time, so we cannot use the template
551 Ok(media_set_uuid.to_string())
552 }
553 }
554
555 // Helpers to simplify testing
556
557 /// Genreate and insert a new free tape (test helper)
558 pub fn generate_free_tape(&mut self, label_text: &str, ctime: i64) -> Uuid {
559
560 let label = MediaLabel {
561 label_text: label_text.to_string(),
562 uuid: Uuid::generate(),
563 ctime,
564 };
565 let uuid = label.uuid.clone();
566
567 self.store(MediaId { label, media_set_label: None }, false).unwrap();
568
569 uuid
570 }
571
572 /// Genreate and insert a new tape assigned to a specific pool
573 /// (test helper)
574 pub fn generate_assigned_tape(
575 &mut self,
576 label_text: &str,
577 pool: &str,
578 ctime: i64,
579 ) -> Uuid {
580
581 let label = MediaLabel {
582 label_text: label_text.to_string(),
583 uuid: Uuid::generate(),
584 ctime,
585 };
586
587 let uuid = label.uuid.clone();
588
589 let set = MediaSetLabel::with_data(pool, [0u8; 16].into(), 0, ctime, None);
590
591 self.store(MediaId { label, media_set_label: Some(set) }, false).unwrap();
592
593 uuid
594 }
595
596 /// Genreate and insert a used tape (test helper)
597 pub fn generate_used_tape(
598 &mut self,
599 label_text: &str,
600 set: MediaSetLabel,
601 ctime: i64,
602 ) -> Uuid {
603 let label = MediaLabel {
604 label_text: label_text.to_string(),
605 uuid: Uuid::generate(),
606 ctime,
607 };
608 let uuid = label.uuid.clone();
609
610 self.store(MediaId { label, media_set_label: Some(set) }, false).unwrap();
611
612 uuid
613 }
614 }
615
616 // Status/location handling
617 impl Inventory {
618
619 /// Returns status and location with reasonable defaults.
620 ///
621 /// Default status is 'MediaStatus::Unknown'.
622 /// Default location is 'MediaLocation::Offline'.
623 pub fn status_and_location(&self, uuid: &Uuid) -> (MediaStatus, MediaLocation) {
624
625 match self.map.get(uuid) {
626 None => {
627 // no info stored - assume media is writable/offline
628 (MediaStatus::Unknown, MediaLocation::Offline)
629 }
630 Some(entry) => {
631 let location = entry.location.clone().unwrap_or(MediaLocation::Offline);
632 let status = entry.status.unwrap_or(MediaStatus::Unknown);
633 (status, location)
634 }
635 }
636 }
637
638 // Lock database, reload database, set status, store database
639 fn set_media_status(&mut self, uuid: &Uuid, status: Option<MediaStatus>) -> Result<(), Error> {
640 let _lock = self.lock()?;
641 self.map = Self::load_media_db(&self.inventory_path)?;
642 if let Some(entry) = self.map.get_mut(uuid) {
643 entry.status = status;
644 self.update_helpers();
645 self.replace_file()?;
646 Ok(())
647 } else {
648 bail!("no such media '{}'", uuid);
649 }
650 }
651
652 /// Lock database, reload database, set status to Full, store database
653 pub fn set_media_status_full(&mut self, uuid: &Uuid) -> Result<(), Error> {
654 self.set_media_status(uuid, Some(MediaStatus::Full))
655 }
656
657 /// Lock database, reload database, set status to Damaged, store database
658 pub fn set_media_status_damaged(&mut self, uuid: &Uuid) -> Result<(), Error> {
659 self.set_media_status(uuid, Some(MediaStatus::Damaged))
660 }
661
662 /// Lock database, reload database, set status to Retired, store database
663 pub fn set_media_status_retired(&mut self, uuid: &Uuid) -> Result<(), Error> {
664 self.set_media_status(uuid, Some(MediaStatus::Retired))
665 }
666
667 /// Lock database, reload database, set status to None, store database
668 pub fn clear_media_status(&mut self, uuid: &Uuid) -> Result<(), Error> {
669 self.set_media_status(uuid, None)
670 }
671
672 // Lock database, reload database, set location, store database
673 fn set_media_location(&mut self, uuid: &Uuid, location: Option<MediaLocation>) -> Result<(), Error> {
674 let _lock = self.lock()?;
675 self.map = Self::load_media_db(&self.inventory_path)?;
676 if let Some(entry) = self.map.get_mut(uuid) {
677 entry.location = location;
678 self.update_helpers();
679 self.replace_file()?;
680 Ok(())
681 } else {
682 bail!("no such media '{}'", uuid);
683 }
684 }
685
686 /// Lock database, reload database, set location to vault, store database
687 pub fn set_media_location_vault(&mut self, uuid: &Uuid, vault: &str) -> Result<(), Error> {
688 self.set_media_location(uuid, Some(MediaLocation::Vault(vault.to_string())))
689 }
690
691 /// Lock database, reload database, set location to offline, store database
692 pub fn set_media_location_offline(&mut self, uuid: &Uuid) -> Result<(), Error> {
693 self.set_media_location(uuid, Some(MediaLocation::Offline))
694 }
695
696 /// Update online status
697 pub fn update_online_status(&mut self, online_map: &OnlineStatusMap) -> Result<(), Error> {
698 let _lock = self.lock()?;
699 self.map = Self::load_media_db(&self.inventory_path)?;
700
701 for (uuid, entry) in self.map.iter_mut() {
702 if let Some(changer_name) = online_map.lookup_changer(uuid) {
703 entry.location = Some(MediaLocation::Online(changer_name.to_string()));
704 } else if let Some(MediaLocation::Online(ref changer_name)) = entry.location {
705 match online_map.online_map(changer_name) {
706 None => {
707 // no such changer device
708 entry.location = Some(MediaLocation::Offline);
709 }
710 Some(None) => {
711 // got no info - do nothing
712 }
713 Some(Some(_)) => {
714 // media changer changed
715 entry.location = Some(MediaLocation::Offline);
716 }
717 }
718 }
719 }
720
721 self.update_helpers();
722 self.replace_file()?;
723
724 Ok(())
725 }
726
727 }
728
729 // shell completion helper
730
731 /// List of known media uuids
732 pub fn complete_media_uuid(
733 _arg: &str,
734 _param: &HashMap<String, String>,
735 ) -> Vec<String> {
736
737 let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
738 Ok(inventory) => inventory,
739 Err(_) => return Vec::new(),
740 };
741
742 inventory.map.keys().map(|uuid| uuid.to_string()).collect()
743 }
744
745 /// List of known media sets
746 pub fn complete_media_set_uuid(
747 _arg: &str,
748 _param: &HashMap<String, String>,
749 ) -> Vec<String> {
750
751 let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
752 Ok(inventory) => inventory,
753 Err(_) => return Vec::new(),
754 };
755
756 inventory.map.values()
757 .filter_map(|entry| entry.id.media_set_label.as_ref())
758 .map(|set| set.uuid.to_string()).collect()
759 }
760
761 /// List of known media labels (barcodes)
762 pub fn complete_media_label_text(
763 _arg: &str,
764 _param: &HashMap<String, String>,
765 ) -> Vec<String> {
766
767 let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
768 Ok(inventory) => inventory,
769 Err(_) => return Vec::new(),
770 };
771
772 inventory.map.values().map(|entry| entry.id.label.label_text.clone()).collect()
773 }