]> git.proxmox.com Git - proxmox-backup.git/blob - src/tape/inventory.rs
clippy: fix for_kv_map
[proxmox-backup.git] / src / tape / inventory.rs
1 //! Backup media Inventory
2 //!
3 //! The Inventory persistently stores the list of known backup
4 //! media. A backup media is identified by its 'MediaId', which is the
5 //! MediaLabel/MediaSetLabel combination.
6
7 use std::collections::{HashMap, BTreeMap};
8 use std::path::{Path, PathBuf};
9
10 use anyhow::{bail, Error};
11 use serde::{Serialize, Deserialize};
12 use serde_json::json;
13
14 use proxmox::tools::{
15 Uuid,
16 fs::{
17 open_file_locked,
18 replace_file,
19 file_get_json,
20 CreateOptions,
21 },
22 };
23
24 use crate::{
25 tools::systemd::time::compute_next_event,
26 api2::types::{
27 MediaSetPolicy,
28 RetentionPolicy,
29 MediaStatus,
30 MediaLocation,
31 },
32 tape::{
33 TAPE_STATUS_DIR,
34 OnlineStatusMap,
35 MediaSet,
36 file_formats::{
37 MediaLabel,
38 MediaSetLabel,
39 },
40 },
41 };
42
43 /// Unique Media Identifier
44 ///
45 /// This combines the label and media set label.
46 #[derive(Debug,Serialize,Deserialize,Clone)]
47 pub struct MediaId {
48 pub label: MediaLabel,
49 #[serde(skip_serializing_if="Option::is_none")]
50 pub media_set_label: Option<MediaSetLabel>,
51 }
52
53
54 #[derive(Serialize,Deserialize)]
55 struct MediaStateEntry {
56 id: MediaId,
57 #[serde(skip_serializing_if="Option::is_none")]
58 location: Option<MediaLocation>,
59 #[serde(skip_serializing_if="Option::is_none")]
60 status: Option<MediaStatus>,
61 }
62
63 /// Media Inventory
64 pub struct Inventory {
65 map: BTreeMap<Uuid, MediaStateEntry>,
66
67 inventory_path: PathBuf,
68 lockfile_path: PathBuf,
69
70 // helpers
71 media_set_start_times: HashMap<Uuid, i64>
72 }
73
74 impl Inventory {
75
76 pub const MEDIA_INVENTORY_FILENAME: &'static str = "inventory.json";
77 pub const MEDIA_INVENTORY_LOCKFILE: &'static str = ".inventory.lck";
78
79 fn new(base_path: &Path) -> Self {
80
81 let mut inventory_path = base_path.to_owned();
82 inventory_path.push(Self::MEDIA_INVENTORY_FILENAME);
83
84 let mut lockfile_path = base_path.to_owned();
85 lockfile_path.push(Self::MEDIA_INVENTORY_LOCKFILE);
86
87 Self {
88 map: BTreeMap::new(),
89 media_set_start_times: HashMap::new(),
90 inventory_path,
91 lockfile_path,
92 }
93 }
94
95 pub fn load(base_path: &Path) -> Result<Self, Error> {
96 let mut me = Self::new(base_path);
97 me.reload()?;
98 Ok(me)
99 }
100
101 /// Reload the database
102 pub fn reload(&mut self) -> Result<(), Error> {
103 self.map = Self::load_media_db(&self.inventory_path)?;
104 self.update_helpers();
105 Ok(())
106 }
107
108 fn update_helpers(&mut self) {
109
110 // recompute media_set_start_times
111
112 let mut set_start_times = HashMap::new();
113
114 for entry in self.map.values() {
115 let set = match &entry.id.media_set_label {
116 None => continue,
117 Some(set) => set,
118 };
119 if set.seq_nr == 0 {
120 set_start_times.insert(set.uuid.clone(), set.ctime);
121 }
122 }
123
124 self.media_set_start_times = set_start_times;
125 }
126
127 /// Lock the database
128 pub fn lock(&self) -> Result<std::fs::File, Error> {
129 open_file_locked(&self.lockfile_path, std::time::Duration::new(10, 0), true)
130 }
131
132 fn load_media_db(path: &Path) -> Result<BTreeMap<Uuid, MediaStateEntry>, Error> {
133
134 let data = file_get_json(path, Some(json!([])))?;
135 let media_list: Vec<MediaStateEntry> = serde_json::from_value(data)?;
136
137 let mut map = BTreeMap::new();
138 for entry in media_list.into_iter() {
139 map.insert(entry.id.label.uuid.clone(), entry);
140 }
141
142 Ok(map)
143 }
144
145 fn replace_file(&self) -> Result<(), Error> {
146 let list: Vec<&MediaStateEntry> = self.map.values().collect();
147 let raw = serde_json::to_string_pretty(&serde_json::to_value(list)?)?;
148
149 let backup_user = crate::backup::backup_user()?;
150 let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640);
151 let options = CreateOptions::new()
152 .perm(mode)
153 .owner(backup_user.uid)
154 .group(backup_user.gid);
155
156 replace_file(&self.inventory_path, raw.as_bytes(), options)?;
157
158 Ok(())
159 }
160
161 /// Stores a single MediaID persistently
162 pub fn store(
163 &mut self,
164 mut media_id: MediaId,
165 clear_media_status: bool,
166 ) -> Result<(), Error> {
167 let _lock = self.lock()?;
168 self.map = Self::load_media_db(&self.inventory_path)?;
169
170 let uuid = media_id.label.uuid.clone();
171
172 if let Some(previous) = self.map.remove(&media_id.label.uuid) {
173 // do not overwrite unsaved pool assignments
174 if media_id.media_set_label.is_none() {
175 if let Some(ref set) = previous.id.media_set_label {
176 if set.uuid.as_ref() == [0u8;16] {
177 media_id.media_set_label = Some(set.clone());
178 }
179 }
180 }
181 let entry = MediaStateEntry {
182 id: media_id,
183 location: previous.location,
184 status: if clear_media_status {
185 None
186 } else {
187 previous.status
188 },
189 };
190 self.map.insert(uuid, entry);
191 } else {
192 let entry = MediaStateEntry { id: media_id, location: None, status: None };
193 self.map.insert(uuid, entry);
194 }
195
196 self.update_helpers();
197 self.replace_file()?;
198 Ok(())
199 }
200
201 /// Remove a single media persistently
202 pub fn remove_media(&mut self, uuid: &Uuid) -> Result<(), Error> {
203 let _lock = self.lock()?;
204 self.map = Self::load_media_db(&self.inventory_path)?;
205 self.map.remove(uuid);
206 self.update_helpers();
207 self.replace_file()?;
208 Ok(())
209 }
210
211 /// Lookup media
212 pub fn lookup_media(&self, uuid: &Uuid) -> Option<&MediaId> {
213 self.map.get(uuid).map(|entry| &entry.id)
214 }
215
216 /// find media by label_text
217 pub fn find_media_by_label_text(&self, label_text: &str) -> Option<&MediaId> {
218 self.map.values().find_map(|entry| {
219 if entry.id.label.label_text == label_text {
220 Some(&entry.id)
221 } else {
222 None
223 }
224 })
225 }
226
227 /// Lookup media pool
228 ///
229 /// Returns (pool, is_empty)
230 pub fn lookup_media_pool(&self, uuid: &Uuid) -> Option<(&str, bool)> {
231 match self.map.get(uuid) {
232 None => None,
233 Some(entry) => {
234 match entry.id.media_set_label {
235 None => None, // not assigned to any pool
236 Some(ref set) => {
237 let is_empty = set.uuid.as_ref() == [0u8;16];
238 Some((&set.pool, is_empty))
239 }
240 }
241 }
242 }
243 }
244
245 /// List all media assigned to the pool
246 pub fn list_pool_media(&self, pool: &str) -> Vec<MediaId> {
247 let mut list = Vec::new();
248
249 for entry in self.map.values() {
250 match entry.id.media_set_label {
251 None => continue, // not assigned to any pool
252 Some(ref set) => {
253 if set.pool != pool {
254 continue; // belong to another pool
255 }
256
257 if set.uuid.as_ref() == [0u8;16] { // should we do this??
258 list.push(MediaId {
259 label: entry.id.label.clone(),
260 media_set_label: None,
261 })
262 } else {
263 list.push(entry.id.clone());
264 }
265 }
266 }
267 }
268
269 list
270 }
271
272 /// List all used media
273 pub fn list_used_media(&self) -> Vec<MediaId> {
274 let mut list = Vec::new();
275
276 for entry in self.map.values() {
277 match entry.id.media_set_label {
278 None => continue, // not assigned to any pool
279 Some(ref set) => {
280 if set.uuid.as_ref() != [0u8;16] {
281 list.push(entry.id.clone());
282 }
283 }
284 }
285 }
286
287 list
288 }
289
290 /// List media not assigned to any pool
291 pub fn list_unassigned_media(&self) -> Vec<MediaId> {
292 self.map.values().filter_map(|entry|
293 if entry.id.media_set_label.is_none() {
294 Some(entry.id.clone())
295 } else {
296 None
297 }
298 ).collect()
299 }
300
301 pub fn media_set_start_time(&self, media_set_uuid: &Uuid) -> Option<i64> {
302 self.media_set_start_times.get(media_set_uuid).copied()
303 }
304
305 /// Lookup media set pool
306 pub fn lookup_media_set_pool(&self, media_set_uuid: &Uuid) -> Result<String, Error> {
307
308 let mut last_pool = None;
309
310 for entry in self.map.values() {
311 match entry.id.media_set_label {
312 None => continue,
313 Some(MediaSetLabel { ref uuid, .. }) => {
314 if uuid != media_set_uuid {
315 continue;
316 }
317 if let Some((pool, _)) = self.lookup_media_pool(&entry.id.label.uuid) {
318 if let Some(last_pool) = last_pool {
319 if last_pool != pool {
320 bail!("detected media set with inconsistent pool assignment - internal error");
321 }
322 } else {
323 last_pool = Some(pool);
324 }
325 }
326 }
327 }
328 }
329
330 match last_pool {
331 Some(pool) => Ok(pool.to_string()),
332 None => bail!("media set {} is incomplete - unable to lookup pool"),
333 }
334 }
335
336 /// Compute a single media sets
337 pub fn compute_media_set_members(&self, media_set_uuid: &Uuid) -> Result<MediaSet, Error> {
338
339 let mut set = MediaSet::with_data(media_set_uuid.clone(), Vec::new());
340
341 for entry in self.map.values() {
342 match entry.id.media_set_label {
343 None => continue,
344 Some(MediaSetLabel { seq_nr, ref uuid, .. }) => {
345 if uuid != media_set_uuid {
346 continue;
347 }
348 set.insert_media(entry.id.label.uuid.clone(), seq_nr)?;
349 }
350 }
351 }
352
353 Ok(set)
354 }
355
356 /// Compute all media sets
357 pub fn compute_media_set_list(&self) -> Result<HashMap<Uuid, MediaSet>, Error> {
358
359 let mut set_map: HashMap<Uuid, MediaSet> = HashMap::new();
360
361 for entry in self.map.values() {
362 match entry.id.media_set_label {
363 None => continue,
364 Some(MediaSetLabel { seq_nr, ref uuid, .. }) => {
365
366 let set = set_map.entry(uuid.clone()).or_insert_with(|| {
367 MediaSet::with_data(uuid.clone(), Vec::new())
368 });
369
370 set.insert_media(entry.id.label.uuid.clone(), seq_nr)?;
371 }
372 }
373 }
374
375 Ok(set_map)
376 }
377
378 /// Returns the latest media set for a pool
379 pub fn latest_media_set(&self, pool: &str) -> Option<Uuid> {
380
381 let mut last_set: Option<(Uuid, i64)> = None;
382
383 let set_list = self.map.values()
384 .filter_map(|entry| entry.id.media_set_label.as_ref())
385 .filter(|set| &set.pool == &pool && set.uuid.as_ref() != [0u8;16]);
386
387 for set in set_list {
388 match last_set {
389 None => {
390 last_set = Some((set.uuid.clone(), set.ctime));
391 }
392 Some((_, last_ctime)) => {
393 if set.ctime > last_ctime {
394 last_set = Some((set.uuid.clone(), set.ctime));
395 }
396 }
397 }
398 }
399
400 let (uuid, ctime) = match last_set {
401 None => return None,
402 Some((uuid, ctime)) => (uuid, ctime),
403 };
404
405 // consistency check - must be the only set with that ctime
406 let set_list = self.map.values()
407 .filter_map(|entry| entry.id.media_set_label.as_ref())
408 .filter(|set| &set.pool == &pool && set.uuid.as_ref() != [0u8;16]);
409
410 for set in set_list {
411 if set.uuid != uuid && set.ctime >= ctime { // should not happen
412 eprintln!("latest_media_set: found set with equal ctime ({}, {})", set.uuid, uuid);
413 return None;
414 }
415 }
416
417 Some(uuid)
418 }
419
420 // Test if there is a media set (in the same pool) newer than this one.
421 // Return the ctime of the nearest media set
422 fn media_set_next_start_time(&self, media_set_uuid: &Uuid) -> Option<i64> {
423
424 let (pool, ctime) = match self.map.values()
425 .filter_map(|entry| entry.id.media_set_label.as_ref())
426 .find_map(|set| {
427 if &set.uuid == media_set_uuid {
428 Some((set.pool.clone(), set.ctime))
429 } else {
430 None
431 }
432 }) {
433 Some((pool, ctime)) => (pool, ctime),
434 None => return None,
435 };
436
437 let set_list = self.map.values()
438 .filter_map(|entry| entry.id.media_set_label.as_ref())
439 .filter(|set| (&set.uuid != media_set_uuid) && (&set.pool == &pool));
440
441 let mut next_ctime = None;
442
443 for set in set_list {
444 if set.ctime > ctime {
445 match next_ctime {
446 None => {
447 next_ctime = Some(set.ctime);
448 }
449 Some(last_next_ctime) => {
450 if set.ctime < last_next_ctime {
451 next_ctime = Some(set.ctime);
452 }
453 }
454 }
455 }
456 }
457
458 next_ctime
459 }
460
461 pub fn media_expire_time(
462 &self,
463 media: &MediaId,
464 media_set_policy: &MediaSetPolicy,
465 retention_policy: &RetentionPolicy,
466 ) -> i64 {
467
468 if let RetentionPolicy::KeepForever = retention_policy {
469 return i64::MAX;
470 }
471
472 let set = match media.media_set_label {
473 None => return i64::MAX,
474 Some(ref set) => set,
475 };
476
477 let set_start_time = match self.media_set_start_time(&set.uuid) {
478 None => {
479 // missing information, use ctime from this
480 // set (always greater than ctime from seq_nr 0)
481 set.ctime
482 }
483 Some(time) => time,
484 };
485
486 let max_use_time = match media_set_policy {
487 MediaSetPolicy::ContinueCurrent => {
488 match self.media_set_next_start_time(&set.uuid) {
489 Some(next_start_time) => next_start_time,
490 None => return i64::MAX,
491 }
492 }
493 MediaSetPolicy::AlwaysCreate => {
494 set_start_time + 1
495 }
496 MediaSetPolicy::CreateAt(ref event) => {
497 match compute_next_event(event, set_start_time, false) {
498 Ok(Some(next)) => next,
499 Ok(None) | Err(_) => return i64::MAX,
500 }
501 }
502 };
503
504 match retention_policy {
505 RetentionPolicy::KeepForever => i64::MAX,
506 RetentionPolicy::OverwriteAlways => max_use_time,
507 RetentionPolicy::ProtectFor(time_span) => {
508 let seconds = f64::from(time_span.clone()) as i64;
509 max_use_time + seconds
510 }
511 }
512 }
513
514 /// Generate a human readable name for the media set
515 ///
516 /// The template can include strftime time format specifications.
517 pub fn generate_media_set_name(
518 &self,
519 media_set_uuid: &Uuid,
520 template: Option<String>,
521 ) -> Result<String, Error> {
522
523 if let Some(ctime) = self.media_set_start_time(media_set_uuid) {
524 let mut template = template.unwrap_or_else(|| String::from("%c"));
525 template = template.replace("%id%", &media_set_uuid.to_string());
526 proxmox::tools::time::strftime_local(&template, ctime)
527 } else {
528 // We don't know the set start time, so we cannot use the template
529 Ok(media_set_uuid.to_string())
530 }
531 }
532
533 // Helpers to simplify testing
534
535 /// Genreate and insert a new free tape (test helper)
536 pub fn generate_free_tape(&mut self, label_text: &str, ctime: i64) -> Uuid {
537
538 let label = MediaLabel {
539 label_text: label_text.to_string(),
540 uuid: Uuid::generate(),
541 ctime,
542 };
543 let uuid = label.uuid.clone();
544
545 self.store(MediaId { label, media_set_label: None }, false).unwrap();
546
547 uuid
548 }
549
550 /// Genreate and insert a new tape assigned to a specific pool
551 /// (test helper)
552 pub fn generate_assigned_tape(
553 &mut self,
554 label_text: &str,
555 pool: &str,
556 ctime: i64,
557 ) -> Uuid {
558
559 let label = MediaLabel {
560 label_text: label_text.to_string(),
561 uuid: Uuid::generate(),
562 ctime,
563 };
564
565 let uuid = label.uuid.clone();
566
567 let set = MediaSetLabel::with_data(pool, [0u8; 16].into(), 0, ctime, None);
568
569 self.store(MediaId { label, media_set_label: Some(set) }, false).unwrap();
570
571 uuid
572 }
573
574 /// Genreate and insert a used tape (test helper)
575 pub fn generate_used_tape(
576 &mut self,
577 label_text: &str,
578 set: MediaSetLabel,
579 ctime: i64,
580 ) -> Uuid {
581 let label = MediaLabel {
582 label_text: label_text.to_string(),
583 uuid: Uuid::generate(),
584 ctime,
585 };
586 let uuid = label.uuid.clone();
587
588 self.store(MediaId { label, media_set_label: Some(set) }, false).unwrap();
589
590 uuid
591 }
592 }
593
594 // Status/location handling
595 impl Inventory {
596
597 /// Returns status and location with reasonable defaults.
598 ///
599 /// Default status is 'MediaStatus::Unknown'.
600 /// Default location is 'MediaLocation::Offline'.
601 pub fn status_and_location(&self, uuid: &Uuid) -> (MediaStatus, MediaLocation) {
602
603 match self.map.get(uuid) {
604 None => {
605 // no info stored - assume media is writable/offline
606 (MediaStatus::Unknown, MediaLocation::Offline)
607 }
608 Some(entry) => {
609 let location = entry.location.clone().unwrap_or(MediaLocation::Offline);
610 let status = entry.status.unwrap_or(MediaStatus::Unknown);
611 (status, location)
612 }
613 }
614 }
615
616 // Lock database, reload database, set status, store database
617 fn set_media_status(&mut self, uuid: &Uuid, status: Option<MediaStatus>) -> Result<(), Error> {
618 let _lock = self.lock()?;
619 self.map = Self::load_media_db(&self.inventory_path)?;
620 if let Some(entry) = self.map.get_mut(uuid) {
621 entry.status = status;
622 self.update_helpers();
623 self.replace_file()?;
624 Ok(())
625 } else {
626 bail!("no such media '{}'", uuid);
627 }
628 }
629
630 /// Lock database, reload database, set status to Full, store database
631 pub fn set_media_status_full(&mut self, uuid: &Uuid) -> Result<(), Error> {
632 self.set_media_status(uuid, Some(MediaStatus::Full))
633 }
634
635 /// Lock database, reload database, set status to Damaged, store database
636 pub fn set_media_status_damaged(&mut self, uuid: &Uuid) -> Result<(), Error> {
637 self.set_media_status(uuid, Some(MediaStatus::Damaged))
638 }
639
640 /// Lock database, reload database, set status to None, store database
641 pub fn clear_media_status(&mut self, uuid: &Uuid) -> Result<(), Error> {
642 self.set_media_status(uuid, None)
643 }
644
645 // Lock database, reload database, set location, store database
646 fn set_media_location(&mut self, uuid: &Uuid, location: Option<MediaLocation>) -> Result<(), Error> {
647 let _lock = self.lock()?;
648 self.map = Self::load_media_db(&self.inventory_path)?;
649 if let Some(entry) = self.map.get_mut(uuid) {
650 entry.location = location;
651 self.update_helpers();
652 self.replace_file()?;
653 Ok(())
654 } else {
655 bail!("no such media '{}'", uuid);
656 }
657 }
658
659 /// Lock database, reload database, set location to vault, store database
660 pub fn set_media_location_vault(&mut self, uuid: &Uuid, vault: &str) -> Result<(), Error> {
661 self.set_media_location(uuid, Some(MediaLocation::Vault(vault.to_string())))
662 }
663
664 /// Lock database, reload database, set location to offline, store database
665 pub fn set_media_location_offline(&mut self, uuid: &Uuid) -> Result<(), Error> {
666 self.set_media_location(uuid, Some(MediaLocation::Offline))
667 }
668
669 /// Update online status
670 pub fn update_online_status(&mut self, online_map: &OnlineStatusMap) -> Result<(), Error> {
671 let _lock = self.lock()?;
672 self.map = Self::load_media_db(&self.inventory_path)?;
673
674 for (uuid, entry) in self.map.iter_mut() {
675 if let Some(changer_name) = online_map.lookup_changer(uuid) {
676 entry.location = Some(MediaLocation::Online(changer_name.to_string()));
677 } else if let Some(MediaLocation::Online(ref changer_name)) = entry.location {
678 match online_map.online_map(changer_name) {
679 None => {
680 // no such changer device
681 entry.location = Some(MediaLocation::Offline);
682 }
683 Some(None) => {
684 // got no info - do nothing
685 }
686 Some(Some(_)) => {
687 // media changer changed
688 entry.location = Some(MediaLocation::Offline);
689 }
690 }
691 }
692 }
693
694 self.update_helpers();
695 self.replace_file()?;
696
697 Ok(())
698 }
699
700 }
701
702 // shell completion helper
703
704 /// List of known media uuids
705 pub fn complete_media_uuid(
706 _arg: &str,
707 _param: &HashMap<String, String>,
708 ) -> Vec<String> {
709
710 let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
711 Ok(inventory) => inventory,
712 Err(_) => return Vec::new(),
713 };
714
715 inventory.map.keys().map(|uuid| uuid.to_string()).collect()
716 }
717
718 /// List of known media sets
719 pub fn complete_media_set_uuid(
720 _arg: &str,
721 _param: &HashMap<String, String>,
722 ) -> Vec<String> {
723
724 let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
725 Ok(inventory) => inventory,
726 Err(_) => return Vec::new(),
727 };
728
729 inventory.map.values()
730 .filter_map(|entry| entry.id.media_set_label.as_ref())
731 .map(|set| set.uuid.to_string()).collect()
732 }
733
734 /// List of known media labels (barcodes)
735 pub fn complete_media_label_text(
736 _arg: &str,
737 _param: &HashMap<String, String>,
738 ) -> Vec<String> {
739
740 let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
741 Ok(inventory) => inventory,
742 Err(_) => return Vec::new(),
743 };
744
745 inventory.map.values().map(|entry| entry.id.label.label_text.clone()).collect()
746 }