src/tape/inventory.rs

   1 //! Backup media Inventory
   2 //!
   3 //! The Inventory persistently stores the list of known backup
   4 //! media. A backup media is identified by its 'MediaId', which is the
   5 //! DriveLabel/MediaSetLabel combination.
   6
   7 use std::collections::{HashMap, BTreeMap};
   8 use std::path::{Path, PathBuf};
   9
  10 use anyhow::{bail, Error};
  11 use serde::{Serialize, Deserialize};
  12 use serde_json::json;
  13
  14 use proxmox::tools::{
  15     Uuid,
  16     fs::{
  17         open_file_locked,
  18         replace_file,
  19         file_get_json,
  20         CreateOptions,
  21     },
  22 };
  23
  24 use crate::{
  25     tools::systemd::time::compute_next_event,
  26     api2::types::{
  27         MediaSetPolicy,
  28         RetentionPolicy,
  29     },
  30     tape::{
  31         TAPE_STATUS_DIR,
  32         MediaLabelInfo,
  33         file_formats::{
  34             DriveLabel,
  35             MediaSetLabel,
  36         },
  37     },
  38 };
  39
  40 /// Unique Media Identifier
  41 ///
  42 /// This combines the label and media set label.
  43 #[derive(Debug,Serialize,Deserialize,Clone)]
  44 pub struct MediaId {
  45     pub label: DriveLabel,
  46     #[serde(skip_serializing_if="Option::is_none")]
  47     pub media_set_label: Option<MediaSetLabel>,
  48 }
  49
  50 impl From<MediaLabelInfo> for MediaId {
  51     fn from(info: MediaLabelInfo) -> Self {
  52         Self {
  53             label: info.label.clone(),
  54             media_set_label: info.media_set_label.map(|(l, _)| l),
  55         }
  56     }
  57 }
  58
  59
  60 /// Media Set
  61 ///
  62 /// A List of backup media
  63 #[derive(Debug, Serialize, Deserialize)]
  64 pub struct MediaSet {
  65     /// Unique media set ID
  66     uuid: Uuid,
  67     /// List of BackupMedia
  68     media_list: Vec<Option<Uuid>>,
  69 }
  70
  71 impl MediaSet {
  72
  73     pub const MEDIA_SET_MAX_SEQ_NR: u64 = 100;
  74
  75     pub fn new() -> Self {
  76         let uuid = Uuid::generate();
  77         Self {
  78             uuid,
  79             media_list: Vec::new(),
  80         }
  81     }
  82
  83     pub fn with_data(uuid: Uuid, media_list: Vec<Option<Uuid>>) -> Self {
  84         Self { uuid, media_list }
  85     }
  86
  87     pub fn uuid(&self) -> &Uuid {
  88         &self.uuid
  89     }
  90
  91     pub fn media_list(&self) -> &[Option<Uuid>] {
  92         &self.media_list
  93     }
  94
  95     pub fn add_media(&mut self, uuid: Uuid) {
  96         self.media_list.push(Some(uuid));
  97     }
  98
  99     pub fn insert_media(&mut self, uuid: Uuid, seq_nr: u64) -> Result<(), Error> {
 100         if seq_nr > Self::MEDIA_SET_MAX_SEQ_NR {
 101             bail!("media set sequence number to large in media set {} ({} > {})",
 102                   self.uuid.to_string(), seq_nr, Self::MEDIA_SET_MAX_SEQ_NR);
 103         }
 104         let seq_nr = seq_nr as usize;
 105         if self.media_list.len() > seq_nr {
 106             if self.media_list[seq_nr].is_some() {
 107                 bail!("found duplicate squence number in media set '{}/{}'",
 108                       self.uuid.to_string(), seq_nr);
 109             }
 110         } else {
 111             self.media_list.resize(seq_nr + 1, None);
 112         }
 113         self.media_list[seq_nr] = Some(uuid);
 114         Ok(())
 115     }
 116
 117     pub fn last_media_uuid(&self) -> Option<&Uuid> {
 118         match self.media_list.last() {
 119             None => None,
 120             Some(None) => None,
 121             Some(Some(ref last_uuid)) => Some(last_uuid),
 122         }
 123     }
 124
 125     pub fn is_last_media(&self, uuid: &Uuid) -> bool {
 126         match self.media_list.last() {
 127             None => false,
 128             Some(None) => false,
 129             Some(Some(last_uuid)) => uuid == last_uuid,
 130         }
 131     }
 132 }
 133
 134 /// Media Inventory
 135 pub struct Inventory {
 136     map: BTreeMap<Uuid, MediaId>,
 137
 138     inventory_path: PathBuf,
 139     lockfile_path: PathBuf,
 140
 141     // helpers
 142     media_set_start_times: HashMap<Uuid, i64>
 143 }
 144
 145 impl Inventory {
 146
 147     pub const MEDIA_INVENTORY_FILENAME: &'static str = "inventory.json";
 148     pub const MEDIA_INVENTORY_LOCKFILE: &'static str = ".inventory.lck";
 149
 150     fn new(base_path: &Path) -> Self {
 151
 152         let mut inventory_path = base_path.to_owned();
 153         inventory_path.push(Self::MEDIA_INVENTORY_FILENAME);
 154
 155         let mut lockfile_path = base_path.to_owned();
 156         lockfile_path.push(Self::MEDIA_INVENTORY_LOCKFILE);
 157
 158         Self {
 159             map: BTreeMap::new(),
 160             media_set_start_times: HashMap::new(),
 161             inventory_path,
 162             lockfile_path,
 163         }
 164     }
 165
 166     pub fn load(base_path: &Path) -> Result<Self, Error> {
 167         let mut me = Self::new(base_path);
 168         me.reload()?;
 169         Ok(me)
 170     }
 171
 172     /// Reload the database
 173     pub fn reload(&mut self) -> Result<(), Error> {
 174         self.map = Self::load_media_db(&self.inventory_path)?;
 175         self.update_helpers();
 176         Ok(())
 177     }
 178
 179     fn update_helpers(&mut self) {
 180
 181         // recompute media_set_start_times
 182
 183         let mut set_start_times = HashMap::new();
 184
 185         for media in self.map.values() {
 186             let set = match &media.media_set_label {
 187                 None => continue,
 188                 Some(set) => set,
 189             };
 190             if set.seq_nr == 0 {
 191                 set_start_times.insert(set.uuid.clone(), set.ctime);
 192             }
 193         }
 194
 195         self.media_set_start_times = set_start_times;
 196     }
 197
 198     /// Lock the database
 199     pub fn lock(&self) -> Result<std::fs::File, Error> {
 200         open_file_locked(&self.lockfile_path, std::time::Duration::new(10, 0), true)
 201     }
 202
 203     fn load_media_db(path: &Path) -> Result<BTreeMap<Uuid, MediaId>, Error> {
 204
 205         let data = file_get_json(path, Some(json!([])))?;
 206         let media_list: Vec<MediaId> = serde_json::from_value(data)?;
 207
 208         let mut map = BTreeMap::new();
 209         for item in media_list.into_iter() {
 210             map.insert(item.label.uuid.clone(), item);
 211         }
 212
 213         Ok(map)
 214     }
 215
 216     fn replace_file(&self) -> Result<(), Error> {
 217         let list: Vec<&MediaId> = self.map.values().collect();
 218         let raw = serde_json::to_string_pretty(&serde_json::to_value(list)?)?;
 219
 220         let backup_user = crate::backup::backup_user()?;
 221         let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640);
 222         let options = CreateOptions::new()
 223             .perm(mode)
 224             .owner(backup_user.uid)
 225             .group(backup_user.gid);
 226
 227         replace_file(&self.inventory_path, raw.as_bytes(), options)?;
 228
 229         Ok(())
 230     }
 231
 232     /// Stores a single MediaID persistently
 233     pub fn store(&mut self, mut media_id: MediaId) -> Result<(), Error> {
 234         let _lock = self.lock()?;
 235         self.map = Self::load_media_db(&self.inventory_path)?;
 236
 237         // do not overwrite unsaved pool assignments
 238         if media_id.media_set_label.is_none() {
 239             if let Some(previous) = self.map.get(&media_id.label.uuid) {
 240                 if let Some(ref set) = previous.media_set_label {
 241                     if set.uuid.as_ref() == [0u8;16] {
 242                         media_id.media_set_label = Some(set.clone());
 243                     }
 244                 }
 245             }
 246         }
 247
 248         self.map.insert(media_id.label.uuid.clone(), media_id);
 249         self.update_helpers();
 250         self.replace_file()?;
 251         Ok(())
 252     }
 253
 254     /// Remove a single media persistently
 255     pub fn remove_media(&mut self, uuid: &Uuid)  -> Result<(), Error> {
 256         let _lock = self.lock()?;
 257         self.map = Self::load_media_db(&self.inventory_path)?;
 258         self.map.remove(uuid);
 259         self.update_helpers();
 260         self.replace_file()?;
 261         Ok(())
 262     }
 263
 264     /// Lookup media
 265     pub fn lookup_media(&self, uuid: &Uuid) -> Option<&MediaId> {
 266         self.map.get(uuid)
 267     }
 268
 269     /// find media by changer_id
 270     pub fn find_media_by_changer_id(&self, changer_id: &str) -> Option<&MediaId> {
 271         for (_uuid, media_id) in &self.map {
 272             if media_id.label.changer_id == changer_id {
 273                 return Some(media_id);
 274             }
 275         }
 276         None
 277     }
 278
 279     /// Lookup media pool
 280     ///
 281     /// Returns (pool, is_empty)
 282     pub fn lookup_media_pool(&self, uuid: &Uuid) -> Option<(&str, bool)> {
 283         match self.map.get(uuid) {
 284             None => None,
 285             Some(media_id) => {
 286                 match media_id.media_set_label {
 287                     None => None, // not assigned to any pool
 288                     Some(ref set) => {
 289                         let is_empty = set.uuid.as_ref() == [0u8;16];
 290                         Some((&set.pool, is_empty))
 291                     }
 292                 }
 293             }
 294         }
 295     }
 296
 297     /// List all media assigned to the pool
 298     pub fn list_pool_media(&self, pool: &str) -> Vec<MediaId> {
 299         let mut list = Vec::new();
 300
 301         for (_uuid, media_id) in &self.map {
 302             match media_id.media_set_label {
 303                 None => continue, // not assigned to any pool
 304                 Some(ref set) => {
 305                     if set.pool != pool {
 306                         continue; // belong to another pool
 307                     }
 308
 309                     if set.uuid.as_ref() == [0u8;16] { // should we do this??
 310                         list.push(MediaId {
 311                             label: media_id.label.clone(),
 312                             media_set_label: None,
 313                         })
 314                     } else {
 315                         list.push(media_id.clone());
 316                     }
 317                 }
 318             }
 319
 320         }
 321
 322         list
 323     }
 324
 325     /// List all used media
 326     pub fn list_used_media(&self) -> Vec<MediaId> {
 327         let mut list = Vec::new();
 328
 329         for (_uuid, media_id) in &self.map {
 330             match media_id.media_set_label {
 331                 None => continue, // not assigned to any pool
 332                 Some(ref set) => {
 333                     if set.uuid.as_ref() != [0u8;16] {
 334                         list.push(media_id.clone());
 335                     }
 336                 }
 337             }
 338         }
 339
 340         list
 341     }
 342
 343     /// List media not assigned to any pool
 344     pub fn list_unassigned_media(&self) -> Vec<MediaId> {
 345         let mut list = Vec::new();
 346
 347         for (_uuid, media_id) in &self.map {
 348             if media_id.media_set_label.is_none() {
 349                 list.push(media_id.clone());
 350             }
 351         }
 352
 353         list
 354     }
 355
 356     pub fn media_set_start_time(&self, media_set_uuid: &Uuid) -> Option<i64> {
 357         self.media_set_start_times.get(media_set_uuid).map(|t| *t)
 358     }
 359
 360     /// Compute a single media sets
 361     pub fn compute_media_set_members(&self, media_set_uuid: &Uuid) -> Result<MediaSet, Error> {
 362
 363         let mut set = MediaSet::with_data(media_set_uuid.clone(), Vec::new());
 364
 365         for media in self.map.values() {
 366             match media.media_set_label {
 367                 None => continue,
 368                 Some(MediaSetLabel { seq_nr, ref uuid, .. }) => {
 369                     if  uuid != media_set_uuid {
 370                         continue;
 371                     }
 372                     set.insert_media(media.label.uuid.clone(), seq_nr)?;
 373                 }
 374             }
 375         }
 376
 377         Ok(set)
 378     }
 379
 380     /// Compute all media sets
 381     pub fn compute_media_set_list(&self) -> Result<HashMap<Uuid, MediaSet>, Error> {
 382
 383         let mut set_map: HashMap<Uuid, MediaSet> = HashMap::new();
 384
 385         for media in self.map.values() {
 386             match media.media_set_label {
 387                 None => continue,
 388                 Some(MediaSetLabel { seq_nr, ref uuid, .. }) => {
 389
 390                     let set = set_map.entry(uuid.clone()).or_insert_with(|| {
 391                         MediaSet::with_data(uuid.clone(), Vec::new())
 392                     });
 393
 394                     set.insert_media(media.label.uuid.clone(), seq_nr)?;
 395                 }
 396             }
 397         }
 398
 399         Ok(set_map)
 400     }
 401
 402     /// Returns the latest media set for a pool
 403     pub fn latest_media_set(&self, pool: &str) -> Option<Uuid> {
 404
 405         let mut last_set: Option<(Uuid, i64)> = None;
 406
 407         let set_list = self.map.values()
 408             .filter_map(|media| media.media_set_label.as_ref())
 409             .filter(|set| &set.pool == &pool && set.uuid.as_ref() != [0u8;16]);
 410
 411         for set in set_list {
 412             match last_set {
 413                 None => {
 414                     last_set = Some((set.uuid.clone(), set.ctime));
 415                 }
 416                 Some((_, last_ctime)) => {
 417                     if set.ctime > last_ctime {
 418                         last_set = Some((set.uuid.clone(), set.ctime));
 419                     }
 420                 }
 421             }
 422         }
 423
 424         let (uuid, ctime) = match last_set {
 425             None => return None,
 426             Some((uuid, ctime)) => (uuid, ctime),
 427         };
 428
 429         // consistency check - must be the only set with that ctime
 430         let set_list = self.map.values()
 431             .filter_map(|media| media.media_set_label.as_ref())
 432             .filter(|set| &set.pool == &pool && set.uuid.as_ref() != [0u8;16]);
 433
 434         for set in set_list {
 435             if set.uuid != uuid && set.ctime >= ctime { // should not happen
 436                 eprintln!("latest_media_set: found set with equal ctime ({}, {})", set.uuid, uuid);
 437                 return None;
 438             }
 439         }
 440
 441         Some(uuid)
 442     }
 443
 444     // Test if there is a media set (in the same pool) newer than this one.
 445     // Return the ctime of the nearest media set
 446     fn media_set_next_start_time(&self, media_set_uuid: &Uuid) -> Option<i64> {
 447
 448         let (pool, ctime) = match self.map.values()
 449             .filter_map(|media| media.media_set_label.as_ref())
 450             .find_map(|set| {
 451                 if &set.uuid == media_set_uuid {
 452                     Some((set.pool.clone(), set.ctime))
 453                 } else {
 454                     None
 455                 }
 456             }) {
 457                 Some((pool, ctime)) => (pool, ctime),
 458                 None => return None,
 459             };
 460
 461         let set_list = self.map.values()
 462             .filter_map(|media| media.media_set_label.as_ref())
 463             .filter(|set| (&set.uuid != media_set_uuid) && (&set.pool == &pool));
 464
 465         let mut next_ctime = None;
 466
 467         for set in set_list {
 468             if set.ctime > ctime {
 469                 match next_ctime {
 470                     None => {
 471                         next_ctime = Some(set.ctime);
 472                     }
 473                     Some(last_next_ctime) => {
 474                         if set.ctime < last_next_ctime {
 475                             next_ctime = Some(set.ctime);
 476                         }
 477                     }
 478                 }
 479             }
 480         }
 481
 482         next_ctime
 483     }
 484
 485     pub fn media_expire_time(
 486         &self,
 487         media: &MediaId,
 488         media_set_policy: &MediaSetPolicy,
 489         retention_policy: &RetentionPolicy,
 490     ) -> i64 {
 491
 492         if let RetentionPolicy::KeepForever = retention_policy {
 493             return i64::MAX;
 494         }
 495
 496         let set = match media.media_set_label {
 497             None => return i64::MAX,
 498             Some(ref set) => set,
 499         };
 500
 501         let set_start_time = match self.media_set_start_time(&set.uuid) {
 502             None => {
 503                 // missing information, use ctime from this
 504                 // set (always greater than ctime from seq_nr 0)
 505                 set.ctime
 506             }
 507             Some(time) => time,
 508         };
 509
 510         let max_use_time = match media_set_policy {
 511             MediaSetPolicy::ContinueCurrent => {
 512                 match self.media_set_next_start_time(&set.uuid) {
 513                     Some(next_start_time) => next_start_time,
 514                     None => return i64::MAX,
 515                 }
 516             }
 517             MediaSetPolicy::AlwaysCreate => {
 518                 set_start_time + 1
 519             }
 520             MediaSetPolicy::CreateAt(ref event) => {
 521                 match compute_next_event(event, set_start_time, false) {
 522                     Ok(Some(next)) => next,
 523                     Ok(None) | Err(_) => return i64::MAX,
 524                 }
 525             }
 526         };
 527
 528         match retention_policy {
 529             RetentionPolicy::KeepForever => i64::MAX,
 530             RetentionPolicy::OverwriteAlways => max_use_time,
 531             RetentionPolicy::ProtectFor(time_span) => {
 532                 let seconds = f64::from(time_span.clone()) as i64;
 533                 max_use_time + seconds
 534             }
 535         }
 536     }
 537
 538     /// Generate a human readable name for the media set
 539     ///
 540     /// The template can include strftime time format specifications.
 541     pub fn generate_media_set_name(
 542         &self,
 543         media_set_uuid: &Uuid,
 544         template: Option<String>,
 545     ) -> Result<String, Error> {
 546
 547         if let Some(ctime) = self.media_set_start_time(media_set_uuid) {
 548             let mut template = template.unwrap_or(String::from("%id%"));
 549             template = template.replace("%id%", &media_set_uuid.to_string());
 550             proxmox::tools::time::strftime_local(&template, ctime)
 551         } else {
 552             // We don't know the set start time, so we cannot use the template
 553             Ok(media_set_uuid.to_string())
 554         }
 555     }
 556
 557     // Helpers to simplify testing
 558
 559     /// Genreate and insert a new free tape (test helper)
 560     pub fn generate_free_tape(&mut self, changer_id: &str, ctime: i64) -> Uuid {
 561
 562         let label = DriveLabel {
 563             changer_id: changer_id.to_string(),
 564             uuid: Uuid::generate(),
 565             ctime,
 566         };
 567         let uuid = label.uuid.clone();
 568
 569         self.store(MediaId { label, media_set_label: None }).unwrap();
 570
 571         uuid
 572     }
 573
 574     /// Genreate and insert a new tape assigned to a specific pool
 575     /// (test helper)
 576     pub fn generate_assigned_tape(
 577         &mut self,
 578         changer_id: &str,
 579         pool: &str,
 580         ctime: i64,
 581     ) -> Uuid {
 582
 583         let label = DriveLabel {
 584             changer_id: changer_id.to_string(),
 585             uuid: Uuid::generate(),
 586             ctime,
 587         };
 588
 589         let uuid = label.uuid.clone();
 590
 591         let set = MediaSetLabel::with_data(pool, [0u8; 16].into(), 0, ctime);
 592
 593         self.store(MediaId { label, media_set_label: Some(set) }).unwrap();
 594
 595         uuid
 596     }
 597
 598     /// Genreate and insert a used tape (test helper)
 599     pub fn generate_used_tape(
 600         &mut self,
 601         changer_id: &str,
 602         set: MediaSetLabel,
 603         ctime: i64,
 604     ) -> Uuid {
 605         let label = DriveLabel {
 606             changer_id: changer_id.to_string(),
 607             uuid: Uuid::generate(),
 608             ctime,
 609         };
 610         let uuid = label.uuid.clone();
 611
 612         self.store(MediaId { label, media_set_label: Some(set) }).unwrap();
 613
 614         uuid
 615     }
 616 }
 617
 618 // shell completion helper
 619
 620 /// List of known media uuids
 621 pub fn complete_media_uuid(
 622     _arg: &str,
 623     _param: &HashMap<String, String>,
 624 ) -> Vec<String> {
 625
 626     let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
 627         Ok(inventory) => inventory,
 628         Err(_) => return Vec::new(),
 629     };
 630
 631     inventory.map.keys().map(|uuid| uuid.to_string()).collect()
 632 }
 633
 634 /// List of known media sets
 635 pub fn complete_media_set_uuid(
 636     _arg: &str,
 637     _param: &HashMap<String, String>,
 638 ) -> Vec<String> {
 639
 640     let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
 641         Ok(inventory) => inventory,
 642         Err(_) => return Vec::new(),
 643     };
 644
 645     inventory.map.values()
 646         .filter_map(|media| media.media_set_label.as_ref())
 647         .map(|set| set.uuid.to_string()).collect()
 648 }
 649
 650 /// List of known media labels (barcodes)
 651 pub fn complete_media_changer_id(
 652     _arg: &str,
 653     _param: &HashMap<String, String>,
 654 ) -> Vec<String> {
 655
 656     let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
 657         Ok(inventory) => inventory,
 658         Err(_) => return Vec::new(),
 659     };
 660
 661     inventory.map.values().map(|media| media.label.changer_id.clone()).collect()
 662 }