src/tape/inventory.rs

   1 //! Backup media Inventory
   2 //!
   3 //! The Inventory persistently stores the list of known backup
   4 //! media. A backup media is identified by its 'MediaId', which is the
   5 //! DriveLabel/MediaSetLabel combination.
   6
   7 use std::collections::{HashMap, BTreeMap};
   8 use std::path::{Path, PathBuf};
   9
  10 use anyhow::{bail, Error};
  11 use serde::{Serialize, Deserialize};
  12 use serde_json::json;
  13
  14 use proxmox::tools::{
  15     Uuid,
  16     fs::{
  17         open_file_locked,
  18         replace_file,
  19         file_get_json,
  20         CreateOptions,
  21     },
  22 };
  23
  24 use crate::{
  25     tools::systemd::time::compute_next_event,
  26     api2::types::{
  27         MediaSetPolicy,
  28         RetentionPolicy,
  29     },
  30     tape::{
  31         TAPE_STATUS_DIR,
  32         file_formats::{
  33             DriveLabel,
  34             MediaSetLabel,
  35         },
  36     },
  37 };
  38
  39 /// Unique Media Identifier
  40 ///
  41 /// This combines the label and media set label.
  42 #[derive(Debug,Serialize,Deserialize,Clone)]
  43 pub struct MediaId {
  44     pub label: DriveLabel,
  45     #[serde(skip_serializing_if="Option::is_none")]
  46     pub media_set_label: Option<MediaSetLabel>,
  47 }
  48
  49 /// Media Set
  50 ///
  51 /// A List of backup media
  52 #[derive(Debug, Serialize, Deserialize)]
  53 pub struct MediaSet {
  54     /// Unique media set ID
  55     uuid: Uuid,
  56     /// List of BackupMedia
  57     media_list: Vec<Option<Uuid>>,
  58 }
  59
  60 impl MediaSet {
  61
  62     pub const MEDIA_SET_MAX_SEQ_NR: u64 = 100;
  63
  64     pub fn new() -> Self {
  65         let uuid = Uuid::generate();
  66         Self {
  67             uuid,
  68             media_list: Vec::new(),
  69         }
  70     }
  71
  72     pub fn with_data(uuid: Uuid, media_list: Vec<Option<Uuid>>) -> Self {
  73         Self { uuid, media_list }
  74     }
  75
  76     pub fn uuid(&self) -> &Uuid {
  77         &self.uuid
  78     }
  79
  80     pub fn media_list(&self) -> &[Option<Uuid>] {
  81         &self.media_list
  82     }
  83
  84     pub fn add_media(&mut self, uuid: Uuid) {
  85         self.media_list.push(Some(uuid));
  86     }
  87
  88     pub fn insert_media(&mut self, uuid: Uuid, seq_nr: u64) -> Result<(), Error> {
  89         if seq_nr > Self::MEDIA_SET_MAX_SEQ_NR {
  90             bail!("media set sequence number to large in media set {} ({} > {})",
  91                   self.uuid.to_string(), seq_nr, Self::MEDIA_SET_MAX_SEQ_NR);
  92         }
  93         let seq_nr = seq_nr as usize;
  94         if self.media_list.len() > seq_nr {
  95             if self.media_list[seq_nr].is_some() {
  96                 bail!("found duplicate squence number in media set '{}/{}'",
  97                       self.uuid.to_string(), seq_nr);
  98             }
  99         } else {
 100             self.media_list.resize(seq_nr + 1, None);
 101         }
 102         self.media_list[seq_nr] = Some(uuid);
 103         Ok(())
 104     }
 105
 106     pub fn last_media_uuid(&self) -> Option<&Uuid> {
 107         match self.media_list.last() {
 108             None => None,
 109             Some(None) => None,
 110             Some(Some(ref last_uuid)) => Some(last_uuid),
 111         }
 112     }
 113
 114     pub fn is_last_media(&self, uuid: &Uuid) -> bool {
 115         match self.media_list.last() {
 116             None => false,
 117             Some(None) => false,
 118             Some(Some(last_uuid)) => uuid == last_uuid,
 119         }
 120     }
 121 }
 122
 123 /// Media Inventory
 124 pub struct Inventory {
 125     map: BTreeMap<Uuid, MediaId>,
 126
 127     inventory_path: PathBuf,
 128     lockfile_path: PathBuf,
 129
 130     // helpers
 131     media_set_start_times: HashMap<Uuid, i64>
 132 }
 133
 134 impl Inventory {
 135
 136     pub const MEDIA_INVENTORY_FILENAME: &'static str = "inventory.json";
 137     pub const MEDIA_INVENTORY_LOCKFILE: &'static str = ".inventory.lck";
 138
 139     fn new(base_path: &Path) -> Self {
 140
 141         let mut inventory_path = base_path.to_owned();
 142         inventory_path.push(Self::MEDIA_INVENTORY_FILENAME);
 143
 144         let mut lockfile_path = base_path.to_owned();
 145         lockfile_path.push(Self::MEDIA_INVENTORY_LOCKFILE);
 146
 147         Self {
 148             map: BTreeMap::new(),
 149             media_set_start_times: HashMap::new(),
 150             inventory_path,
 151             lockfile_path,
 152         }
 153     }
 154
 155     pub fn load(base_path: &Path) -> Result<Self, Error> {
 156         let mut me = Self::new(base_path);
 157         me.reload()?;
 158         Ok(me)
 159     }
 160
 161     /// Reload the database
 162     pub fn reload(&mut self) -> Result<(), Error> {
 163         self.map = Self::load_media_db(&self.inventory_path)?;
 164         self.update_helpers();
 165         Ok(())
 166     }
 167
 168     fn update_helpers(&mut self) {
 169
 170         // recompute media_set_start_times
 171
 172         let mut set_start_times = HashMap::new();
 173
 174         for media in self.map.values() {
 175             let set = match &media.media_set_label {
 176                 None => continue,
 177                 Some(set) => set,
 178             };
 179             if set.seq_nr == 0 {
 180                 set_start_times.insert(set.uuid.clone(), set.ctime);
 181             }
 182         }
 183
 184         self.media_set_start_times = set_start_times;
 185     }
 186
 187     /// Lock the database
 188     pub fn lock(&self) -> Result<std::fs::File, Error> {
 189         open_file_locked(&self.lockfile_path, std::time::Duration::new(10, 0), true)
 190     }
 191
 192     fn load_media_db(path: &Path) -> Result<BTreeMap<Uuid, MediaId>, Error> {
 193
 194         let data = file_get_json(path, Some(json!([])))?;
 195         let media_list: Vec<MediaId> = serde_json::from_value(data)?;
 196
 197         let mut map = BTreeMap::new();
 198         for item in media_list.into_iter() {
 199             map.insert(item.label.uuid.clone(), item);
 200         }
 201
 202         Ok(map)
 203     }
 204
 205     fn replace_file(&self) -> Result<(), Error> {
 206         let list: Vec<&MediaId> = self.map.values().collect();
 207         let raw = serde_json::to_string_pretty(&serde_json::to_value(list)?)?;
 208
 209         let backup_user = crate::backup::backup_user()?;
 210         let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640);
 211         let options = CreateOptions::new()
 212             .perm(mode)
 213             .owner(backup_user.uid)
 214             .group(backup_user.gid);
 215
 216         replace_file(&self.inventory_path, raw.as_bytes(), options)?;
 217
 218         Ok(())
 219     }
 220
 221     /// Stores a single MediaID persistently
 222     pub fn store(&mut self, mut media_id: MediaId) -> Result<(), Error> {
 223         let _lock = self.lock()?;
 224         self.map = Self::load_media_db(&self.inventory_path)?;
 225
 226         // do not overwrite unsaved pool assignments
 227         if media_id.media_set_label.is_none() {
 228             if let Some(previous) = self.map.get(&media_id.label.uuid) {
 229                 if let Some(ref set) = previous.media_set_label {
 230                     if set.uuid.as_ref() == [0u8;16] {
 231                         media_id.media_set_label = Some(set.clone());
 232                     }
 233                 }
 234             }
 235         }
 236
 237         self.map.insert(media_id.label.uuid.clone(), media_id);
 238         self.update_helpers();
 239         self.replace_file()?;
 240         Ok(())
 241     }
 242
 243     /*
 244     /// Same a store, but extract MediaId form MediaLabelInfo
 245     pub fn store_media_info(&mut self, info: &MediaLabelInfo) -> Result<(), Error> {
 246         let media_id = MediaId {
 247             label: info.label.clone(),
 248             media_set_label: info.media_set_label.clone().map(|(l, _)| l),
 249         };
 250         self.store(media_id)
 251     }
 252     */
 253
 254     /// Lookup media
 255     pub fn lookup_media(&self, uuid: &Uuid) -> Option<&MediaId> {
 256         self.map.get(uuid)
 257     }
 258
 259     /// find media by changer_id
 260     pub fn find_media_by_changer_id(&self, changer_id: &str) -> Option<&MediaId> {
 261         for (_uuid, media_id) in &self.map {
 262             if media_id.label.changer_id == changer_id {
 263                 return Some(media_id);
 264             }
 265         }
 266         None
 267     }
 268
 269     /// Lookup media pool
 270     ///
 271     /// Returns (pool, is_empty)
 272     pub fn lookup_media_pool(&self, uuid: &Uuid) -> Option<(&str, bool)> {
 273         match self.map.get(uuid) {
 274             None => None,
 275             Some(media_id) => {
 276                 match media_id.media_set_label {
 277                     None => None, // not assigned to any pool
 278                     Some(ref set) => {
 279                         let is_empty = set.uuid.as_ref() == [0u8;16];
 280                         Some((&set.pool, is_empty))
 281                     }
 282                 }
 283             }
 284         }
 285     }
 286
 287     /// List all media assigned to the pool
 288     pub fn list_pool_media(&self, pool: &str) -> Vec<MediaId> {
 289         let mut list = Vec::new();
 290
 291         for (_uuid, media_id) in &self.map {
 292             match media_id.media_set_label {
 293                 None => continue, // not assigned to any pool
 294                 Some(ref set) => {
 295                     if set.pool != pool {
 296                         continue; // belong to another pool
 297                     }
 298
 299                     if set.uuid.as_ref() == [0u8;16] { // should we do this??
 300                         list.push(MediaId {
 301                             label: media_id.label.clone(),
 302                             media_set_label: None,
 303                         })
 304                     } else {
 305                         list.push(media_id.clone());
 306                     }
 307                 }
 308             }
 309
 310         }
 311
 312         list
 313     }
 314
 315     /// List all used media
 316     pub fn list_used_media(&self) -> Vec<MediaId> {
 317         let mut list = Vec::new();
 318
 319         for (_uuid, media_id) in &self.map {
 320             match media_id.media_set_label {
 321                 None => continue, // not assigned to any pool
 322                 Some(ref set) => {
 323                     if set.uuid.as_ref() != [0u8;16] {
 324                         list.push(media_id.clone());
 325                     }
 326                 }
 327             }
 328         }
 329
 330         list
 331     }
 332
 333     /// List media not assigned to any pool
 334     pub fn list_unassigned_media(&self) -> Vec<MediaId> {
 335         let mut list = Vec::new();
 336
 337         for (_uuid, media_id) in &self.map {
 338             if media_id.media_set_label.is_none() {
 339                 list.push(media_id.clone());
 340             }
 341         }
 342
 343         list
 344     }
 345
 346     pub fn media_set_start_time(&self, media_set_uuid: &Uuid) -> Option<i64> {
 347         self.media_set_start_times.get(media_set_uuid).map(|t| *t)
 348     }
 349
 350     /// Compute a single media sets
 351     pub fn compute_media_set_members(&self, media_set_uuid: &Uuid) -> Result<MediaSet, Error> {
 352
 353         let mut set = MediaSet::with_data(media_set_uuid.clone(), Vec::new());
 354
 355         for media in self.map.values() {
 356             match media.media_set_label {
 357                 None => continue,
 358                 Some(MediaSetLabel { seq_nr, ref uuid, .. }) => {
 359                     if  uuid != media_set_uuid {
 360                         continue;
 361                     }
 362                     set.insert_media(media.label.uuid.clone(), seq_nr)?;
 363                 }
 364             }
 365         }
 366
 367         Ok(set)
 368     }
 369
 370     /// Compute all media sets
 371     pub fn compute_media_set_list(&self) -> Result<HashMap<Uuid, MediaSet>, Error> {
 372
 373         let mut set_map: HashMap<Uuid, MediaSet> = HashMap::new();
 374
 375         for media in self.map.values() {
 376             match media.media_set_label {
 377                 None => continue,
 378                 Some(MediaSetLabel { seq_nr, ref uuid, .. }) => {
 379
 380                     let set = set_map.entry(uuid.clone()).or_insert_with(|| {
 381                         MediaSet::with_data(uuid.clone(), Vec::new())
 382                     });
 383
 384                     set.insert_media(media.label.uuid.clone(), seq_nr)?;
 385                 }
 386             }
 387         }
 388
 389         Ok(set_map)
 390     }
 391
 392     /// Returns the latest media set for a pool
 393     pub fn latest_media_set(&self, pool: &str) -> Option<Uuid> {
 394
 395         let mut last_set: Option<(Uuid, i64)> = None;
 396
 397         let set_list = self.map.values()
 398             .filter_map(|media| media.media_set_label.as_ref())
 399             .filter(|set| &set.pool == &pool && set.uuid.as_ref() != [0u8;16]);
 400
 401         for set in set_list {
 402             match last_set {
 403                 None => {
 404                     last_set = Some((set.uuid.clone(), set.ctime));
 405                 }
 406                 Some((_, last_ctime)) => {
 407                     if set.ctime > last_ctime {
 408                         last_set = Some((set.uuid.clone(), set.ctime));
 409                     }
 410                 }
 411             }
 412         }
 413
 414         let (uuid, ctime) = match last_set {
 415             None => return None,
 416             Some((uuid, ctime)) => (uuid, ctime),
 417         };
 418
 419         // consistency check - must be the only set with that ctime
 420         let set_list = self.map.values()
 421             .filter_map(|media| media.media_set_label.as_ref())
 422             .filter(|set| &set.pool == &pool && set.uuid.as_ref() != [0u8;16]);
 423
 424         for set in set_list {
 425             if set.uuid != uuid && set.ctime >= ctime { // should not happen
 426                 eprintln!("latest_media_set: found set with equal ctime ({}, {})", set.uuid, uuid);
 427                 return None;
 428             }
 429         }
 430
 431         Some(uuid)
 432     }
 433
 434     // Test if there is a media set (in the same pool) newer than this one.
 435     // Return the ctime of the nearest media set
 436     fn media_set_next_start_time(&self, media_set_uuid: &Uuid) -> Option<i64> {
 437
 438         let (pool, ctime) = match self.map.values()
 439             .filter_map(|media| media.media_set_label.as_ref())
 440             .find_map(|set| {
 441                 if &set.uuid == media_set_uuid {
 442                     Some((set.pool.clone(), set.ctime))
 443                 } else {
 444                     None
 445                 }
 446             }) {
 447                 Some((pool, ctime)) => (pool, ctime),
 448                 None => return None,
 449             };
 450
 451         let set_list = self.map.values()
 452             .filter_map(|media| media.media_set_label.as_ref())
 453             .filter(|set| (&set.uuid != media_set_uuid) && (&set.pool == &pool));
 454
 455         let mut next_ctime = None;
 456
 457         for set in set_list {
 458             if set.ctime > ctime {
 459                 match next_ctime {
 460                     None => {
 461                         next_ctime = Some(set.ctime);
 462                     }
 463                     Some(last_next_ctime) => {
 464                         if set.ctime < last_next_ctime {
 465                             next_ctime = Some(set.ctime);
 466                         }
 467                     }
 468                 }
 469             }
 470         }
 471
 472         next_ctime
 473     }
 474
 475     pub fn media_expire_time(
 476         &self,
 477         media: &MediaId,
 478         media_set_policy: &MediaSetPolicy,
 479         retention_policy: &RetentionPolicy,
 480     ) -> i64 {
 481
 482         if let RetentionPolicy::KeepForever = retention_policy {
 483             return i64::MAX;
 484         }
 485
 486         let set = match media.media_set_label {
 487             None => return i64::MAX,
 488             Some(ref set) => set,
 489         };
 490
 491         let set_start_time = match self.media_set_start_time(&set.uuid) {
 492             None => {
 493                 // missing information, use ctime from this
 494                 // set (always greater than ctime from seq_nr 0)
 495                 set.ctime
 496             }
 497             Some(time) => time,
 498         };
 499
 500         let max_use_time = match media_set_policy {
 501             MediaSetPolicy::ContinueCurrent => {
 502                 match self.media_set_next_start_time(&set.uuid) {
 503                     Some(next_start_time) => next_start_time,
 504                     None => return i64::MAX,
 505                 }
 506             }
 507             MediaSetPolicy::AlwaysCreate => {
 508                 set_start_time + 1
 509             }
 510             MediaSetPolicy::CreateAt(ref event) => {
 511                 match compute_next_event(event, set_start_time, false) {
 512                     Ok(Some(next)) => next,
 513                     Ok(None) | Err(_) => return i64::MAX,
 514                 }
 515             }
 516         };
 517
 518         match retention_policy {
 519             RetentionPolicy::KeepForever => i64::MAX,
 520             RetentionPolicy::OverwriteAlways => max_use_time,
 521             RetentionPolicy::ProtectFor(time_span) => {
 522                 let seconds = f64::from(time_span.clone()) as i64;
 523                 max_use_time + seconds
 524             }
 525         }
 526     }
 527
 528     /// Generate a human readable name for the media set
 529     ///
 530     /// The template can include strftime time format specifications.
 531     pub fn generate_media_set_name(
 532         &self,
 533         media_set_uuid: &Uuid,
 534         template: Option<String>,
 535     ) -> Result<String, Error> {
 536
 537         if let Some(ctime) = self.media_set_start_time(media_set_uuid) {
 538             let mut template = template.unwrap_or(String::from("%id%"));
 539             template = template.replace("%id%", &media_set_uuid.to_string());
 540             proxmox::tools::time::strftime_local(&template, ctime)
 541         } else {
 542             // We don't know the set start time, so we cannot use the template
 543             Ok(media_set_uuid.to_string())
 544         }
 545     }
 546
 547     // Helpers to simplify testing
 548
 549     /// Genreate and insert a new free tape (test helper)
 550     pub fn generate_free_tape(&mut self, changer_id: &str, ctime: i64) -> Uuid {
 551
 552         let label = DriveLabel {
 553             changer_id: changer_id.to_string(),
 554             uuid: Uuid::generate(),
 555             ctime,
 556         };
 557         let uuid = label.uuid.clone();
 558
 559         self.store(MediaId { label, media_set_label: None }).unwrap();
 560
 561         uuid
 562     }
 563
 564     /// Genreate and insert a new tape assigned to a specific pool
 565     /// (test helper)
 566     pub fn generate_assigned_tape(
 567         &mut self,
 568         changer_id: &str,
 569         pool: &str,
 570         ctime: i64,
 571     ) -> Uuid {
 572
 573         let label = DriveLabel {
 574             changer_id: changer_id.to_string(),
 575             uuid: Uuid::generate(),
 576             ctime,
 577         };
 578
 579         let uuid = label.uuid.clone();
 580
 581         let set = MediaSetLabel::with_data(pool, [0u8; 16].into(), 0, ctime);
 582
 583         self.store(MediaId { label, media_set_label: Some(set) }).unwrap();
 584
 585         uuid
 586     }
 587
 588     /// Genreate and insert a used tape (test helper)
 589     pub fn generate_used_tape(
 590         &mut self,
 591         changer_id: &str,
 592         set: MediaSetLabel,
 593         ctime: i64,
 594     ) -> Uuid {
 595         let label = DriveLabel {
 596             changer_id: changer_id.to_string(),
 597             uuid: Uuid::generate(),
 598             ctime,
 599         };
 600         let uuid = label.uuid.clone();
 601
 602         self.store(MediaId { label, media_set_label: Some(set) }).unwrap();
 603
 604         uuid
 605     }
 606 }
 607
 608 // shell completion helper
 609
 610 /// List of known media uuids
 611 pub fn complete_media_uuid(
 612     _arg: &str,
 613     _param: &HashMap<String, String>,
 614 ) -> Vec<String> {
 615
 616     let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
 617         Ok(inventory) => inventory,
 618         Err(_) => return Vec::new(),
 619     };
 620
 621     inventory.map.keys().map(|uuid| uuid.to_string()).collect()
 622 }
 623
 624 /// List of known media sets
 625 pub fn complete_media_set_uuid(
 626     _arg: &str,
 627     _param: &HashMap<String, String>,
 628 ) -> Vec<String> {
 629
 630     let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
 631         Ok(inventory) => inventory,
 632         Err(_) => return Vec::new(),
 633     };
 634
 635     inventory.map.values()
 636         .filter_map(|media| media.media_set_label.as_ref())
 637         .map(|set| set.uuid.to_string()).collect()
 638 }
 639
 640 /// List of known media labels (barcodes)
 641 pub fn complete_media_changer_id(
 642     _arg: &str,
 643     _param: &HashMap<String, String>,
 644 ) -> Vec<String> {
 645
 646     let inventory = match Inventory::load(Path::new(TAPE_STATUS_DIR)) {
 647         Ok(inventory) => inventory,
 648         Err(_) => return Vec::new(),
 649     };
 650
 651     inventory.map.values().map(|media| media.label.changer_id.clone()).collect()
 652 }