src/tape/media_catalog.rs

   1 use std::convert::TryFrom;
   2 use std::fs::File;
   3 use std::io::{Write, Read, BufReader, Seek, SeekFrom};
   4 use std::os::unix::io::AsRawFd;
   5 use std::path::{PathBuf, Path};
   6 use std::collections::{HashSet, HashMap};
   7
   8 use anyhow::{bail, format_err, Error};
   9 use endian_trait::Endian;
  10
  11 use proxmox_sys::fs::read_subdir;
  12 use pbs_datastore::backup_info::BackupDir;
  13
  14 use proxmox_sys::fs::{
  15     fchown,
  16     create_path,
  17     CreateOptions,
  18 };
  19 use proxmox_io::{WriteExt, ReadExt};
  20 use proxmox_uuid::Uuid;
  21
  22 use crate::{
  23     tape::{
  24         MediaId,
  25         file_formats::MediaSetLabel,
  26     },
  27 };
  28
  29 pub struct DatastoreContent {
  30     pub snapshot_index: HashMap<String, u64>, // snapshot => file_nr
  31     pub chunk_index: HashMap<[u8;32], u64>, // chunk => file_nr
  32 }
  33
  34 impl DatastoreContent {
  35
  36     pub fn new() -> Self {
  37         Self {
  38             chunk_index: HashMap::new(),
  39             snapshot_index: HashMap::new(),
  40         }
  41     }
  42 }
  43
  44 /// The Media Catalog
  45 ///
  46 /// Stores what chunks and snapshots are stored on a specific media,
  47 /// including the file position.
  48 ///
  49 /// We use a simple binary format to store data on disk.
  50 pub struct MediaCatalog  {
  51
  52     uuid: Uuid, // BackupMedia uuid
  53
  54     file: Option<File>,
  55
  56     log_to_stdout: bool,
  57
  58     current_archive: Option<(Uuid, u64, String)>, // (uuid, file_nr, store)
  59
  60     last_entry: Option<(Uuid, u64)>,
  61
  62     content: HashMap<String, DatastoreContent>,
  63
  64     pending: Vec<u8>,
  65 }
  66
  67 impl MediaCatalog {
  68
  69     /// Magic number for media catalog files.
  70     // openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.0")[0..8]
  71     // Note: this version did not store datastore names (not supported anymore)
  72     pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0: [u8; 8] = [221, 29, 164, 1, 59, 69, 19, 40];
  73
  74     // openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.1")[0..8]
  75     pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1: [u8; 8] = [76, 142, 232, 193, 32, 168, 137, 113];
  76
  77     /// List media with catalogs
  78     pub fn media_with_catalogs(base_path: &Path) -> Result<HashSet<Uuid>, Error> {
  79         let mut catalogs = HashSet::new();
  80
  81         for entry in read_subdir(libc::AT_FDCWD, base_path)? {
  82             let entry = entry?;
  83             let name = unsafe { entry.file_name_utf8_unchecked() };
  84             if !name.ends_with(".log") { continue; }
  85             if let Ok(uuid) = Uuid::parse_str(&name[..(name.len()-4)]) {
  86                 catalogs.insert(uuid);
  87             }
  88         }
  89
  90         Ok(catalogs)
  91     }
  92
  93     pub fn catalog_path(base_path: &Path, uuid: &Uuid) -> PathBuf {
  94         let mut path = base_path.to_owned();
  95         path.push(uuid.to_string());
  96         path.set_extension("log");
  97         path
  98     }
  99
 100     fn tmp_catalog_path(base_path: &Path, uuid: &Uuid) -> PathBuf {
 101         let mut path = base_path.to_owned();
 102         path.push(uuid.to_string());
 103         path.set_extension("tmp");
 104         path
 105     }
 106
 107     /// Test if a catalog exists
 108     pub fn exists(base_path: &Path, uuid: &Uuid) -> bool {
 109         Self::catalog_path(base_path, uuid).exists()
 110     }
 111
 112     /// Destroy the media catalog (remove all files)
 113     pub fn destroy(base_path: &Path, uuid: &Uuid) -> Result<(), Error> {
 114
 115         let path = Self::catalog_path(base_path, uuid);
 116
 117         match std::fs::remove_file(path) {
 118             Ok(()) => Ok(()),
 119             Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
 120             Err(err) => Err(err.into()),
 121         }
 122     }
 123
 124     /// Destroy the media catalog if media_set uuid does not match
 125     pub fn destroy_unrelated_catalog(
 126         base_path: &Path,
 127         media_id: &MediaId,
 128     ) -> Result<(), Error> {
 129
 130         let uuid = &media_id.label.uuid;
 131
 132         let path = Self::catalog_path(base_path, uuid);
 133
 134         let file = match std::fs::OpenOptions::new().read(true).open(&path) {
 135             Ok(file) => file,
 136             Err(ref err) if err.kind() == std::io::ErrorKind::NotFound => {
 137                 return Ok(());
 138             }
 139             Err(err) => return Err(err.into()),
 140         };
 141
 142         let mut file = BufReader::new(file);
 143
 144         let expected_media_set_id = match media_id.media_set_label {
 145             None => {
 146                 std::fs::remove_file(path)?;
 147                 return Ok(())
 148             },
 149             Some(ref set) => &set.uuid,
 150         };
 151
 152         let (found_magic_number, media_uuid, media_set_uuid) =
 153             Self::parse_catalog_header(&mut file)?;
 154
 155         if !found_magic_number {
 156             return Ok(());
 157         }
 158
 159         if let Some(ref media_uuid) = media_uuid {
 160             if media_uuid != uuid {
 161                 std::fs::remove_file(path)?;
 162                 return Ok(());
 163             }
 164         }
 165
 166         if let Some(ref media_set_uuid) = media_set_uuid {
 167             if media_set_uuid != expected_media_set_id {
 168                 std::fs::remove_file(path)?;
 169             }
 170         }
 171
 172         Ok(())
 173     }
 174
 175     /// Enable/Disable logging to stdout (disabled by default)
 176     pub fn log_to_stdout(&mut self, enable: bool) {
 177         self.log_to_stdout = enable;
 178     }
 179
 180     fn create_basedir(base_path: &Path) -> Result<(), Error> {
 181         let backup_user = pbs_config::backup_user()?;
 182         let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640);
 183         let opts = CreateOptions::new()
 184             .perm(mode)
 185             .owner(backup_user.uid)
 186             .group(backup_user.gid);
 187
 188         create_path(base_path, None, Some(opts))
 189             .map_err(|err: Error| format_err!("unable to create media catalog dir - {}", err))?;
 190         Ok(())
 191     }
 192
 193     /// Open a catalog database, load into memory
 194     pub fn open(
 195         base_path: &Path,
 196         media_id: &MediaId,
 197         write: bool,
 198         create: bool,
 199     ) -> Result<Self, Error> {
 200
 201         let uuid = &media_id.label.uuid;
 202
 203         let path = Self::catalog_path(base_path, uuid);
 204
 205         let me = proxmox_lang::try_block!({
 206
 207             Self::create_basedir(base_path)?;
 208
 209             let mut file = std::fs::OpenOptions::new()
 210                 .read(true)
 211                 .write(write)
 212                 .create(create)
 213                 .open(&path)?;
 214
 215             let backup_user = pbs_config::backup_user()?;
 216             fchown(file.as_raw_fd(), Some(backup_user.uid), Some(backup_user.gid))
 217                 .map_err(|err| format_err!("fchown failed - {}", err))?;
 218
 219             let mut me = Self {
 220                 uuid: uuid.clone(),
 221                 file: None,
 222                 log_to_stdout: false,
 223                 current_archive: None,
 224                 last_entry: None,
 225                 content: HashMap::new(),
 226                 pending: Vec::new(),
 227             };
 228
 229             // Note: lock file, to get a consistent view with load_catalog
 230             nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockExclusive)?;
 231             let result = me.load_catalog(&mut file, media_id.media_set_label.as_ref());
 232             nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::Unlock)?;
 233
 234             let (found_magic_number, _) = result?;
 235
 236             if !found_magic_number {
 237                 me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
 238             }
 239
 240             if write {
 241                 me.file = Some(file);
 242             }
 243             Ok(me)
 244         }).map_err(|err: Error| {
 245             format_err!("unable to open media catalog {:?} - {}", path, err)
 246         })?;
 247
 248         Ok(me)
 249     }
 250
 251     /// Creates a temporary empty catalog file
 252     pub fn create_temporary_database_file(
 253         base_path: &Path,
 254         uuid: &Uuid,
 255     ) -> Result<File, Error> {
 256
 257         Self::create_basedir(base_path)?;
 258
 259         let tmp_path = Self::tmp_catalog_path(base_path, uuid);
 260
 261         let file = std::fs::OpenOptions::new()
 262             .read(true)
 263             .write(true)
 264             .create(true)
 265             .truncate(true)
 266             .open(&tmp_path)?;
 267
 268         if cfg!(test) {
 269             // We cannot use chown inside test environment (no permissions)
 270             return Ok(file);
 271         }
 272
 273         let backup_user = pbs_config::backup_user()?;
 274         fchown(file.as_raw_fd(), Some(backup_user.uid), Some(backup_user.gid))
 275             .map_err(|err| format_err!("fchown failed - {}", err))?;
 276
 277         Ok(file)
 278     }
 279
 280     /// Creates a temporary, empty catalog database
 281     ///
 282     /// Creates a new catalog file using a ".tmp" file extension.
 283     pub fn create_temporary_database(
 284         base_path: &Path,
 285         media_id: &MediaId,
 286         log_to_stdout: bool,
 287     ) -> Result<Self, Error> {
 288
 289         let uuid = &media_id.label.uuid;
 290
 291         let tmp_path = Self::tmp_catalog_path(base_path, uuid);
 292
 293         let me = proxmox_lang::try_block!({
 294
 295             let file = Self::create_temporary_database_file(base_path, uuid)?;
 296
 297             let mut me = Self {
 298                 uuid: uuid.clone(),
 299                 file: Some(file),
 300                 log_to_stdout: false,
 301                 current_archive: None,
 302                 last_entry: None,
 303                 content: HashMap::new(),
 304                 pending: Vec::new(),
 305             };
 306
 307             me.log_to_stdout = log_to_stdout;
 308
 309             me.pending.extend(&Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1);
 310
 311             me.register_label(&media_id.label.uuid, 0, 0)?;
 312
 313             if let Some(ref set) = media_id.media_set_label {
 314                 me.register_label(&set.uuid, set.seq_nr, 1)?;
 315             }
 316
 317             me.commit()?;
 318
 319             Ok(me)
 320         }).map_err(|err: Error| {
 321             format_err!("unable to create temporary media catalog {:?} - {}", tmp_path, err)
 322         })?;
 323
 324         Ok(me)
 325     }
 326
 327     /// Commit or Abort a temporary catalog database
 328     ///
 329     /// With commit set, we rename the ".tmp" file extension to
 330     /// ".log". When commit is false, we remove the ".tmp" file.
 331     pub fn finish_temporary_database(
 332         base_path: &Path,
 333         uuid: &Uuid,
 334         commit: bool,
 335     ) -> Result<(), Error> {
 336
 337         let tmp_path = Self::tmp_catalog_path(base_path, uuid);
 338
 339         if commit {
 340             let mut catalog_path = tmp_path.clone();
 341             catalog_path.set_extension("log");
 342
 343             if let Err(err) = std::fs::rename(&tmp_path, &catalog_path) {
 344                 bail!("Atomic rename catalog {:?} failed - {}", catalog_path, err);
 345             }
 346         } else {
 347             std::fs::remove_file(&tmp_path)?;
 348         }
 349         Ok(())
 350     }
 351
 352     /// Returns the BackupMedia uuid
 353     pub fn uuid(&self) -> &Uuid {
 354         &self.uuid
 355     }
 356
 357     /// Accessor to content list
 358     pub fn content(&self) -> &HashMap<String, DatastoreContent> {
 359         &self.content
 360     }
 361
 362     /// Commit pending changes
 363     ///
 364     /// This is necessary to store changes persistently.
 365     ///
 366     /// Fixme: this should be atomic ...
 367     pub fn commit(&mut self) -> Result<(), Error> {
 368
 369         if self.pending.is_empty() {
 370             return Ok(());
 371         }
 372
 373         match self.file {
 374             Some(ref mut file) => {
 375                 let pending = &self.pending;
 376                 // Note: lock file, to get a consistent view with load_catalog
 377                 nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::LockExclusive)?;
 378                 let result: Result<(), Error> = proxmox_lang::try_block!({
 379                     file.write_all(pending)?;
 380                     file.flush()?;
 381                     file.sync_data()?;
 382                     Ok(())
 383                 });
 384                 nix::fcntl::flock(file.as_raw_fd(), nix::fcntl::FlockArg::Unlock)?;
 385
 386                 result?;
 387             }
 388             None => bail!("media catalog not writable (opened read only)"),
 389         }
 390
 391         self.pending = Vec::new();
 392
 393         Ok(())
 394     }
 395
 396     /// Conditionally commit if in pending data is large (> 1Mb)
 397     pub fn commit_if_large(&mut self) -> Result<(), Error> {
 398         if self.current_archive.is_some() {
 399             bail!("can't commit catalog in the middle of an chunk archive");
 400         }
 401         if self.pending.len() > 1024*1024 {
 402             self.commit()?;
 403         }
 404         Ok(())
 405     }
 406
 407     /// Destroy existing catalog, opens a new one
 408     pub fn overwrite(
 409         base_path: &Path,
 410         media_id: &MediaId,
 411         log_to_stdout: bool,
 412     ) ->  Result<Self, Error> {
 413
 414         let uuid = &media_id.label.uuid;
 415
 416         let me = Self::create_temporary_database(base_path, &media_id, log_to_stdout)?;
 417
 418         Self::finish_temporary_database(base_path, uuid, true)?;
 419
 420         Ok(me)
 421     }
 422
 423     /// Test if the catalog already contain a snapshot
 424     pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
 425         match self.content.get(store) {
 426             None => false,
 427             Some(content) => content.snapshot_index.contains_key(snapshot),
 428         }
 429     }
 430
 431     /// Returns the snapshot archive file number
 432     pub fn lookup_snapshot(&self, store: &str, snapshot: &str) -> Option<u64> {
 433         match self.content.get(store) {
 434             None => None,
 435             Some(content) => content.snapshot_index.get(snapshot).copied(),
 436         }
 437     }
 438
 439     /// Test if the catalog already contain a chunk
 440     pub fn contains_chunk(&self, store: &str, digest: &[u8;32]) -> bool {
 441         match self.content.get(store) {
 442             None => false,
 443             Some(content) => content.chunk_index.contains_key(digest),
 444         }
 445     }
 446
 447     /// Returns the chunk archive file number
 448     pub fn lookup_chunk(&self, store: &str, digest: &[u8;32]) -> Option<u64> {
 449         match self.content.get(store) {
 450             None => None,
 451             Some(content) => content.chunk_index.get(digest).copied(),
 452         }
 453     }
 454
 455     fn check_register_label(&self, file_number: u64, uuid: &Uuid) -> Result<(), Error> {
 456
 457         if file_number >= 2 {
 458             bail!("register label failed: got wrong file number ({} >= 2)", file_number);
 459         }
 460
 461         if file_number == 0 && uuid != &self.uuid {
 462             bail!("register label failed: uuid does not match");
 463         }
 464
 465         if self.current_archive.is_some() {
 466             bail!("register label failed: inside chunk archive");
 467         }
 468
 469         let expected_file_number = match self.last_entry {
 470             Some((_, last_number)) => last_number + 1,
 471             None => 0,
 472         };
 473
 474         if file_number != expected_file_number {
 475             bail!("register label failed: got unexpected file number ({} < {})",
 476                   file_number, expected_file_number);
 477         }
 478         Ok(())
 479     }
 480
 481     /// Register media labels (file 0 and 1)
 482     pub fn register_label(
 483         &mut self,
 484         uuid: &Uuid, // Media/MediaSet Uuid
 485         seq_nr: u64, // onyl used for media set labels
 486         file_number: u64,
 487     ) -> Result<(), Error> {
 488
 489         self.check_register_label(file_number, uuid)?;
 490
 491         if file_number == 0 && seq_nr != 0 {
 492             bail!("register_label failed - seq_nr should be 0 - iternal error");
 493         }
 494
 495         let entry = LabelEntry {
 496             file_number,
 497             uuid: *uuid.as_bytes(),
 498             seq_nr,
 499         };
 500
 501         if self.log_to_stdout {
 502             println!("L|{}|{}", file_number, uuid.to_string());
 503         }
 504
 505         self.pending.push(b'L');
 506
 507         unsafe { self.pending.write_le_value(entry)?; }
 508
 509         self.last_entry = Some((uuid.clone(), file_number));
 510
 511         Ok(())
 512     }
 513
 514     /// Register a chunk archive
 515     pub fn register_chunk_archive(
 516         &mut self,
 517         uuid: Uuid, // Uuid form MediaContentHeader
 518         file_number: u64,
 519         store: &str,
 520         chunk_list: &[[u8; 32]],
 521     ) -> Result<(), Error> {
 522         self.start_chunk_archive(uuid, file_number, store)?;
 523         for digest in chunk_list {
 524             self.register_chunk(digest)?;
 525         }
 526         self.end_chunk_archive()?;
 527         Ok(())
 528     }
 529
 530     /// Register a chunk
 531     ///
 532     /// Only valid after start_chunk_archive.
 533     fn register_chunk(
 534         &mut self,
 535         digest: &[u8;32],
 536     ) -> Result<(), Error> {
 537
 538         let (file_number, store) = match self.current_archive {
 539             None => bail!("register_chunk failed: no archive started"),
 540             Some((_, file_number, ref store)) => (file_number, store),
 541         };
 542
 543         if self.log_to_stdout {
 544             println!("C|{}", hex::encode(digest));
 545         }
 546
 547         self.pending.push(b'C');
 548         self.pending.extend(digest);
 549
 550         match self.content.get_mut(store) {
 551             None => bail!("storage {} not registered - internal error", store),
 552             Some(content) => {
 553                 content.chunk_index.insert(*digest, file_number);
 554             }
 555         }
 556
 557         Ok(())
 558     }
 559
 560     fn check_start_chunk_archive(&self, file_number: u64) -> Result<(), Error> {
 561
 562         if self.current_archive.is_some() {
 563             bail!("start_chunk_archive failed: already started");
 564         }
 565
 566         if file_number < 2 {
 567             bail!("start_chunk_archive failed: got wrong file number ({} < 2)", file_number);
 568         }
 569
 570         let expect_min_file_number = match self.last_entry {
 571             Some((_, last_number)) => last_number + 1,
 572             None => 0,
 573         };
 574
 575         if file_number < expect_min_file_number {
 576             bail!("start_chunk_archive: got unexpected file number ({} < {})",
 577                   file_number, expect_min_file_number);
 578         }
 579
 580         Ok(())
 581     }
 582
 583     /// Start a chunk archive section
 584     fn start_chunk_archive(
 585         &mut self,
 586         uuid: Uuid, // Uuid form MediaContentHeader
 587         file_number: u64,
 588         store: &str,
 589    ) -> Result<(), Error> {
 590
 591         self.check_start_chunk_archive(file_number)?;
 592
 593         let entry = ChunkArchiveStart {
 594             file_number,
 595             uuid: *uuid.as_bytes(),
 596             store_name_len: u8::try_from(store.len())?,
 597         };
 598
 599         if self.log_to_stdout {
 600             println!("A|{}|{}|{}", file_number, uuid.to_string(), store);
 601         }
 602
 603         self.pending.push(b'A');
 604
 605         unsafe { self.pending.write_le_value(entry)?; }
 606         self.pending.extend(store.as_bytes());
 607
 608         self.content.entry(store.to_string()).or_insert(DatastoreContent::new());
 609
 610         self.current_archive = Some((uuid, file_number, store.to_string()));
 611
 612         Ok(())
 613     }
 614
 615     fn check_end_chunk_archive(&self, uuid: &Uuid, file_number: u64) -> Result<(), Error> {
 616
 617         match self.current_archive {
 618             None => bail!("end_chunk archive failed: not started"),
 619             Some((ref expected_uuid, expected_file_number, ..)) => {
 620                 if uuid != expected_uuid {
 621                     bail!("end_chunk_archive failed: got unexpected uuid");
 622                 }
 623                 if file_number != expected_file_number {
 624                     bail!("end_chunk_archive failed: got unexpected file number ({} != {})",
 625                           file_number, expected_file_number);
 626                 }
 627             }
 628         }
 629         Ok(())
 630     }
 631
 632     /// End a chunk archive section
 633     fn end_chunk_archive(&mut self) -> Result<(), Error> {
 634
 635         match self.current_archive.take() {
 636             None => bail!("end_chunk_archive failed: not started"),
 637             Some((uuid, file_number, ..)) => {
 638
 639                 let entry = ChunkArchiveEnd {
 640                     file_number,
 641                     uuid: *uuid.as_bytes(),
 642                 };
 643
 644                 if self.log_to_stdout {
 645                     println!("E|{}|{}\n", file_number, uuid.to_string());
 646                 }
 647
 648                 self.pending.push(b'E');
 649
 650                 unsafe { self.pending.write_le_value(entry)?; }
 651
 652                 self.last_entry = Some((uuid, file_number));
 653             }
 654         }
 655
 656         Ok(())
 657     }
 658
 659     fn check_register_snapshot(&self, file_number: u64, snapshot: &str) -> Result<(), Error> {
 660
 661         if self.current_archive.is_some() {
 662             bail!("register_snapshot failed: inside chunk_archive");
 663         }
 664
 665         if file_number < 2 {
 666             bail!("register_snapshot failed: got wrong file number ({} < 2)", file_number);
 667         }
 668
 669         let expect_min_file_number = match self.last_entry {
 670             Some((_, last_number)) => last_number + 1,
 671             None => 0,
 672         };
 673
 674         if file_number < expect_min_file_number {
 675             bail!("register_snapshot failed: got unexpected file number ({} < {})",
 676                   file_number, expect_min_file_number);
 677         }
 678
 679         if let Err(err) = snapshot.parse::<BackupDir>() {
 680             bail!("register_snapshot failed: unable to parse snapshot '{}' - {}", snapshot, err);
 681         }
 682
 683         Ok(())
 684     }
 685
 686     /// Register a snapshot
 687     pub fn register_snapshot(
 688         &mut self,
 689         uuid: Uuid, // Uuid form MediaContentHeader
 690         file_number: u64,
 691         store: &str,
 692         snapshot: &str,
 693     ) -> Result<(), Error> {
 694
 695         self.check_register_snapshot(file_number, snapshot)?;
 696
 697         let entry = SnapshotEntry {
 698             file_number,
 699             uuid: *uuid.as_bytes(),
 700             store_name_len: u8::try_from(store.len())?,
 701             name_len: u16::try_from(snapshot.len())?,
 702         };
 703
 704         if self.log_to_stdout {
 705             println!("S|{}|{}|{}:{}", file_number, uuid.to_string(), store, snapshot);
 706         }
 707
 708         self.pending.push(b'S');
 709
 710         unsafe { self.pending.write_le_value(entry)?; }
 711         self.pending.extend(store.as_bytes());
 712         self.pending.push(b':');
 713         self.pending.extend(snapshot.as_bytes());
 714
 715         let content = self.content.entry(store.to_string())
 716             .or_insert(DatastoreContent::new());
 717
 718         content.snapshot_index.insert(snapshot.to_string(), file_number);
 719
 720         self.last_entry = Some((uuid, file_number));
 721
 722         Ok(())
 723     }
 724
 725     /// Parse the catalog header
 726     pub fn parse_catalog_header<R: Read>(
 727         reader: &mut R,
 728     ) -> Result<(bool, Option<Uuid>, Option<Uuid>), Error> {
 729
 730         // read/check magic number
 731         let mut magic = [0u8; 8];
 732         if !reader.read_exact_or_eof(&mut magic)? {
 733             /* EOF */
 734             return Ok((false, None, None));
 735         }
 736
 737         if magic == Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
 738             // only use in unreleased versions
 739             bail!("old catalog format (v1.0) is no longer supported");
 740         }
 741         if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1 {
 742             bail!("wrong magic number");
 743         }
 744
 745         let mut entry_type = [0u8; 1];
 746         if !reader.read_exact_or_eof(&mut entry_type)? {
 747             /* EOF */
 748             return Ok((true, None, None));
 749         }
 750
 751         if entry_type[0] != b'L' {
 752             bail!("got unexpected entry type");
 753         }
 754
 755         let entry0: LabelEntry = unsafe { reader.read_le_value()? };
 756
 757         let mut entry_type = [0u8; 1];
 758         if !reader.read_exact_or_eof(&mut entry_type)? {
 759             /* EOF */
 760             return Ok((true, Some(entry0.uuid.into()), None));
 761         }
 762
 763         if entry_type[0] != b'L' {
 764             bail!("got unexpected entry type");
 765         }
 766
 767         let entry1: LabelEntry = unsafe { reader.read_le_value()? };
 768
 769         Ok((true, Some(entry0.uuid.into()), Some(entry1.uuid.into())))
 770     }
 771
 772     fn load_catalog(
 773         &mut self,
 774         file: &mut File,
 775         media_set_label: Option<&MediaSetLabel>,
 776     ) -> Result<(bool, Option<Uuid>), Error> {
 777
 778         let mut file = BufReader::new(file);
 779         let mut found_magic_number = false;
 780         let mut media_set_uuid = None;
 781
 782         loop {
 783             let pos = file.seek(SeekFrom::Current(0))?; // get current pos
 784
 785             if pos == 0 { // read/check magic number
 786                 let mut magic = [0u8; 8];
 787                 match file.read_exact_or_eof(&mut magic) {
 788                     Ok(false) => { /* EOF */ break; }
 789                     Ok(true) => { /* OK */ }
 790                     Err(err) => bail!("read failed - {}", err),
 791                 }
 792                 if magic == Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
 793                     // only use in unreleased versions
 794                     bail!("old catalog format (v1.0) is no longer supported");
 795                 }
 796                 if magic != Self::PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_1 {
 797                     bail!("wrong magic number");
 798                 }
 799                 found_magic_number = true;
 800                 continue;
 801             }
 802
 803             let mut entry_type = [0u8; 1];
 804             match file.read_exact_or_eof(&mut entry_type) {
 805                 Ok(false) => { /* EOF */ break; }
 806                 Ok(true) => { /* OK */ }
 807                 Err(err) => bail!("read failed - {}", err),
 808             }
 809
 810             match entry_type[0] {
 811                 b'C' => {
 812                     let (file_number, store) = match self.current_archive {
 813                         None => bail!("register_chunk failed: no archive started"),
 814                         Some((_, file_number, ref store)) => (file_number, store),
 815                     };
 816                     let mut digest = [0u8; 32];
 817                     file.read_exact(&mut digest)?;
 818                     match self.content.get_mut(store) {
 819                         None => bail!("storage {} not registered - internal error", store),
 820                         Some(content) => {
 821                             content.chunk_index.insert(digest, file_number);
 822                         }
 823                     }
 824                 }
 825                 b'A' => {
 826                     let entry: ChunkArchiveStart = unsafe { file.read_le_value()? };
 827                     let file_number = entry.file_number;
 828                     let uuid = Uuid::from(entry.uuid);
 829                     let store_name_len = entry.store_name_len as usize;
 830
 831                     let store = file.read_exact_allocated(store_name_len)?;
 832                     let store = std::str::from_utf8(&store)?;
 833
 834                     self.check_start_chunk_archive(file_number)?;
 835
 836                     self.content.entry(store.to_string())
 837                         .or_insert(DatastoreContent::new());
 838
 839                     self.current_archive = Some((uuid, file_number, store.to_string()));
 840                }
 841                 b'E' => {
 842                     let entry: ChunkArchiveEnd = unsafe { file.read_le_value()? };
 843                     let file_number = entry.file_number;
 844                     let uuid = Uuid::from(entry.uuid);
 845
 846                     self.check_end_chunk_archive(&uuid, file_number)?;
 847
 848                     self.current_archive = None;
 849                     self.last_entry = Some((uuid, file_number));
 850                 }
 851                 b'S' => {
 852                     let entry: SnapshotEntry = unsafe { file.read_le_value()? };
 853                     let file_number = entry.file_number;
 854                     let store_name_len = entry.store_name_len as usize;
 855                     let name_len = entry.name_len as usize;
 856                     let uuid = Uuid::from(entry.uuid);
 857
 858                     let store = file.read_exact_allocated(store_name_len + 1)?;
 859                     if store[store_name_len] != b':' {
 860                         bail!("parse-error: missing separator in SnapshotEntry");
 861                     }
 862
 863                     let store = std::str::from_utf8(&store[..store_name_len])?;
 864
 865                     let snapshot = file.read_exact_allocated(name_len)?;
 866                     let snapshot = std::str::from_utf8(&snapshot)?;
 867
 868                     self.check_register_snapshot(file_number, snapshot)?;
 869
 870                     let content = self.content.entry(store.to_string())
 871                         .or_insert(DatastoreContent::new());
 872
 873                     content.snapshot_index.insert(snapshot.to_string(), file_number);
 874
 875                     self.last_entry = Some((uuid, file_number));
 876                 }
 877                 b'L' => {
 878                     let entry: LabelEntry = unsafe { file.read_le_value()? };
 879                     let file_number = entry.file_number;
 880                     let uuid = Uuid::from(entry.uuid);
 881
 882                     self.check_register_label(file_number, &uuid)?;
 883
 884                     if file_number == 1 {
 885                         if let Some(set) = media_set_label {
 886                             if set.uuid != uuid {
 887                                 bail!("got unexpected media set uuid");
 888                             }
 889                             if set.seq_nr != entry.seq_nr {
 890                                 bail!("got unexpected media set sequence number");
 891                             }
 892                         }
 893                         media_set_uuid = Some(uuid.clone());
 894                     }
 895
 896                     self.last_entry = Some((uuid, file_number));
 897                 }
 898                 _ => {
 899                     bail!("unknown entry type '{}'", entry_type[0]);
 900                 }
 901             }
 902
 903         }
 904
 905         Ok((found_magic_number, media_set_uuid))
 906     }
 907 }
 908
 909 /// Media set catalog
 910 ///
 911 /// Catalog for multiple media.
 912 pub struct MediaSetCatalog  {
 913     catalog_list: HashMap<Uuid, MediaCatalog>,
 914 }
 915
 916 impl MediaSetCatalog {
 917
 918     /// Creates a new instance
 919     pub fn new() -> Self {
 920         Self {
 921             catalog_list: HashMap::new(),
 922         }
 923     }
 924
 925     /// Add a catalog
 926     pub fn append_catalog(&mut self, catalog: MediaCatalog) -> Result<(), Error> {
 927
 928         if self.catalog_list.get(&catalog.uuid).is_some() {
 929             bail!("MediaSetCatalog already contains media '{}'", catalog.uuid);
 930         }
 931
 932         self.catalog_list.insert(catalog.uuid.clone(), catalog);
 933
 934         Ok(())
 935     }
 936
 937     /// Remove a catalog
 938     pub fn remove_catalog(&mut self, media_uuid: &Uuid) {
 939         self.catalog_list.remove(media_uuid);
 940     }
 941
 942     /// Test if the catalog already contain a snapshot
 943     pub fn contains_snapshot(&self, store: &str, snapshot: &str) -> bool {
 944         for catalog in self.catalog_list.values() {
 945             if catalog.contains_snapshot(store, snapshot) {
 946                 return true;
 947             }
 948         }
 949         false
 950     }
 951
 952     /// Returns the media uuid and snapshot archive file number
 953     pub fn lookup_snapshot(&self, store: &str, snapshot: &str) -> Option<(&Uuid, u64)> {
 954         for (uuid, catalog) in self.catalog_list.iter() {
 955             if let Some(nr) = catalog.lookup_snapshot(store, snapshot) {
 956                 return Some((uuid, nr));
 957             }
 958         }
 959         None
 960     }
 961
 962     /// Test if the catalog already contain a chunk
 963     pub fn contains_chunk(&self, store: &str, digest: &[u8;32]) -> bool {
 964         for catalog in self.catalog_list.values() {
 965             if catalog.contains_chunk(store, digest) {
 966                 return true;
 967             }
 968         }
 969         false
 970     }
 971
 972     /// Returns the media uuid and chunk archive file number
 973     pub fn lookup_chunk(&self, store: &str, digest: &[u8;32]) -> Option<(&Uuid, u64)> {
 974         for (uuid, catalog) in self.catalog_list.iter() {
 975             if let Some(nr) = catalog.lookup_chunk(store, digest) {
 976                 return Some((uuid, nr));
 977             }
 978         }
 979         None
 980     }
 981 }
 982
 983 // Type definitions for internal binary catalog encoding
 984
 985 #[derive(Endian)]
 986 #[repr(C)]
 987 struct LabelEntry {
 988     file_number: u64,
 989     uuid: [u8;16],
 990     seq_nr: u64, // only used for media set labels
 991 }
 992
 993 #[derive(Endian)]
 994 #[repr(C)]
 995 struct ChunkArchiveStart {
 996     file_number: u64,
 997     uuid: [u8;16],
 998     store_name_len: u8,
 999     /* datastore name follows */
1000 }
1001
1002 #[derive(Endian)]
1003 #[repr(C)]
1004 struct ChunkArchiveEnd{
1005     file_number: u64,
1006     uuid: [u8;16],
1007 }
1008
1009 #[derive(Endian)]
1010 #[repr(C)]
1011 struct SnapshotEntry{
1012     file_number: u64,
1013     uuid: [u8;16],
1014     store_name_len: u8,
1015     name_len: u16,
1016     /* datastore name,  ':', snapshot name follows */
1017 }