src/tape/media_catalog.rs

   1 use std::convert::TryFrom;
   2 use std::fs::File;
   3 use std::io::{Write, Read, BufReader, Seek, SeekFrom};
   4 use std::os::unix::io::AsRawFd;
   5 use std::path::Path;
   6 use std::collections::HashMap;
   7
   8 use anyhow::{bail, format_err, Error};
   9 use endian_trait::Endian;
  10
  11 use proxmox::tools::{
  12     Uuid,
  13     fs::{
  14         fchown,
  15         create_path,
  16         CreateOptions,
  17     },
  18     io::{
  19         WriteExt,
  20         ReadExt,
  21     },
  22 };
  23
  24 use crate::{
  25     backup::BackupDir,
  26     tape::{
  27         MediaId,
  28     },
  29 };
  30
  31 // openssl::sha::sha256(b"Proxmox Backup Media Catalog v1.0")[0..8]
  32 pub const PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0: [u8; 8] = [221, 29, 164, 1, 59, 69, 19, 40];
  33
  34 /// The Media Catalog
  35 ///
  36 /// Stores what chunks and snapshots are stored on a specific media,
  37 /// including the file position.
  38 ///
  39 /// We use a simple binary format to store data on disk.
  40 pub struct MediaCatalog  {
  41
  42     uuid: Uuid, // BackupMedia uuid
  43
  44     file: Option<File>,
  45
  46     pub log_to_stdout: bool,
  47
  48     current_archive: Option<(Uuid, u64)>,
  49
  50     last_entry: Option<(Uuid, u64)>,
  51
  52     chunk_index: HashMap<[u8;32], u64>,
  53
  54     snapshot_index: HashMap<String, u64>,
  55
  56     pending: Vec<u8>,
  57 }
  58
  59 impl MediaCatalog {
  60
  61     /// Test if a catalog exists
  62     pub fn exists(base_path: &Path, uuid: &Uuid) -> bool {
  63         let mut path = base_path.to_owned();
  64         path.push(uuid.to_string());
  65         path.set_extension("log");
  66         path.exists()
  67     }
  68
  69     /// Destroy the media catalog (remove all files)
  70     pub fn destroy(base_path: &Path, uuid: &Uuid) -> Result<(), Error> {
  71
  72         let mut path = base_path.to_owned();
  73         path.push(uuid.to_string());
  74         path.set_extension("log");
  75
  76         match std::fs::remove_file(path) {
  77             Ok(()) => Ok(()),
  78             Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
  79             Err(err) => Err(err.into()),
  80         }
  81     }
  82
  83     fn create_basedir(base_path: &Path) -> Result<(), Error> {
  84         let backup_user = crate::backup::backup_user()?;
  85         let mode = nix::sys::stat::Mode::from_bits_truncate(0o0640);
  86         let opts = CreateOptions::new()
  87             .perm(mode)
  88             .owner(backup_user.uid)
  89             .group(backup_user.gid);
  90
  91         create_path(base_path, None, Some(opts))
  92             .map_err(|err: Error| format_err!("unable to create media catalog dir - {}", err))?;
  93         Ok(())
  94     }
  95
  96     /// Open a catalog database, load into memory
  97     pub fn open(
  98         base_path: &Path,
  99         uuid: &Uuid,
 100         write: bool,
 101         create: bool,
 102     ) -> Result<Self, Error> {
 103
 104         let mut path = base_path.to_owned();
 105         path.push(uuid.to_string());
 106         path.set_extension("log");
 107
 108         let me = proxmox::try_block!({
 109
 110             Self::create_basedir(base_path)?;
 111
 112             let mut file = std::fs::OpenOptions::new()
 113                 .read(true)
 114                 .write(write)
 115                 .create(create)
 116                 .open(&path)?;
 117
 118             let backup_user = crate::backup::backup_user()?;
 119             fchown(file.as_raw_fd(), Some(backup_user.uid), Some(backup_user.gid))
 120                 .map_err(|err| format_err!("fchown failed - {}", err))?;
 121
 122             let mut me = Self {
 123                 uuid: uuid.clone(),
 124                 file: None,
 125                 log_to_stdout: false,
 126                 current_archive: None,
 127                 last_entry: None,
 128                 chunk_index: HashMap::new(),
 129                 snapshot_index: HashMap::new(),
 130                 pending: Vec::new(),
 131             };
 132
 133             let found_magic_number = me.load_catalog(&mut file)?;
 134
 135             if !found_magic_number {
 136                 me.pending.extend(&PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
 137             }
 138
 139             if write {
 140                 me.file = Some(file);
 141             }
 142             Ok(me)
 143         }).map_err(|err: Error| {
 144             format_err!("unable to open media catalog {:?} - {}", path, err)
 145         })?;
 146
 147         Ok(me)
 148     }
 149
 150     /// Creates a temporary, empty catalog database
 151     pub fn create_temporary_database(
 152         base_path: &Path,
 153         media_id: &MediaId,
 154         log_to_stdout: bool,
 155     ) -> Result<Self, Error> {
 156
 157         let uuid = &media_id.label.uuid;
 158
 159         let mut tmp_path = base_path.to_owned();
 160         tmp_path.push(uuid.to_string());
 161         tmp_path.set_extension("tmp");
 162
 163         let me = proxmox::try_block!({
 164
 165             Self::create_basedir(base_path)?;
 166
 167             let file = std::fs::OpenOptions::new()
 168                 .read(true)
 169                 .write(true)
 170                 .create(true)
 171                 .truncate(true)
 172                 .open(&tmp_path)?;
 173
 174             let backup_user = crate::backup::backup_user()?;
 175             fchown(file.as_raw_fd(), Some(backup_user.uid), Some(backup_user.gid))
 176                 .map_err(|err| format_err!("fchown failed - {}", err))?;
 177
 178             let mut me = Self {
 179                 uuid: uuid.clone(),
 180                 file: Some(file),
 181                 log_to_stdout: false,
 182                 current_archive: None,
 183                 last_entry: None,
 184                 chunk_index: HashMap::new(),
 185                 snapshot_index: HashMap::new(),
 186                 pending: Vec::new(),
 187             };
 188
 189             me.log_to_stdout = log_to_stdout;
 190
 191             me.register_label(&media_id.label.uuid, 0)?;
 192
 193             if let Some(ref set) = media_id.media_set_label {
 194                 me.register_label(&set.uuid, 1)?;
 195             }
 196
 197             me.pending.extend(&PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0);
 198             me.commit()?;
 199
 200             Ok(me)
 201         }).map_err(|err: Error| {
 202             format_err!("unable to create temporary media catalog {:?} - {}", tmp_path, err)
 203         })?;
 204
 205         Ok(me)
 206     }
 207
 208     /// Commit or Abort a temporary catalog database
 209     pub fn finish_temporary_database(
 210         base_path: &Path,
 211         uuid: &Uuid,
 212         commit: bool,
 213     ) -> Result<(), Error> {
 214
 215         let mut tmp_path = base_path.to_owned();
 216         tmp_path.push(uuid.to_string());
 217         tmp_path.set_extension("tmp");
 218
 219         if commit {
 220             let mut catalog_path = tmp_path.clone();
 221             catalog_path.set_extension("log");
 222
 223             if let Err(err) = std::fs::rename(&tmp_path, &catalog_path) {
 224                 bail!("Atomic rename catalog {:?} failed - {}", catalog_path, err);
 225             }
 226         } else {
 227             std::fs::remove_file(&tmp_path)?;
 228         }
 229         Ok(())
 230     }
 231
 232     /// Returns the BackupMedia uuid
 233     pub fn uuid(&self) -> &Uuid {
 234         &self.uuid
 235     }
 236
 237     /// Accessor to content list
 238     pub fn snapshot_index(&self) -> &HashMap<String, u64> {
 239         &self.snapshot_index
 240     }
 241
 242     /// Commit pending changes
 243     ///
 244     /// This is necessary to store changes persistently.
 245     ///
 246     /// Fixme: this should be atomic ...
 247     pub fn commit(&mut self) -> Result<(), Error> {
 248
 249         if self.pending.is_empty() {
 250             return Ok(());
 251         }
 252
 253         match self.file {
 254             Some(ref mut file) => {
 255                 file.write_all(&self.pending)?;
 256                 file.flush()?;
 257                 file.sync_data()?;
 258             }
 259             None => bail!("media catalog not writable (opened read only)"),
 260         }
 261
 262         self.pending = Vec::new();
 263
 264         Ok(())
 265     }
 266
 267     /// Conditionally commit if in pending data is large (> 1Mb)
 268     pub fn commit_if_large(&mut self) -> Result<(), Error> {
 269         if self.pending.len() > 1024*1024 {
 270             self.commit()?;
 271         }
 272         Ok(())
 273     }
 274
 275     /// Destroy existing catalog, opens a new one
 276     pub fn overwrite(
 277         base_path: &Path,
 278         media_id: &MediaId,
 279         log_to_stdout: bool,
 280     ) ->  Result<Self, Error> {
 281
 282         let uuid = &media_id.label.uuid;
 283
 284         let me = Self::create_temporary_database(base_path, &media_id, log_to_stdout)?;
 285
 286         Self::finish_temporary_database(base_path, uuid, true)?;
 287
 288         Ok(me)
 289     }
 290
 291     /// Test if the catalog already contain a snapshot
 292     pub fn contains_snapshot(&self, snapshot: &str) -> bool {
 293         self.snapshot_index.contains_key(snapshot)
 294     }
 295
 296     /// Returns the chunk archive file number
 297     pub fn lookup_snapshot(&self, snapshot: &str) -> Option<u64> {
 298         self.snapshot_index.get(snapshot).map(|n| *n)
 299     }
 300
 301     /// Test if the catalog already contain a chunk
 302     pub fn contains_chunk(&self, digest: &[u8;32]) -> bool {
 303         self.chunk_index.contains_key(digest)
 304     }
 305
 306     /// Returns the chunk archive file number
 307     pub fn lookup_chunk(&self, digest: &[u8;32]) -> Option<u64> {
 308         self.chunk_index.get(digest).map(|n| *n)
 309     }
 310
 311     fn check_register_label(&self, file_number: u64) -> Result<(), Error> {
 312
 313         if file_number >= 2 {
 314             bail!("register label failed: got wrong file number ({} >= 2)", file_number);
 315         }
 316
 317         if self.current_archive.is_some() {
 318             bail!("register label failed: inside chunk archive");
 319         }
 320
 321         let expected_file_number = match self.last_entry {
 322             Some((_, last_number)) => last_number + 1,
 323             None => 0,
 324         };
 325
 326         if file_number != expected_file_number {
 327             bail!("register label failed: got unexpected file number ({} < {})",
 328                   file_number, expected_file_number);
 329         }
 330         Ok(())
 331     }
 332
 333     /// Register media labels (file 0 and 1)
 334     pub fn register_label(
 335         &mut self,
 336         uuid: &Uuid, // Uuid form MediaContentHeader
 337         file_number: u64,
 338     ) -> Result<(), Error> {
 339
 340         self.check_register_label(file_number)?;
 341
 342         let entry = LabelEntry {
 343             file_number,
 344             uuid: *uuid.as_bytes(),
 345         };
 346
 347         if self.log_to_stdout {
 348             println!("L|{}|{}", file_number, uuid.to_string());
 349         }
 350
 351         self.pending.push(b'L');
 352
 353         unsafe { self.pending.write_le_value(entry)?; }
 354
 355         self.last_entry = Some((uuid.clone(), file_number));
 356
 357         Ok(())
 358     }
 359
 360     /// Register a chunk
 361     ///
 362     /// Only valid after start_chunk_archive.
 363     pub fn register_chunk(
 364         &mut self,
 365         digest: &[u8;32],
 366     ) -> Result<(), Error> {
 367
 368         let file_number = match self.current_archive {
 369             None => bail!("register_chunk failed: no archive started"),
 370             Some((_, file_number)) => file_number,
 371         };
 372
 373         if self.log_to_stdout {
 374             println!("C|{}", proxmox::tools::digest_to_hex(digest));
 375         }
 376
 377         self.pending.push(b'C');
 378         self.pending.extend(digest);
 379
 380         self.chunk_index.insert(*digest, file_number);
 381
 382         Ok(())
 383     }
 384
 385     fn check_start_chunk_archive(&self, file_number: u64) -> Result<(), Error> {
 386
 387         if self.current_archive.is_some() {
 388             bail!("start_chunk_archive failed: already started");
 389         }
 390
 391         if file_number < 2 {
 392             bail!("start_chunk_archive failed: got wrong file number ({} < 2)", file_number);
 393         }
 394
 395         let expect_min_file_number = match self.last_entry {
 396             Some((_, last_number)) => last_number + 1,
 397             None => 0,
 398         };
 399
 400         if file_number < expect_min_file_number {
 401             bail!("start_chunk_archive: got unexpected file number ({} < {})",
 402                   file_number, expect_min_file_number);
 403         }
 404
 405         Ok(())
 406     }
 407
 408     /// Start a chunk archive section
 409     pub fn start_chunk_archive(
 410         &mut self,
 411         uuid: Uuid, // Uuid form MediaContentHeader
 412         file_number: u64,
 413     ) -> Result<(), Error> {
 414
 415         self.check_start_chunk_archive(file_number)?;
 416
 417         let entry = ChunkArchiveStart {
 418             file_number,
 419             uuid: *uuid.as_bytes(),
 420         };
 421
 422         if self.log_to_stdout {
 423             println!("A|{}|{}", file_number, uuid.to_string());
 424         }
 425
 426         self.pending.push(b'A');
 427
 428         unsafe { self.pending.write_le_value(entry)?; }
 429
 430         self.current_archive = Some((uuid, file_number));
 431
 432         Ok(())
 433     }
 434
 435     fn check_end_chunk_archive(&self, uuid: &Uuid, file_number: u64) -> Result<(), Error> {
 436
 437         match self.current_archive {
 438             None => bail!("end_chunk archive failed: not started"),
 439             Some((ref expected_uuid, expected_file_number)) => {
 440                 if uuid != expected_uuid {
 441                     bail!("end_chunk_archive failed: got unexpected uuid");
 442                 }
 443                 if file_number != expected_file_number {
 444                     bail!("end_chunk_archive failed: got unexpected file number ({} != {})",
 445                           file_number, expected_file_number);
 446                 }
 447             }
 448         }
 449
 450         Ok(())
 451     }
 452
 453     /// End a chunk archive section
 454     pub fn end_chunk_archive(&mut self) -> Result<(), Error> {
 455
 456         match self.current_archive.take() {
 457             None => bail!("end_chunk_archive failed: not started"),
 458             Some((uuid, file_number)) => {
 459
 460                 let entry = ChunkArchiveEnd {
 461                     file_number,
 462                     uuid: *uuid.as_bytes(),
 463                 };
 464
 465                 if self.log_to_stdout {
 466                     println!("E|{}|{}\n", file_number, uuid.to_string());
 467                 }
 468
 469                 self.pending.push(b'E');
 470
 471                 unsafe { self.pending.write_le_value(entry)?; }
 472
 473                 self.last_entry = Some((uuid, file_number));
 474             }
 475         }
 476
 477         Ok(())
 478     }
 479
 480     fn check_register_snapshot(&self, file_number: u64, snapshot: &str) -> Result<(), Error> {
 481
 482         if self.current_archive.is_some() {
 483             bail!("register_snapshot failed: inside chunk_archive");
 484         }
 485
 486         if file_number < 2 {
 487             bail!("register_snapshot failed: got wrong file number ({} < 2)", file_number);
 488         }
 489
 490         let expect_min_file_number = match self.last_entry {
 491             Some((_, last_number)) => last_number + 1,
 492             None => 0,
 493         };
 494
 495         if file_number < expect_min_file_number {
 496             bail!("register_snapshot failed: got unexpected file number ({} < {})",
 497                   file_number, expect_min_file_number);
 498         }
 499
 500         if let Err(err) = snapshot.parse::<BackupDir>() {
 501             bail!("register_snapshot failed: unable to parse snapshot '{}' - {}", snapshot, err);
 502         }
 503
 504         Ok(())
 505     }
 506
 507     /// Register a snapshot
 508     pub fn register_snapshot(
 509         &mut self,
 510         uuid: Uuid, // Uuid form MediaContentHeader
 511         file_number: u64,
 512         snapshot: &str,
 513     ) -> Result<(), Error> {
 514
 515         self.check_register_snapshot(file_number, snapshot)?;
 516
 517         let entry = SnapshotEntry {
 518             file_number,
 519             uuid: *uuid.as_bytes(),
 520             name_len: u16::try_from(snapshot.len())?,
 521         };
 522
 523         if self.log_to_stdout {
 524             println!("S|{}|{}|{}", file_number, uuid.to_string(), snapshot);
 525         }
 526
 527         self.pending.push(b'S');
 528
 529         unsafe { self.pending.write_le_value(entry)?; }
 530         self.pending.extend(snapshot.as_bytes());
 531
 532         self.snapshot_index.insert(snapshot.to_string(), file_number);
 533
 534         self.last_entry = Some((uuid, file_number));
 535
 536         Ok(())
 537     }
 538
 539     fn load_catalog(&mut self, file: &mut File) -> Result<bool, Error> {
 540
 541         let mut file = BufReader::new(file);
 542         let mut found_magic_number = false;
 543
 544         loop {
 545             let pos = file.seek(SeekFrom::Current(0))?;
 546
 547             if pos == 0 { // read/check magic number
 548                 let mut magic = [0u8; 8];
 549                 match file.read_exact_or_eof(&mut magic) {
 550                     Ok(false) => { /* EOF */ break; }
 551                     Ok(true) => { /* OK */ }
 552                     Err(err) => bail!("read failed - {}", err),
 553                 }
 554                 if magic != PROXMOX_BACKUP_MEDIA_CATALOG_MAGIC_1_0 {
 555                     bail!("wrong magic number");
 556                 }
 557                 found_magic_number = true;
 558                 continue;
 559             }
 560
 561             let mut entry_type = [0u8; 1];
 562             match file.read_exact_or_eof(&mut entry_type) {
 563                 Ok(false) => { /* EOF */ break; }
 564                 Ok(true) => { /* OK */ }
 565                 Err(err) => bail!("read failed - {}", err),
 566             }
 567
 568             match entry_type[0] {
 569                 b'C' => {
 570                     let file_number = match self.current_archive {
 571                         None => bail!("register_chunk failed: no archive started"),
 572                         Some((_, file_number)) => file_number,
 573                     };
 574                     let mut digest = [0u8; 32];
 575                     file.read_exact(&mut digest)?;
 576                     self.chunk_index.insert(digest, file_number);
 577                 }
 578                 b'A' => {
 579                     let entry: ChunkArchiveStart = unsafe { file.read_le_value()? };
 580                     let file_number = entry.file_number;
 581                     let uuid = Uuid::from(entry.uuid);
 582
 583                     self.check_start_chunk_archive(file_number)?;
 584
 585                     self.current_archive = Some((uuid, file_number));
 586                 }
 587                 b'E' => {
 588                     let entry: ChunkArchiveEnd = unsafe { file.read_le_value()? };
 589                     let file_number = entry.file_number;
 590                     let uuid = Uuid::from(entry.uuid);
 591
 592                     self.check_end_chunk_archive(&uuid, file_number)?;
 593
 594                     self.current_archive = None;
 595                     self.last_entry = Some((uuid, file_number));
 596                 }
 597                 b'S' => {
 598                     let entry: SnapshotEntry = unsafe { file.read_le_value()? };
 599                     let file_number = entry.file_number;
 600                     let name_len = entry.name_len;
 601                     let uuid = Uuid::from(entry.uuid);
 602
 603                     let snapshot = file.read_exact_allocated(name_len.into())?;
 604                     let snapshot = std::str::from_utf8(&snapshot)?;
 605
 606                     self.check_register_snapshot(file_number, snapshot)?;
 607
 608                     self.snapshot_index.insert(snapshot.to_string(), file_number);
 609
 610                     self.last_entry = Some((uuid, file_number));
 611                 }
 612                 b'L' => {
 613                     let entry: LabelEntry = unsafe { file.read_le_value()? };
 614                     let file_number = entry.file_number;
 615                     let uuid = Uuid::from(entry.uuid);
 616
 617                     self.check_register_label(file_number)?;
 618
 619                     self.last_entry = Some((uuid, file_number));
 620                 }
 621                 _ => {
 622                     bail!("unknown entry type '{}'", entry_type[0]);
 623                 }
 624             }
 625
 626         }
 627
 628         Ok(found_magic_number)
 629     }
 630 }
 631
 632 /// Media set catalog
 633 ///
 634 /// Catalog for multiple media.
 635 pub struct MediaSetCatalog  {
 636     catalog_list: HashMap<Uuid, MediaCatalog>,
 637 }
 638
 639 impl MediaSetCatalog {
 640
 641     /// Creates a new instance
 642     pub fn new() -> Self {
 643         Self {
 644             catalog_list: HashMap::new(),
 645         }
 646     }
 647
 648     /// Add a catalog
 649     pub fn append_catalog(&mut self, catalog: MediaCatalog) -> Result<(), Error> {
 650
 651         if self.catalog_list.get(&catalog.uuid).is_some() {
 652             bail!("MediaSetCatalog already contains media '{}'", catalog.uuid);
 653         }
 654
 655         self.catalog_list.insert(catalog.uuid.clone(), catalog);
 656
 657         Ok(())
 658     }
 659
 660     /// Remove a catalog
 661     pub fn remove_catalog(&mut self, media_uuid: &Uuid) {
 662         self.catalog_list.remove(media_uuid);
 663     }
 664
 665     /// Test if the catalog already contain a snapshot
 666     pub fn contains_snapshot(&self, snapshot: &str) -> bool {
 667         for catalog in self.catalog_list.values() {
 668             if catalog.contains_snapshot(snapshot) {
 669                 return true;
 670             }
 671         }
 672         false
 673     }
 674
 675     /// Test if the catalog already contain a chunk
 676     pub fn contains_chunk(&self, digest: &[u8;32]) -> bool {
 677         for catalog in self.catalog_list.values() {
 678             if catalog.contains_chunk(digest) {
 679                 return true;
 680             }
 681         }
 682         false
 683     }
 684 }
 685
 686 // Type definitions for internal binary catalog encoding
 687
 688 #[derive(Endian)]
 689 #[repr(C)]
 690 struct LabelEntry {
 691     file_number: u64,
 692     uuid: [u8;16],
 693 }
 694
 695 #[derive(Endian)]
 696 #[repr(C)]
 697 struct ChunkArchiveStart {
 698     file_number: u64,
 699     uuid: [u8;16],
 700 }
 701
 702 #[derive(Endian)]
 703 #[repr(C)]
 704 struct ChunkArchiveEnd{
 705     file_number: u64,
 706     uuid: [u8;16],
 707 }
 708
 709 #[derive(Endian)]
 710 #[repr(C)]
 711 struct SnapshotEntry{
 712     file_number: u64,
 713     uuid: [u8;16],
 714     name_len: u16,
 715     /* snapshot name follows */
 716 }