src/tape/file_formats/chunk_archive.rs

   1 use std::io::Read;
   2
   3 use anyhow::{bail, Error};
   4 use endian_trait::Endian;
   5
   6 use proxmox_io::ReadExt;
   7 use proxmox_uuid::Uuid;
   8
   9 use pbs_datastore::DataBlob;
  10 use pbs_tape::{MediaContentHeader, TapeWrite, PROXMOX_TAPE_BLOCK_SIZE};
  11
  12 use crate::tape::file_formats::{
  13     ChunkArchiveEntryHeader, ChunkArchiveHeader, PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
  14     PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1,
  15 };
  16
  17 /// Writes chunk archives to tape.
  18 ///
  19 /// A chunk archive consists of a `MediaContentHeader` followed by a
  20 /// list of chunks entries. Each chunk entry consists of a
  21 /// `ChunkArchiveEntryHeader` followed by the chunk data (`DataBlob`).
  22 ///
  23 /// `| MediaContentHeader | ( ChunkArchiveEntryHeader | DataBlob )* |`
  24 pub struct ChunkArchiveWriter<'a> {
  25     writer: Option<Box<dyn TapeWrite + 'a>>,
  26     bytes_written: usize, // does not include bytes from current writer
  27     close_on_leom: bool,
  28 }
  29
  30 impl<'a> ChunkArchiveWriter<'a> {
  31     pub const MAGIC: [u8; 8] = PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1;
  32
  33     /// Creates a new instance
  34     pub fn new(
  35         mut writer: Box<dyn TapeWrite + 'a>,
  36         store: &str,
  37         close_on_leom: bool,
  38     ) -> Result<(Self, Uuid), Error> {
  39         let archive_header = ChunkArchiveHeader {
  40             store: store.to_string(),
  41         };
  42         let header_data = serde_json::to_string_pretty(&archive_header)?
  43             .as_bytes()
  44             .to_vec();
  45
  46         let header = MediaContentHeader::new(Self::MAGIC, header_data.len() as u32);
  47         writer.write_header(&header, &header_data)?;
  48
  49         let me = Self {
  50             writer: Some(writer),
  51             bytes_written: 0,
  52             close_on_leom,
  53         };
  54
  55         Ok((me, header.uuid.into()))
  56     }
  57
  58     /// Returns the number of bytes written so far.
  59     pub fn bytes_written(&self) -> usize {
  60         match self.writer {
  61             Some(ref writer) => writer.bytes_written(),
  62             None => self.bytes_written, // finalize sets this
  63         }
  64     }
  65
  66     fn write_all(&mut self, data: &[u8]) -> Result<bool, std::io::Error> {
  67         match self.writer {
  68             Some(ref mut writer) => writer.write_all(data),
  69             None => {
  70                 proxmox_lang::io_bail!("detected write after archive finished - internal error")
  71             }
  72         }
  73     }
  74
  75     /// Write chunk into archive.
  76     ///
  77     /// This may return false when `LEOM` is detected (when close_on_leom is set).
  78     /// In that case the archive only contains parts of the last chunk.
  79     pub fn try_write_chunk(
  80         &mut self,
  81         digest: &[u8; 32],
  82         blob: &DataBlob,
  83     ) -> Result<bool, std::io::Error> {
  84         if self.writer.is_none() {
  85             return Ok(false);
  86         }
  87
  88         let head = ChunkArchiveEntryHeader {
  89             magic: PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
  90             digest: *digest,
  91             size: blob.raw_size(),
  92         };
  93
  94         let head = head.to_le();
  95         let data = unsafe {
  96             std::slice::from_raw_parts(
  97                 &head as *const ChunkArchiveEntryHeader as *const u8,
  98                 std::mem::size_of::<ChunkArchiveEntryHeader>(),
  99             )
 100         };
 101
 102         self.write_all(data)?;
 103
 104         let mut start = 0;
 105         let blob_data = blob.raw_data();
 106         loop {
 107             if start >= blob_data.len() {
 108                 break;
 109             }
 110
 111             let end = start + PROXMOX_TAPE_BLOCK_SIZE;
 112             let mut chunk_is_complete = false;
 113             let leom = if end > blob_data.len() {
 114                 chunk_is_complete = true;
 115                 self.write_all(&blob_data[start..])?
 116             } else {
 117                 self.write_all(&blob_data[start..end])?
 118             };
 119             if leom && self.close_on_leom {
 120                 let mut writer = self.writer.take().unwrap();
 121                 writer.finish(false)?;
 122                 self.bytes_written = writer.bytes_written();
 123                 return Ok(chunk_is_complete);
 124             }
 125             start = end;
 126         }
 127
 128         Ok(true)
 129     }
 130
 131     /// This must be called at the end to add padding and `EOF`
 132     ///
 133     /// Returns true on `LEOM` or when we hit max archive size
 134     pub fn finish(&mut self) -> Result<bool, std::io::Error> {
 135         match self.writer.take() {
 136             Some(mut writer) => {
 137                 self.bytes_written = writer.bytes_written();
 138                 writer.finish(false)
 139             }
 140             None => Ok(true),
 141         }
 142     }
 143 }
 144
 145 /// Read chunk archives.
 146 pub struct ChunkArchiveDecoder<R> {
 147     reader: R,
 148 }
 149
 150 impl<R: Read> ChunkArchiveDecoder<R> {
 151     /// Creates a new instance
 152     pub fn new(reader: R) -> Self {
 153         Self { reader }
 154     }
 155
 156     /// Allow access to the underlying reader
 157     pub fn reader(&self) -> &R {
 158         &self.reader
 159     }
 160
 161     /// Returns the next chunk (if any).
 162     pub fn next_chunk(&mut self) -> Result<Option<([u8; 32], DataBlob)>, Error> {
 163         let mut header = ChunkArchiveEntryHeader {
 164             magic: [0u8; 8],
 165             digest: [0u8; 32],
 166             size: 0,
 167         };
 168         let data = unsafe {
 169             std::slice::from_raw_parts_mut(
 170                 (&mut header as *mut ChunkArchiveEntryHeader) as *mut u8,
 171                 std::mem::size_of::<ChunkArchiveEntryHeader>(),
 172             )
 173         };
 174
 175         match self.reader.read_exact_or_eof(data) {
 176             Ok(true) => {}
 177             Ok(false) => {
 178                 // last chunk is allowed to be incomplete - simply report EOD
 179                 return Ok(None);
 180             }
 181             Err(err) => return Err(err.into()),
 182         };
 183
 184         if header.magic != PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0 {
 185             bail!("wrong magic number");
 186         }
 187
 188         let raw_data = match self.reader.read_exact_allocated(header.size as usize) {
 189             Ok(data) => data,
 190             Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => {
 191                 // last chunk is allowed to be incomplete - simply report EOD
 192                 return Ok(None);
 193             }
 194             Err(err) => return Err(err.into()),
 195         };
 196
 197         let blob = DataBlob::from_raw(raw_data)?;
 198         blob.verify_crc()?;
 199
 200         Ok(Some((header.digest, blob)))
 201     }
 202 }