src/tape/file_formats/chunk_archive.rs

   1 use std::io::Read;
   2
   3 use anyhow::{bail, Error};
   4 use endian_trait::Endian;
   5
   6 use proxmox_io::ReadExt;
   7 use proxmox_uuid::Uuid;
   8
   9 use pbs_datastore::DataBlob;
  10 use pbs_tape::{
  11     PROXMOX_TAPE_BLOCK_SIZE,
  12     TapeWrite, MediaContentHeader,
  13 };
  14
  15 use crate::tape::file_formats::{
  16     PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1,
  17     PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
  18     ChunkArchiveHeader,
  19     ChunkArchiveEntryHeader,
  20 };
  21
  22 /// Writes chunk archives to tape.
  23 ///
  24 /// A chunk archive consists of a `MediaContentHeader` followed by a
  25 /// list of chunks entries. Each chunk entry consists of a
  26 /// `ChunkArchiveEntryHeader` followed by the chunk data (`DataBlob`).
  27 ///
  28 /// `| MediaContentHeader | ( ChunkArchiveEntryHeader | DataBlob )* |`
  29 pub struct ChunkArchiveWriter<'a> {
  30     writer: Option<Box<dyn TapeWrite + 'a>>,
  31     bytes_written: usize, // does not include bytes from current writer
  32     close_on_leom: bool,
  33 }
  34
  35 impl <'a> ChunkArchiveWriter<'a> {
  36
  37     pub const MAGIC: [u8; 8] = PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_1;
  38
  39     /// Creates a new instance
  40     pub fn new(
  41         mut writer: Box<dyn TapeWrite + 'a>,
  42         store: &str,
  43         close_on_leom: bool,
  44     ) -> Result<(Self,Uuid), Error> {
  45
  46         let archive_header = ChunkArchiveHeader { store: store.to_string() };
  47         let header_data = serde_json::to_string_pretty(&archive_header)?.as_bytes().to_vec();
  48
  49         let header = MediaContentHeader::new(Self::MAGIC, header_data.len() as u32);
  50         writer.write_header(&header, &header_data)?;
  51
  52         let me = Self {
  53             writer: Some(writer),
  54             bytes_written: 0,
  55             close_on_leom,
  56         };
  57
  58         Ok((me, header.uuid.into()))
  59     }
  60
  61     /// Returns the number of bytes written so far.
  62     pub fn bytes_written(&self) -> usize {
  63         match self.writer {
  64             Some(ref writer) => writer.bytes_written(),
  65             None => self.bytes_written, // finalize sets this
  66         }
  67     }
  68
  69     fn write_all(&mut self, data: &[u8]) -> Result<bool, std::io::Error> {
  70         match self.writer {
  71             Some(ref mut writer) => writer.write_all(data),
  72             None => proxmox::io_bail!(
  73                 "detected write after archive finished - internal error"),
  74         }
  75     }
  76
  77     /// Write chunk into archive.
  78     ///
  79     /// This may return false when `LEOM` is detected (when close_on_leom is set).
  80     /// In that case the archive only contains parts of the last chunk.
  81     pub fn try_write_chunk(
  82         &mut self,
  83         digest: &[u8;32],
  84         blob: &DataBlob,
  85     ) -> Result<bool, std::io::Error> {
  86
  87         if self.writer.is_none() {
  88             return Ok(false);
  89         }
  90
  91         let head = ChunkArchiveEntryHeader {
  92             magic: PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
  93             digest: *digest,
  94             size: blob.raw_size(),
  95         };
  96
  97         let head = head.to_le();
  98         let data = unsafe { std::slice::from_raw_parts(
  99             &head as *const ChunkArchiveEntryHeader as *const u8,
 100             std::mem::size_of::<ChunkArchiveEntryHeader>())
 101         };
 102
 103         self.write_all(data)?;
 104
 105         let mut start = 0;
 106         let blob_data = blob.raw_data();
 107         loop {
 108             if start >= blob_data.len() {
 109                 break;
 110             }
 111
 112             let end = start + PROXMOX_TAPE_BLOCK_SIZE;
 113             let mut chunk_is_complete = false;
 114             let leom = if end > blob_data.len() {
 115                 chunk_is_complete = true;
 116                 self.write_all(&blob_data[start..])?
 117             } else {
 118                 self.write_all(&blob_data[start..end])?
 119             };
 120             if leom {
 121                 if self.close_on_leom {
 122                     let mut writer = self.writer.take().unwrap();
 123                     writer.finish(false)?;
 124                     self.bytes_written = writer.bytes_written();
 125                     return Ok(chunk_is_complete);
 126                 }
 127             }
 128             start = end;
 129         }
 130
 131         Ok(true)
 132     }
 133
 134     /// This must be called at the end to add padding and `EOF`
 135     ///
 136     /// Returns true on `LEOM` or when we hit max archive size
 137     pub fn finish(&mut self) -> Result<bool, std::io::Error> {
 138         match self.writer.take() {
 139             Some(mut writer) => {
 140                 self.bytes_written = writer.bytes_written();
 141                 writer.finish(false)
 142             }
 143             None => Ok(true),
 144         }
 145     }
 146 }
 147
 148 /// Read chunk archives.
 149 pub struct ChunkArchiveDecoder<R> {
 150     reader: R,
 151 }
 152
 153 impl <R: Read> ChunkArchiveDecoder<R> {
 154
 155     /// Creates a new instance
 156     pub fn new(reader: R) -> Self {
 157         Self { reader }
 158     }
 159
 160     /// Allow access to the underlying reader
 161     pub fn reader(&self) -> &R {
 162         &self.reader
 163     }
 164
 165     /// Returns the next chunk (if any).
 166     pub fn next_chunk(&mut self) -> Result<Option<([u8;32], DataBlob)>, Error> {
 167
 168         let mut header = ChunkArchiveEntryHeader {
 169             magic: [0u8; 8],
 170             digest: [0u8; 32],
 171             size: 0,
 172         };
 173         let data = unsafe {
 174             std::slice::from_raw_parts_mut(
 175                 (&mut header as *mut ChunkArchiveEntryHeader) as *mut u8,
 176                 std::mem::size_of::<ChunkArchiveEntryHeader>())
 177         };
 178
 179         match self.reader.read_exact_or_eof(data) {
 180             Ok(true) => {},
 181             Ok(false) => {
 182                 // last chunk is allowed to be incomplete - simply report EOD
 183                 return Ok(None);
 184             }
 185             Err(err) => return Err(err.into()),
 186         };
 187
 188         if header.magic != PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0 {
 189             bail!("wrong magic number");
 190         }
 191
 192          let raw_data = match self.reader.read_exact_allocated(header.size as usize) {
 193             Ok(data) => data,
 194             Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => {
 195                 // last chunk is allowed to be incomplete - simply report EOD
 196                 return Ok(None);
 197             }
 198             Err(err) => return Err(err.into()),
 199         };
 200
 201         let blob = DataBlob::from_raw(raw_data)?;
 202         blob.verify_crc()?;
 203
 204         Ok(Some((header.digest, blob)))
 205     }
 206 }