TapeRead,
MediaId,
MediaCatalog,
- ChunkArchiveDecoder,
MediaPool,
Inventory,
file_formats::{
PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0,
MediaContentHeader,
+ ChunkArchiveDecoder,
},
drive::{
TapeDriver,
complete_media_set_uuid,
file_formats::{
PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
- PROXMOX_BACKUP_CONTENT_NAME,
MediaContentHeader,
+ proxmox_tape_magic_to_text,
},
},
};
Ok(header) => {
if header.magic != PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0 {
println!("got MediaContentHeader with wrong magic: {:?}", header.magic);
- } else if let Some(name) = PROXMOX_BACKUP_CONTENT_NAME.get(&header.content_magic) {
+ } else if let Some(name) = proxmox_tape_magic_to_text(&header.content_magic) {
println!("got content header: {}", name);
println!(" uuid: {}", header.content_uuid());
println!(" ctime: {}", strftime_local("%c", header.ctime)?);
+++ /dev/null
-use std::io::Read;
-
-use anyhow::{bail, Error};
-use endian_trait::Endian;
-
-use proxmox::tools::{
- Uuid,
- io::ReadExt,
-};
-
-use crate::backup::DataBlob;
-
-use crate::tape::{
- TapeWrite,
- file_formats::{
- PROXMOX_TAPE_BLOCK_SIZE,
- PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0,
- PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
- MediaContentHeader,
- ChunkArchiveEntryHeader,
- },
-};
-
-/// Writes chunk archives to tape.
-///
-/// A chunk archive consists of a `MediaContentHeader` followed by a
-/// list of chunks entries. Each chunk entry consists of a
-/// `ChunkArchiveEntryHeader` folowed by the chunk data (`DataBlob`).
-///
-/// `| MediaContentHeader | ( ChunkArchiveEntryHeader | DataBlob )* |`
-pub struct ChunkArchiveWriter<'a> {
- writer: Option<Box<dyn TapeWrite + 'a>>,
- bytes_written: usize, // does not include bytes from current writer
- close_on_leom: bool,
-}
-
-impl <'a> ChunkArchiveWriter<'a> {
-
- pub const MAGIC: [u8; 8] = PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0;
-
- /// Creates a new instance
- pub fn new(mut writer: Box<dyn TapeWrite + 'a>, close_on_leom: bool) -> Result<(Self,Uuid), Error> {
-
- let header = MediaContentHeader::new(Self::MAGIC, 0);
- writer.write_header(&header, &[])?;
-
- let me = Self {
- writer: Some(writer),
- bytes_written: 0,
- close_on_leom,
- };
-
- Ok((me, header.uuid.into()))
- }
-
- /// Returns the number of bytes written so far.
- pub fn bytes_written(&self) -> usize {
- match self.writer {
- Some(ref writer) => writer.bytes_written(),
- None => self.bytes_written, // finalize sets this
- }
- }
-
- fn write_all(&mut self, data: &[u8]) -> Result<bool, std::io::Error> {
- match self.writer {
- Some(ref mut writer) => writer.write_all(data),
- None => proxmox::io_bail!(
- "detected write after archive finished - internal error"),
- }
- }
-
- /// Write chunk into archive.
- ///
- /// This may return false when `LEOM` is detected (when close_on_leom is set).
- /// In that case the archive only contains parts of the last chunk.
- pub fn try_write_chunk(
- &mut self,
- digest: &[u8;32],
- blob: &DataBlob,
- ) -> Result<bool, std::io::Error> {
-
- if self.writer.is_none() {
- return Ok(false);
- }
-
- let head = ChunkArchiveEntryHeader {
- magic: PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
- digest: *digest,
- size: blob.raw_size(),
- };
-
- let head = head.to_le();
- let data = unsafe { std::slice::from_raw_parts(
- &head as *const ChunkArchiveEntryHeader as *const u8,
- std::mem::size_of::<ChunkArchiveEntryHeader>())
- };
-
- self.write_all(data)?;
-
- let mut start = 0;
- let blob_data = blob.raw_data();
- loop {
- if start >= blob_data.len() {
- break;
- }
-
- let end = start + PROXMOX_TAPE_BLOCK_SIZE;
- let mut chunk_is_complete = false;
- let leom = if end > blob_data.len() {
- chunk_is_complete = true;
- self.write_all(&blob_data[start..])?
- } else {
- self.write_all(&blob_data[start..end])?
- };
- if leom {
- println!("WRITE DATA LEOM at pos {}", self.bytes_written());
- if self.close_on_leom {
- let mut writer = self.writer.take().unwrap();
- writer.finish(false)?;
- self.bytes_written = writer.bytes_written();
- return Ok(chunk_is_complete);
- }
- }
- start = end;
- }
-
- Ok(true)
- }
-
- /// This must be called at the end to add padding and `EOF`
- ///
- /// Returns true on `LEOM` or when we hit max archive size
- pub fn finish(&mut self) -> Result<bool, std::io::Error> {
- match self.writer.take() {
- Some(mut writer) => {
- self.bytes_written = writer.bytes_written();
- writer.finish(false)
- }
- None => Ok(true),
- }
- }
-}
-
-/// Read chunk archives.
-pub struct ChunkArchiveDecoder<R> {
- reader: R,
-}
-
-impl <R: Read> ChunkArchiveDecoder<R> {
-
- /// Creates a new instance
- pub fn new(reader: R) -> Self {
- Self { reader }
- }
-
- /// Allow access to the underyling reader
- pub fn reader(&self) -> &R {
- &self.reader
- }
-
- /// Returns the next chunk (if any).
- pub fn next_chunk(&mut self) -> Result<Option<([u8;32], DataBlob)>, Error> {
-
- let mut header = ChunkArchiveEntryHeader {
- magic: [0u8; 8],
- digest: [0u8; 32],
- size: 0,
- };
- let data = unsafe {
- std::slice::from_raw_parts_mut(
- (&mut header as *mut ChunkArchiveEntryHeader) as *mut u8,
- std::mem::size_of::<ChunkArchiveEntryHeader>())
- };
-
- match self.reader.read_exact_or_eof(data) {
- Ok(true) => {},
- Ok(false) => {
- // last chunk is allowed to be incomplete - simply report EOD
- return Ok(None);
- }
- Err(err) => return Err(err.into()),
- };
-
- if header.magic != PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0 {
- bail!("wrong magic number");
- }
-
- let raw_data = match self.reader.read_exact_allocated(header.size as usize) {
- Ok(data) => data,
- Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => {
- // last chunk is allowed to be incomplete - simply report EOD
- return Ok(None);
- }
- Err(err) => return Err(err.into()),
- };
-
- let blob = DataBlob::from_raw(raw_data)?;
- blob.verify_crc()?;
-
- Ok(Some((header.digest, blob)))
- }
-}
},
file_formats::{
PROXMOX_TAPE_BLOCK_SIZE,
+ PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
MediaSetLabel,
MediaContentHeader,
- PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
- },
- helpers::{
BlockedReader,
BlockedWriter,
},
- }
+ },
};
fn run_sg_tape_cmd(subcmd: &str, args: &[&str], fd: RawFd) -> Result<String, Error> {
MediaSetLabel,
MediaContentHeader,
PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0,
+ BlockedReader,
+ BlockedWriter,
},
helpers::{
EmulateTapeReader,
EmulateTapeWriter,
- BlockedReader,
- BlockedWriter,
},
},
};
+++ /dev/null
-//! File format definitions for data written to tapes
-
-use std::collections::HashMap;
-
-use anyhow::{bail, Error};
-use ::serde::{Deserialize, Serialize};
-use endian_trait::Endian;
-use bitflags::bitflags;
-
-use proxmox::tools::Uuid;
-
-use crate::backup::Fingerprint;
-
-/// We use 256KB blocksize (always)
-pub const PROXMOX_TAPE_BLOCK_SIZE: usize = 256*1024;
-
-// openssl::sha::sha256(b"Proxmox Tape Block Header v1.0")[0..8]
-pub const PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0: [u8; 8] = [220, 189, 175, 202, 235, 160, 165, 40];
-
-// openssl::sha::sha256(b"Proxmox Backup Content Header v1.0")[0..8];
-pub const PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0: [u8; 8] = [99, 238, 20, 159, 205, 242, 155, 12];
-// openssl::sha::sha256(b"Proxmox Backup Tape Label v1.0")[0..8];
-pub const PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0: [u8; 8] = [42, 5, 191, 60, 176, 48, 170, 57];
-// openssl::sha::sha256(b"Proxmox Backup MediaSet Label v1.0")
-pub const PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0: [u8; 8] = [8, 96, 99, 249, 47, 151, 83, 216];
-
-// openssl::sha::sha256(b"Proxmox Backup Chunk Archive v1.0")[0..8]
-pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0: [u8; 8] = [62, 173, 167, 95, 49, 76, 6, 110];
-// openssl::sha::sha256(b"Proxmox Backup Chunk Archive Entry v1.0")[0..8]
-pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0: [u8; 8] = [72, 87, 109, 242, 222, 66, 143, 220];
-
-// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.0")[0..8];
-pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0: [u8; 8] = [9, 182, 2, 31, 125, 232, 114, 133];
-
-lazy_static::lazy_static!{
- /// Map content Uuid to human readable names.
- pub static ref PROXMOX_BACKUP_CONTENT_NAME: HashMap<&'static [u8;8], &'static str> = {
- let mut map = HashMap::new();
- map.insert(&PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0, "Proxmox Backup Tape Label v1.0");
- map.insert(&PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, "Proxmox Backup MediaSet Label v1.0");
- map.insert(&PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0, "Proxmox Backup Chunk Archive v1.0");
- map.insert(&PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, "Proxmox Backup Snapshot Archive v1.0");
- map
- };
-}
-
-/// Tape Block Header with data payload
-///
-/// All tape files are written as sequence of blocks.
-///
-/// Note: this struct is large, never put this on the stack!
-/// so we use an unsized type to avoid that.
-///
-/// Tape data block are always read/written with a fixed size
-/// (`PROXMOX_TAPE_BLOCK_SIZE`). But they may contain less data, so the
-/// header has an additional size field. For streams of blocks, there
-/// is a sequence number (`seq_nr`) which may be use for additional
-/// error checking.
-#[repr(C,packed)]
-pub struct BlockHeader {
- /// fixed value `PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0`
- pub magic: [u8; 8],
- pub flags: BlockHeaderFlags,
- /// size as 3 bytes unsigned, little endian
- pub size: [u8; 3],
- /// block sequence number
- pub seq_nr: u32,
- pub payload: [u8],
-}
-
-bitflags! {
- /// Header flags (e.g. `END_OF_STREAM` or `INCOMPLETE`)
- pub struct BlockHeaderFlags: u8 {
- /// Marks the last block in a stream.
- const END_OF_STREAM = 0b00000001;
- /// Mark multivolume streams (when set in the last block)
- const INCOMPLETE = 0b00000010;
- }
-}
-
-#[derive(Endian, Copy, Clone, Debug)]
-#[repr(C,packed)]
-/// Media Content Header
-///
-/// All tape files start with this header. The header may contain some
-/// informational data indicated by `size`.
-///
-/// `| MediaContentHeader | header data (size) | stream data |`
-///
-/// Note: The stream data following may be of any size.
-pub struct MediaContentHeader {
- /// fixed value `PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0`
- pub magic: [u8; 8],
- /// magic number for the content following
- pub content_magic: [u8; 8],
- /// unique ID to identify this data stream
- pub uuid: [u8; 16],
- /// stream creation time
- pub ctime: i64,
- /// Size of header data
- pub size: u32,
- /// Part number for multipart archives.
- pub part_number: u8,
- /// Reserved for future use
- pub reserved_0: u8,
- /// Reserved for future use
- pub reserved_1: u8,
- /// Reserved for future use
- pub reserved_2: u8,
-}
-
-impl MediaContentHeader {
-
- /// Create a new instance with autogenerated Uuid
- pub fn new(content_magic: [u8; 8], size: u32) -> Self {
- let uuid = *proxmox::tools::uuid::Uuid::generate()
- .into_inner();
- Self {
- magic: PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
- content_magic,
- uuid,
- ctime: proxmox::tools::time::epoch_i64(),
- size,
- part_number: 0,
- reserved_0: 0,
- reserved_1: 0,
- reserved_2: 0,
- }
- }
-
- /// Helper to check magic numbers and size constraints
- pub fn check(&self, content_magic: [u8; 8], min_size: u32, max_size: u32) -> Result<(), Error> {
- if self.magic != PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0 {
- bail!("MediaContentHeader: wrong magic");
- }
- if self.content_magic != content_magic {
- bail!("MediaContentHeader: wrong content magic");
- }
- if self.size < min_size || self.size > max_size {
- bail!("MediaContentHeader: got unexpected size");
- }
- Ok(())
- }
-
- /// Returns the content Uuid
- pub fn content_uuid(&self) -> Uuid {
- Uuid::from(self.uuid)
- }
-}
-
-#[derive(Endian)]
-#[repr(C,packed)]
-/// Header for data blobs inside a chunk archive
-pub struct ChunkArchiveEntryHeader {
- /// fixed value `PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0`
- pub magic: [u8; 8],
- /// Chunk digest
- pub digest: [u8; 32],
- /// Chunk size
- pub size: u64,
-}
-
-#[derive(Serialize,Deserialize,Clone,Debug)]
-/// Media Label
-///
-/// Media labels are used to uniquely identify a media. They are
-/// stored as first file on the tape.
-pub struct MediaLabel {
- /// Unique ID
- pub uuid: Uuid,
- /// Media label text (or Barcode)
- pub label_text: String,
- /// Creation time stamp
- pub ctime: i64,
-}
-
-
-#[derive(Serialize,Deserialize,Clone,Debug)]
-/// `MediaSet` Label
-///
-/// Used to uniquely identify a `MediaSet`. They are stored as second
-/// file on the tape.
-pub struct MediaSetLabel {
- /// The associated `MediaPool`
- pub pool: String,
- /// Uuid. We use the all-zero Uuid to reseve an empty media for a specific pool
- pub uuid: Uuid,
- /// Media sequence number
- pub seq_nr: u64,
- /// Creation time stamp
- pub ctime: i64,
- /// Encryption key finkerprint (if encryped)
- #[serde(skip_serializing_if="Option::is_none")]
- pub encryption_key_fingerprint: Option<Fingerprint>,
-}
-
-impl MediaSetLabel {
-
- pub fn with_data(
- pool: &str,
- uuid: Uuid,
- seq_nr: u64,
- ctime: i64,
- encryption_key_fingerprint: Option<Fingerprint>,
- ) -> Self {
- Self {
- pool: pool.to_string(),
- uuid,
- seq_nr,
- ctime,
- encryption_key_fingerprint,
- }
- }
-}
-
-impl BlockHeader {
-
- pub const SIZE: usize = PROXMOX_TAPE_BLOCK_SIZE;
-
- /// Allocates a new instance on the heap
- pub fn new() -> Box<Self> {
- use std::alloc::{alloc_zeroed, Layout};
-
- let mut buffer = unsafe {
- let ptr = alloc_zeroed(
- Layout::from_size_align(Self::SIZE, std::mem::align_of::<u64>())
- .unwrap(),
- );
- Box::from_raw(
- std::slice::from_raw_parts_mut(ptr, Self::SIZE - 16)
- as *mut [u8] as *mut Self
- )
- };
- buffer.magic = PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0;
- buffer
- }
-
- /// Set the `size` field
- pub fn set_size(&mut self, size: usize) {
- let size = size.to_le_bytes();
- self.size.copy_from_slice(&size[..3]);
- }
-
- /// Returns the `size` field
- pub fn size(&self) -> usize {
- (self.size[0] as usize) + ((self.size[1] as usize)<<8) + ((self.size[2] as usize)<<16)
- }
-
- /// Set the `seq_nr` field
- pub fn set_seq_nr(&mut self, seq_nr: u32) {
- self.seq_nr = seq_nr.to_le();
- }
-
- /// Returns the `seq_nr` field
- pub fn seq_nr(&self) -> u32 {
- u32::from_le(self.seq_nr)
- }
-}
--- /dev/null
+use std::io::Read;
+
+use crate::tape::{
+ TapeRead,
+ tape_device_read_block,
+ file_formats::{
+ PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0,
+ BlockHeader,
+ BlockHeaderFlags,
+ },
+};
+
+/// Read a block stream generated by 'BlockWriter'.
+///
+/// This class implements 'TapeRead'. It always read whole blocks from
+/// the underlying reader, and does additional error checks:
+///
+/// - check magic number (detect streams not written by 'BlockWriter')
+/// - check block size
+/// - check block sequence numbers
+///
+/// The reader consumes the EOF mark after the data stream (if read to
+/// the end of the stream).
+pub struct BlockedReader<R> {
+ reader: R,
+ buffer: Box<BlockHeader>,
+ seq_nr: u32,
+ found_end_marker: bool,
+ incomplete: bool,
+ got_eod: bool,
+ read_error: bool,
+ read_pos: usize,
+}
+
+impl <R: Read> BlockedReader<R> {
+
+ /// Create a new BlockedReader instance.
+ ///
+ /// This tries to read the first block, and returns None if we are
+ /// at EOT.
+ pub fn open(mut reader: R) -> Result<Option<Self>, std::io::Error> {
+
+ let mut buffer = BlockHeader::new();
+
+ if !Self::read_block_frame(&mut buffer, &mut reader)? {
+ return Ok(None);
+ }
+
+ let (_size, found_end_marker) = Self::check_buffer(&buffer, 0)?;
+
+ let mut incomplete = false;
+ let mut got_eod = false;
+
+ if found_end_marker {
+ incomplete = buffer.flags.contains(BlockHeaderFlags::INCOMPLETE);
+ Self::consume_eof_marker(&mut reader)?;
+ got_eod = true;
+ }
+
+ Ok(Some(Self {
+ reader,
+ buffer,
+ found_end_marker,
+ incomplete,
+ got_eod,
+ seq_nr: 1,
+ read_error: false,
+ read_pos: 0,
+ }))
+ }
+
+ fn check_buffer(buffer: &BlockHeader, seq_nr: u32) -> Result<(usize, bool), std::io::Error> {
+
+ if buffer.magic != PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0 {
+ proxmox::io_bail!("detected tape block with wrong magic number - not written by proxmox tape");
+ }
+
+ if seq_nr != buffer.seq_nr() {
+ proxmox::io_bail!(
+ "detected tape block with wrong seqence number ({} != {})",
+ seq_nr, buffer.seq_nr())
+ }
+
+ let size = buffer.size();
+ let found_end_marker = buffer.flags.contains(BlockHeaderFlags::END_OF_STREAM);
+
+ if size > buffer.payload.len() {
+ proxmox::io_bail!("detected tape block with wrong payload size ({} > {}", size, buffer.payload.len());
+ } else if size == 0 && !found_end_marker {
+ proxmox::io_bail!("detected tape block with zero payload size");
+ }
+
+
+ Ok((size, found_end_marker))
+ }
+
+ fn read_block_frame(buffer: &mut BlockHeader, reader: &mut R) -> Result<bool, std::io::Error> {
+
+ let data = unsafe {
+ std::slice::from_raw_parts_mut(
+ (buffer as *mut BlockHeader) as *mut u8,
+ BlockHeader::SIZE,
+ )
+ };
+
+ tape_device_read_block(reader, data)
+ }
+
+ fn consume_eof_marker(reader: &mut R) -> Result<(), std::io::Error> {
+ let mut tmp_buf = [0u8; 512]; // use a small buffer for testing EOF
+ if tape_device_read_block(reader, &mut tmp_buf)? {
+ proxmox::io_bail!("detected tape block after stream end marker");
+ }
+ Ok(())
+ }
+
+ fn read_block(&mut self) -> Result<usize, std::io::Error> {
+
+ if !Self::read_block_frame(&mut self.buffer, &mut self.reader)? {
+ self.got_eod = true;
+ self.read_pos = self.buffer.payload.len();
+ if !self.found_end_marker {
+ proxmox::io_bail!("detected tape stream without end marker");
+ }
+ return Ok(0); // EOD
+ }
+
+ let (size, found_end_marker) = Self::check_buffer(&self.buffer, self.seq_nr)?;
+ self.seq_nr += 1;
+
+ if found_end_marker { // consume EOF mark
+ self.found_end_marker = true;
+ self.incomplete = self.buffer.flags.contains(BlockHeaderFlags::INCOMPLETE);
+ Self::consume_eof_marker(&mut self.reader)?;
+ self.got_eod = true;
+ }
+
+ self.read_pos = 0;
+
+ Ok(size)
+ }
+}
+
+impl <R: Read> TapeRead for BlockedReader<R> {
+
+ fn is_incomplete(&self) -> Result<bool, std::io::Error> {
+ if !self.got_eod {
+ proxmox::io_bail!("is_incomplete failed: EOD not reached");
+ }
+ if !self.found_end_marker {
+ proxmox::io_bail!("is_incomplete failed: no end marker found");
+ }
+
+ Ok(self.incomplete)
+ }
+
+ fn has_end_marker(&self) -> Result<bool, std::io::Error> {
+ if !self.got_eod {
+ proxmox::io_bail!("has_end_marker failed: EOD not reached");
+ }
+
+ Ok(self.found_end_marker)
+ }
+}
+
+impl <R: Read> Read for BlockedReader<R> {
+
+ fn read(&mut self, buffer: &mut [u8]) -> Result<usize, std::io::Error> {
+
+ if self.read_error {
+ proxmox::io_bail!("detected read after error - internal error");
+ }
+
+ let mut buffer_size = self.buffer.size();
+ let mut rest = (buffer_size as isize) - (self.read_pos as isize);
+
+ if rest <= 0 && !self.got_eod { // try to refill buffer
+ buffer_size = match self.read_block() {
+ Ok(len) => len,
+ err => {
+ self.read_error = true;
+ return err;
+ }
+ };
+ rest = buffer_size as isize;
+ }
+
+ if rest <= 0 {
+ Ok(0)
+ } else {
+ let copy_len = if (buffer.len() as isize) < rest {
+ buffer.len()
+ } else {
+ rest as usize
+ };
+ buffer[..copy_len].copy_from_slice(
+ &self.buffer.payload[self.read_pos..(self.read_pos + copy_len)]);
+ self.read_pos += copy_len;
+ Ok(copy_len)
+ }
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use std::io::Read;
+ use anyhow::Error;
+ use crate::tape::{
+ TapeWrite,
+ file_formats::PROXMOX_TAPE_BLOCK_SIZE,
+ helpers::{
+ BlockedReader,
+ BlockedWriter,
+ },
+ };
+
+ fn write_and_verify(data: &[u8]) -> Result<(), Error> {
+
+ let mut tape_data = Vec::new();
+
+ let mut writer = BlockedWriter::new(&mut tape_data);
+
+ writer.write_all(data)?;
+
+ writer.finish(false)?;
+
+ assert_eq!(
+ tape_data.len(),
+ ((data.len() + PROXMOX_TAPE_BLOCK_SIZE)/PROXMOX_TAPE_BLOCK_SIZE)
+ *PROXMOX_TAPE_BLOCK_SIZE
+ );
+
+ let reader = &mut &tape_data[..];
+ let mut reader = BlockedReader::open(reader)?.unwrap();
+
+ let mut read_data = Vec::with_capacity(PROXMOX_TAPE_BLOCK_SIZE);
+ reader.read_to_end(&mut read_data)?;
+
+ assert_eq!(data.len(), read_data.len());
+
+ assert_eq!(data, &read_data[..]);
+
+ Ok(())
+ }
+
+ #[test]
+ fn empty_stream() -> Result<(), Error> {
+ write_and_verify(b"")
+ }
+
+ #[test]
+ fn small_data() -> Result<(), Error> {
+ write_and_verify(b"ABC")
+ }
+
+ #[test]
+ fn large_data() -> Result<(), Error> {
+ let data = proxmox::sys::linux::random_data(1024*1024*5)?;
+ write_and_verify(&data)
+ }
+
+ #[test]
+ fn no_data() -> Result<(), Error> {
+ let tape_data = Vec::new();
+ let reader = &mut &tape_data[..];
+ let reader = BlockedReader::open(reader)?;
+ assert!(reader.is_none());
+
+ Ok(())
+ }
+
+ #[test]
+ fn no_end_marker() -> Result<(), Error> {
+ let mut tape_data = Vec::new();
+ {
+ let mut writer = BlockedWriter::new(&mut tape_data);
+ // write at least one block
+ let data = proxmox::sys::linux::random_data(PROXMOX_TAPE_BLOCK_SIZE)?;
+ writer.write_all(&data)?;
+ // but do not call finish here
+ }
+ let reader = &mut &tape_data[..];
+ let mut reader = BlockedReader::open(reader)?.unwrap();
+
+ let mut data = Vec::with_capacity(PROXMOX_TAPE_BLOCK_SIZE);
+ assert!(reader.read_to_end(&mut data).is_err());
+
+ Ok(())
+ }
+
+ #[test]
+ fn small_read_buffer() -> Result<(), Error> {
+ let mut tape_data = Vec::new();
+
+ let mut writer = BlockedWriter::new(&mut tape_data);
+
+ writer.write_all(b"ABC")?;
+
+ writer.finish(false)?;
+
+ let reader = &mut &tape_data[..];
+ let mut reader = BlockedReader::open(reader)?.unwrap();
+
+ let mut buf = [0u8; 1];
+ assert_eq!(reader.read(&mut buf)?, 1, "wrong byte count");
+ assert_eq!(&buf, b"A");
+ assert_eq!(reader.read(&mut buf)?, 1, "wrong byte count");
+ assert_eq!(&buf, b"B");
+ assert_eq!(reader.read(&mut buf)?, 1, "wrong byte count");
+ assert_eq!(&buf, b"C");
+ assert_eq!(reader.read(&mut buf)?, 0, "wrong byte count");
+ assert_eq!(reader.read(&mut buf)?, 0, "wrong byte count");
+
+ Ok(())
+ }
+}
--- /dev/null
+use std::io::Write;
+
+use proxmox::tools::vec;
+
+use crate::tape::{
+ TapeWrite,
+ tape_device_write_block,
+ file_formats::{
+ BlockHeader,
+ BlockHeaderFlags,
+ },
+};
+
+/// Assemble and write blocks of data
+///
+/// This type implement 'TapeWrite'. Data written is assembled to
+/// equally sized blocks (see 'BlockHeader'), which are then written
+/// to the underlying writer.
+pub struct BlockedWriter<W> {
+ writer: W,
+ buffer: Box<BlockHeader>,
+ buffer_pos: usize,
+ seq_nr: u32,
+ logical_end_of_media: bool,
+ bytes_written: usize,
+}
+
+impl <W: Write> BlockedWriter<W> {
+
+ /// Allow access to underlying writer
+ pub fn writer_ref_mut(&mut self) -> &mut W {
+ &mut self.writer
+ }
+
+ /// Creates a new instance.
+ pub fn new(writer: W) -> Self {
+ Self {
+ writer,
+ buffer: BlockHeader::new(),
+ buffer_pos: 0,
+ seq_nr: 0,
+ logical_end_of_media: false,
+ bytes_written: 0,
+ }
+ }
+
+ fn write_block(buffer: &BlockHeader, writer: &mut W) -> Result<bool, std::io::Error> {
+
+ let data = unsafe {
+ std::slice::from_raw_parts(
+ (buffer as *const BlockHeader) as *const u8,
+ BlockHeader::SIZE,
+ )
+ };
+ tape_device_write_block(writer, data)
+ }
+
+ fn write(&mut self, data: &[u8]) -> Result<usize, std::io::Error> {
+
+ if data.is_empty() { return Ok(0); }
+
+ let rest = self.buffer.payload.len() - self.buffer_pos;
+ let bytes = if data.len() < rest { data.len() } else { rest };
+ self.buffer.payload[self.buffer_pos..(self.buffer_pos+bytes)]
+ .copy_from_slice(&data[..bytes]);
+
+ let rest = rest - bytes;
+
+ if rest == 0 {
+ self.buffer.flags = BlockHeaderFlags::empty();
+ self.buffer.set_size(self.buffer.payload.len());
+ self.buffer.set_seq_nr(self.seq_nr);
+ self.seq_nr += 1;
+ let leom = Self::write_block(&self.buffer, &mut self.writer)?;
+ if leom { self.logical_end_of_media = true; }
+ self.buffer_pos = 0;
+ self.bytes_written += BlockHeader::SIZE;
+
+ } else {
+ self.buffer_pos += bytes;
+ }
+
+ Ok(bytes)
+ }
+
+}
+
+impl <W: Write> TapeWrite for BlockedWriter<W> {
+
+ fn write_all(&mut self, mut data: &[u8]) -> Result<bool, std::io::Error> {
+ while !data.is_empty() {
+ match self.write(data) {
+ Ok(n) => data = &data[n..],
+ Err(e) => return Err(e),
+ }
+ }
+ Ok(self.logical_end_of_media)
+ }
+
+ fn bytes_written(&self) -> usize {
+ self.bytes_written
+ }
+
+ /// flush last block, set END_OF_STREAM flag
+ ///
+ /// Note: This may write an empty block just including the
+ /// END_OF_STREAM flag.
+ fn finish(&mut self, incomplete: bool) -> Result<bool, std::io::Error> {
+ vec::clear(&mut self.buffer.payload[self.buffer_pos..]);
+ self.buffer.flags = BlockHeaderFlags::END_OF_STREAM;
+ if incomplete { self.buffer.flags |= BlockHeaderFlags::INCOMPLETE; }
+ self.buffer.set_size(self.buffer_pos);
+ self.buffer.set_seq_nr(self.seq_nr);
+ self.seq_nr += 1;
+ self.bytes_written += BlockHeader::SIZE;
+ Self::write_block(&self.buffer, &mut self.writer)
+ }
+
+ /// Returns if the writer already detected the logical end of media
+ fn logical_end_of_media(&self) -> bool {
+ self.logical_end_of_media
+ }
+
+}
--- /dev/null
+use std::io::Read;
+
+use anyhow::{bail, Error};
+use endian_trait::Endian;
+
+use proxmox::tools::{
+ Uuid,
+ io::ReadExt,
+};
+
+use crate::backup::DataBlob;
+
+use crate::tape::{
+ TapeWrite,
+ file_formats::{
+ PROXMOX_TAPE_BLOCK_SIZE,
+ PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0,
+ PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
+ MediaContentHeader,
+ ChunkArchiveEntryHeader,
+ },
+};
+
+/// Writes chunk archives to tape.
+///
+/// A chunk archive consists of a `MediaContentHeader` followed by a
+/// list of chunks entries. Each chunk entry consists of a
+/// `ChunkArchiveEntryHeader` folowed by the chunk data (`DataBlob`).
+///
+/// `| MediaContentHeader | ( ChunkArchiveEntryHeader | DataBlob )* |`
+pub struct ChunkArchiveWriter<'a> {
+ writer: Option<Box<dyn TapeWrite + 'a>>,
+ bytes_written: usize, // does not include bytes from current writer
+ close_on_leom: bool,
+}
+
+impl <'a> ChunkArchiveWriter<'a> {
+
+ pub const MAGIC: [u8; 8] = PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0;
+
+ /// Creates a new instance
+ pub fn new(mut writer: Box<dyn TapeWrite + 'a>, close_on_leom: bool) -> Result<(Self,Uuid), Error> {
+
+ let header = MediaContentHeader::new(Self::MAGIC, 0);
+ writer.write_header(&header, &[])?;
+
+ let me = Self {
+ writer: Some(writer),
+ bytes_written: 0,
+ close_on_leom,
+ };
+
+ Ok((me, header.uuid.into()))
+ }
+
+ /// Returns the number of bytes written so far.
+ pub fn bytes_written(&self) -> usize {
+ match self.writer {
+ Some(ref writer) => writer.bytes_written(),
+ None => self.bytes_written, // finalize sets this
+ }
+ }
+
+ fn write_all(&mut self, data: &[u8]) -> Result<bool, std::io::Error> {
+ match self.writer {
+ Some(ref mut writer) => writer.write_all(data),
+ None => proxmox::io_bail!(
+ "detected write after archive finished - internal error"),
+ }
+ }
+
+ /// Write chunk into archive.
+ ///
+ /// This may return false when `LEOM` is detected (when close_on_leom is set).
+ /// In that case the archive only contains parts of the last chunk.
+ pub fn try_write_chunk(
+ &mut self,
+ digest: &[u8;32],
+ blob: &DataBlob,
+ ) -> Result<bool, std::io::Error> {
+
+ if self.writer.is_none() {
+ return Ok(false);
+ }
+
+ let head = ChunkArchiveEntryHeader {
+ magic: PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0,
+ digest: *digest,
+ size: blob.raw_size(),
+ };
+
+ let head = head.to_le();
+ let data = unsafe { std::slice::from_raw_parts(
+ &head as *const ChunkArchiveEntryHeader as *const u8,
+ std::mem::size_of::<ChunkArchiveEntryHeader>())
+ };
+
+ self.write_all(data)?;
+
+ let mut start = 0;
+ let blob_data = blob.raw_data();
+ loop {
+ if start >= blob_data.len() {
+ break;
+ }
+
+ let end = start + PROXMOX_TAPE_BLOCK_SIZE;
+ let mut chunk_is_complete = false;
+ let leom = if end > blob_data.len() {
+ chunk_is_complete = true;
+ self.write_all(&blob_data[start..])?
+ } else {
+ self.write_all(&blob_data[start..end])?
+ };
+ if leom {
+ println!("WRITE DATA LEOM at pos {}", self.bytes_written());
+ if self.close_on_leom {
+ let mut writer = self.writer.take().unwrap();
+ writer.finish(false)?;
+ self.bytes_written = writer.bytes_written();
+ return Ok(chunk_is_complete);
+ }
+ }
+ start = end;
+ }
+
+ Ok(true)
+ }
+
+ /// This must be called at the end to add padding and `EOF`
+ ///
+ /// Returns true on `LEOM` or when we hit max archive size
+ pub fn finish(&mut self) -> Result<bool, std::io::Error> {
+ match self.writer.take() {
+ Some(mut writer) => {
+ self.bytes_written = writer.bytes_written();
+ writer.finish(false)
+ }
+ None => Ok(true),
+ }
+ }
+}
+
+/// Read chunk archives.
+pub struct ChunkArchiveDecoder<R> {
+ reader: R,
+}
+
+impl <R: Read> ChunkArchiveDecoder<R> {
+
+ /// Creates a new instance
+ pub fn new(reader: R) -> Self {
+ Self { reader }
+ }
+
+ /// Allow access to the underyling reader
+ pub fn reader(&self) -> &R {
+ &self.reader
+ }
+
+ /// Returns the next chunk (if any).
+ pub fn next_chunk(&mut self) -> Result<Option<([u8;32], DataBlob)>, Error> {
+
+ let mut header = ChunkArchiveEntryHeader {
+ magic: [0u8; 8],
+ digest: [0u8; 32],
+ size: 0,
+ };
+ let data = unsafe {
+ std::slice::from_raw_parts_mut(
+ (&mut header as *mut ChunkArchiveEntryHeader) as *mut u8,
+ std::mem::size_of::<ChunkArchiveEntryHeader>())
+ };
+
+ match self.reader.read_exact_or_eof(data) {
+ Ok(true) => {},
+ Ok(false) => {
+ // last chunk is allowed to be incomplete - simply report EOD
+ return Ok(None);
+ }
+ Err(err) => return Err(err.into()),
+ };
+
+ if header.magic != PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0 {
+ bail!("wrong magic number");
+ }
+
+ let raw_data = match self.reader.read_exact_allocated(header.size as usize) {
+ Ok(data) => data,
+ Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => {
+ // last chunk is allowed to be incomplete - simply report EOD
+ return Ok(None);
+ }
+ Err(err) => return Err(err.into()),
+ };
+
+ let blob = DataBlob::from_raw(raw_data)?;
+ blob.verify_crc()?;
+
+ Ok(Some((header.digest, blob)))
+ }
+}
--- /dev/null
+//! File format definitions and implementations for data written to
+//! tapes
+
+mod blocked_reader;
+pub use blocked_reader::*;
+
+mod blocked_writer;
+pub use blocked_writer::*;
+
+mod chunk_archive;
+pub use chunk_archive::*;
+
+mod snapshot_archive;
+pub use snapshot_archive::*;
+
+use std::collections::HashMap;
+
+use anyhow::{bail, Error};
+use ::serde::{Deserialize, Serialize};
+use endian_trait::Endian;
+use bitflags::bitflags;
+
+use proxmox::tools::Uuid;
+
+use crate::backup::Fingerprint;
+
+/// We use 256KB blocksize (always)
+pub const PROXMOX_TAPE_BLOCK_SIZE: usize = 256*1024;
+
+// openssl::sha::sha256(b"Proxmox Tape Block Header v1.0")[0..8]
+pub const PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0: [u8; 8] = [220, 189, 175, 202, 235, 160, 165, 40];
+
+// openssl::sha::sha256(b"Proxmox Backup Content Header v1.0")[0..8];
+pub const PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0: [u8; 8] = [99, 238, 20, 159, 205, 242, 155, 12];
+// openssl::sha::sha256(b"Proxmox Backup Tape Label v1.0")[0..8];
+pub const PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0: [u8; 8] = [42, 5, 191, 60, 176, 48, 170, 57];
+// openssl::sha::sha256(b"Proxmox Backup MediaSet Label v1.0")
+pub const PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0: [u8; 8] = [8, 96, 99, 249, 47, 151, 83, 216];
+
+// openssl::sha::sha256(b"Proxmox Backup Chunk Archive v1.0")[0..8]
+pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0: [u8; 8] = [62, 173, 167, 95, 49, 76, 6, 110];
+// openssl::sha::sha256(b"Proxmox Backup Chunk Archive Entry v1.0")[0..8]
+pub const PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0: [u8; 8] = [72, 87, 109, 242, 222, 66, 143, 220];
+
+// openssl::sha::sha256(b"Proxmox Backup Snapshot Archive v1.0")[0..8];
+pub const PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0: [u8; 8] = [9, 182, 2, 31, 125, 232, 114, 133];
+
+lazy_static::lazy_static!{
+ // Map content magic numbers to human readable names.
+ static ref PROXMOX_TAPE_CONTENT_NAME: HashMap<&'static [u8;8], &'static str> = {
+ let mut map = HashMap::new();
+ map.insert(&PROXMOX_BACKUP_MEDIA_LABEL_MAGIC_1_0, "Proxmox Backup Tape Label v1.0");
+ map.insert(&PROXMOX_BACKUP_MEDIA_SET_LABEL_MAGIC_1_0, "Proxmox Backup MediaSet Label v1.0");
+ map.insert(&PROXMOX_BACKUP_CHUNK_ARCHIVE_MAGIC_1_0, "Proxmox Backup Chunk Archive v1.0");
+ map.insert(&PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, "Proxmox Backup Snapshot Archive v1.0");
+ map
+ };
+}
+
+/// Map content magic numbers to human readable names.
+pub fn proxmox_tape_magic_to_text(magic: &[u8; 8]) -> Option<String> {
+ PROXMOX_TAPE_CONTENT_NAME.get(magic).map(|s| String::from(*s))
+}
+
+/// Tape Block Header with data payload
+///
+/// All tape files are written as sequence of blocks.
+///
+/// Note: this struct is large, never put this on the stack!
+/// so we use an unsized type to avoid that.
+///
+/// Tape data block are always read/written with a fixed size
+/// (`PROXMOX_TAPE_BLOCK_SIZE`). But they may contain less data, so the
+/// header has an additional size field. For streams of blocks, there
+/// is a sequence number (`seq_nr`) which may be use for additional
+/// error checking.
+#[repr(C,packed)]
+pub struct BlockHeader {
+ /// fixed value `PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0`
+ pub magic: [u8; 8],
+ pub flags: BlockHeaderFlags,
+ /// size as 3 bytes unsigned, little endian
+ pub size: [u8; 3],
+ /// block sequence number
+ pub seq_nr: u32,
+ pub payload: [u8],
+}
+
+bitflags! {
+ /// Header flags (e.g. `END_OF_STREAM` or `INCOMPLETE`)
+ pub struct BlockHeaderFlags: u8 {
+ /// Marks the last block in a stream.
+ const END_OF_STREAM = 0b00000001;
+ /// Mark multivolume streams (when set in the last block)
+ const INCOMPLETE = 0b00000010;
+ }
+}
+
+#[derive(Endian, Copy, Clone, Debug)]
+#[repr(C,packed)]
+/// Media Content Header
+///
+/// All tape files start with this header. The header may contain some
+/// informational data indicated by `size`.
+///
+/// `| MediaContentHeader | header data (size) | stream data |`
+///
+/// Note: The stream data following may be of any size.
+pub struct MediaContentHeader {
+ /// fixed value `PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0`
+ pub magic: [u8; 8],
+ /// magic number for the content following
+ pub content_magic: [u8; 8],
+ /// unique ID to identify this data stream
+ pub uuid: [u8; 16],
+ /// stream creation time
+ pub ctime: i64,
+ /// Size of header data
+ pub size: u32,
+ /// Part number for multipart archives.
+ pub part_number: u8,
+ /// Reserved for future use
+ pub reserved_0: u8,
+ /// Reserved for future use
+ pub reserved_1: u8,
+ /// Reserved for future use
+ pub reserved_2: u8,
+}
+
+impl MediaContentHeader {
+
+ /// Create a new instance with autogenerated Uuid
+ pub fn new(content_magic: [u8; 8], size: u32) -> Self {
+ let uuid = *proxmox::tools::uuid::Uuid::generate()
+ .into_inner();
+ Self {
+ magic: PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0,
+ content_magic,
+ uuid,
+ ctime: proxmox::tools::time::epoch_i64(),
+ size,
+ part_number: 0,
+ reserved_0: 0,
+ reserved_1: 0,
+ reserved_2: 0,
+ }
+ }
+
+ /// Helper to check magic numbers and size constraints
+ pub fn check(&self, content_magic: [u8; 8], min_size: u32, max_size: u32) -> Result<(), Error> {
+ if self.magic != PROXMOX_BACKUP_CONTENT_HEADER_MAGIC_1_0 {
+ bail!("MediaContentHeader: wrong magic");
+ }
+ if self.content_magic != content_magic {
+ bail!("MediaContentHeader: wrong content magic");
+ }
+ if self.size < min_size || self.size > max_size {
+ bail!("MediaContentHeader: got unexpected size");
+ }
+ Ok(())
+ }
+
+ /// Returns the content Uuid
+ pub fn content_uuid(&self) -> Uuid {
+ Uuid::from(self.uuid)
+ }
+}
+
+#[derive(Endian)]
+#[repr(C,packed)]
+/// Header for data blobs inside a chunk archive
+pub struct ChunkArchiveEntryHeader {
+ /// fixed value `PROXMOX_BACKUP_CHUNK_ARCHIVE_ENTRY_MAGIC_1_0`
+ pub magic: [u8; 8],
+ /// Chunk digest
+ pub digest: [u8; 32],
+ /// Chunk size
+ pub size: u64,
+}
+
+#[derive(Serialize,Deserialize,Clone,Debug)]
+/// Media Label
+///
+/// Media labels are used to uniquely identify a media. They are
+/// stored as first file on the tape.
+pub struct MediaLabel {
+ /// Unique ID
+ pub uuid: Uuid,
+ /// Media label text (or Barcode)
+ pub label_text: String,
+ /// Creation time stamp
+ pub ctime: i64,
+}
+
+
+#[derive(Serialize,Deserialize,Clone,Debug)]
+/// `MediaSet` Label
+///
+/// Used to uniquely identify a `MediaSet`. They are stored as second
+/// file on the tape.
+pub struct MediaSetLabel {
+ /// The associated `MediaPool`
+ pub pool: String,
+ /// Uuid. We use the all-zero Uuid to reseve an empty media for a specific pool
+ pub uuid: Uuid,
+ /// Media sequence number
+ pub seq_nr: u64,
+ /// Creation time stamp
+ pub ctime: i64,
+ /// Encryption key finkerprint (if encryped)
+ #[serde(skip_serializing_if="Option::is_none")]
+ pub encryption_key_fingerprint: Option<Fingerprint>,
+}
+
+impl MediaSetLabel {
+
+ pub fn with_data(
+ pool: &str,
+ uuid: Uuid,
+ seq_nr: u64,
+ ctime: i64,
+ encryption_key_fingerprint: Option<Fingerprint>,
+ ) -> Self {
+ Self {
+ pool: pool.to_string(),
+ uuid,
+ seq_nr,
+ ctime,
+ encryption_key_fingerprint,
+ }
+ }
+}
+
+impl BlockHeader {
+
+ pub const SIZE: usize = PROXMOX_TAPE_BLOCK_SIZE;
+
+ /// Allocates a new instance on the heap
+ pub fn new() -> Box<Self> {
+ use std::alloc::{alloc_zeroed, Layout};
+
+ let mut buffer = unsafe {
+ let ptr = alloc_zeroed(
+ Layout::from_size_align(Self::SIZE, std::mem::align_of::<u64>())
+ .unwrap(),
+ );
+ Box::from_raw(
+ std::slice::from_raw_parts_mut(ptr, Self::SIZE - 16)
+ as *mut [u8] as *mut Self
+ )
+ };
+ buffer.magic = PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0;
+ buffer
+ }
+
+ /// Set the `size` field
+ pub fn set_size(&mut self, size: usize) {
+ let size = size.to_le_bytes();
+ self.size.copy_from_slice(&size[..3]);
+ }
+
+ /// Returns the `size` field
+ pub fn size(&self) -> usize {
+ (self.size[0] as usize) + ((self.size[1] as usize)<<8) + ((self.size[2] as usize)<<16)
+ }
+
+ /// Set the `seq_nr` field
+ pub fn set_seq_nr(&mut self, seq_nr: u32) {
+ self.seq_nr = seq_nr.to_le();
+ }
+
+ /// Returns the `seq_nr` field
+ pub fn seq_nr(&self) -> u32 {
+ u32::from_le(self.seq_nr)
+ }
+}
--- /dev/null
+use std::io::{Read, Write};
+use std::pin::Pin;
+use std::task::{Context, Poll};
+
+use proxmox::{
+ sys::error::SysError,
+ tools::Uuid,
+};
+
+use crate::tape::{
+ TapeWrite,
+ SnapshotReader,
+ file_formats::{
+ PROXMOX_TAPE_BLOCK_SIZE,
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
+ MediaContentHeader,
+ },
+};
+
+/// Write a set of files as `pxar` archive to the tape
+///
+/// This ignores file attributes like ACLs and xattrs.
+///
+/// Returns `Ok(Some(content_uuid))` on succees, and `Ok(None)` if
+/// `LEOM` was detected before all data was written. The stream is
+/// marked inclomplete in that case and does not contain all data (The
+/// backup task must rewrite the whole file on the next media).
+pub fn tape_write_snapshot_archive<'a>(
+ writer: &mut (dyn TapeWrite + 'a),
+ snapshot_reader: &SnapshotReader,
+) -> Result<Option<Uuid>, std::io::Error> {
+
+ let snapshot = snapshot_reader.snapshot().to_string();
+ let file_list = snapshot_reader.file_list();
+
+ let header_data = snapshot.as_bytes().to_vec();
+
+ let header = MediaContentHeader::new(
+ PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, header_data.len() as u32);
+ let content_uuid = header.uuid.into();
+
+ let root_metadata = pxar::Metadata::dir_builder(0o0664).build();
+
+ let mut file_copy_buffer = proxmox::tools::vec::undefined(PROXMOX_TAPE_BLOCK_SIZE);
+
+ let result: Result<(), std::io::Error> = proxmox::try_block!({
+
+ let leom = writer.write_header(&header, &header_data)?;
+ if leom {
+ return Err(std::io::Error::from_raw_os_error(nix::errno::Errno::ENOSPC as i32));
+ }
+
+ let mut encoder = pxar::encoder::sync::Encoder::new(PxarTapeWriter::new(writer), &root_metadata)?;
+
+ for filename in file_list.iter() {
+
+ let mut file = snapshot_reader.open_file(filename)
+ .map_err(|err| proxmox::io_format_err!("open file '{}' failed - {}", filename, err))?;
+ let metadata = file.metadata()?;
+ let file_size = metadata.len();
+
+ let metadata: pxar::Metadata = metadata.into();
+
+ if !metadata.is_regular_file() {
+ proxmox::io_bail!("file '{}' is not a regular file", filename);
+ }
+
+ let mut remaining = file_size;
+ let mut out = encoder.create_file(&metadata, filename, file_size)?;
+ while remaining != 0 {
+ let got = file.read(&mut file_copy_buffer[..])?;
+ if got as u64 > remaining {
+ proxmox::io_bail!("file '{}' changed while reading", filename);
+ }
+ out.write_all(&file_copy_buffer[..got])?;
+ remaining -= got as u64;
+
+ }
+ if remaining > 0 {
+ proxmox::io_bail!("file '{}' shrunk while reading", filename);
+ }
+ }
+ encoder.finish()?;
+ Ok(())
+ });
+
+ match result {
+ Ok(()) => {
+ writer.finish(false)?;
+ Ok(Some(content_uuid))
+ }
+ Err(err) => {
+ if err.is_errno(nix::errno::Errno::ENOSPC) && writer.logical_end_of_media() {
+ writer.finish(true)?; // mark as incomplete
+ Ok(None)
+ } else {
+ Err(err)
+ }
+ }
+ }
+}
+
+// Helper to create pxar archives on tape
+//
+// We generate and error at LEOM,
+struct PxarTapeWriter<'a, T: TapeWrite + ?Sized> {
+ inner: &'a mut T,
+}
+
+impl<'a, T: TapeWrite + ?Sized> PxarTapeWriter<'a, T> {
+ pub fn new(inner: &'a mut T) -> Self {
+ Self { inner }
+ }
+}
+
+impl<'a, T: TapeWrite + ?Sized> pxar::encoder::SeqWrite for PxarTapeWriter<'a, T> {
+
+ fn poll_seq_write(
+ self: Pin<&mut Self>,
+ _cx: &mut Context,
+ buf: &[u8],
+ ) -> Poll<std::io::Result<usize>> {
+ let this = unsafe { self.get_unchecked_mut() };
+ Poll::Ready(match this.inner.write_all(buf) {
+ Ok(leom) => {
+ if leom {
+ Err(std::io::Error::from_raw_os_error(nix::errno::Errno::ENOSPC as i32))
+ } else {
+ Ok(buf.len())
+ }
+ }
+ Err(err) => Err(err),
+ })
+ }
+
+ fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<std::io::Result<()>> {
+ Poll::Ready(Ok(()))
+ }
+}
+++ /dev/null
-use std::io::Read;
-
-use crate::tape::{
- TapeRead,
- tape_device_read_block,
- file_formats::{
- PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0,
- BlockHeader,
- BlockHeaderFlags,
- },
-};
-
-/// Read a block stream generated by 'BlockWriter'.
-///
-/// This class implements 'TapeRead'. It always read whole blocks from
-/// the underlying reader, and does additional error checks:
-///
-/// - check magic number (detect streams not written by 'BlockWriter')
-/// - check block size
-/// - check block sequence numbers
-///
-/// The reader consumes the EOF mark after the data stream (if read to
-/// the end of the stream).
-pub struct BlockedReader<R> {
- reader: R,
- buffer: Box<BlockHeader>,
- seq_nr: u32,
- found_end_marker: bool,
- incomplete: bool,
- got_eod: bool,
- read_error: bool,
- read_pos: usize,
-}
-
-impl <R: Read> BlockedReader<R> {
-
- /// Create a new BlockedReader instance.
- ///
- /// This tries to read the first block, and returns None if we are
- /// at EOT.
- pub fn open(mut reader: R) -> Result<Option<Self>, std::io::Error> {
-
- let mut buffer = BlockHeader::new();
-
- if !Self::read_block_frame(&mut buffer, &mut reader)? {
- return Ok(None);
- }
-
- let (_size, found_end_marker) = Self::check_buffer(&buffer, 0)?;
-
- let mut incomplete = false;
- let mut got_eod = false;
-
- if found_end_marker {
- incomplete = buffer.flags.contains(BlockHeaderFlags::INCOMPLETE);
- Self::consume_eof_marker(&mut reader)?;
- got_eod = true;
- }
-
- Ok(Some(Self {
- reader,
- buffer,
- found_end_marker,
- incomplete,
- got_eod,
- seq_nr: 1,
- read_error: false,
- read_pos: 0,
- }))
- }
-
- fn check_buffer(buffer: &BlockHeader, seq_nr: u32) -> Result<(usize, bool), std::io::Error> {
-
- if buffer.magic != PROXMOX_TAPE_BLOCK_HEADER_MAGIC_1_0 {
- proxmox::io_bail!("detected tape block with wrong magic number - not written by proxmox tape");
- }
-
- if seq_nr != buffer.seq_nr() {
- proxmox::io_bail!(
- "detected tape block with wrong seqence number ({} != {})",
- seq_nr, buffer.seq_nr())
- }
-
- let size = buffer.size();
- let found_end_marker = buffer.flags.contains(BlockHeaderFlags::END_OF_STREAM);
-
- if size > buffer.payload.len() {
- proxmox::io_bail!("detected tape block with wrong payload size ({} > {}", size, buffer.payload.len());
- } else if size == 0 && !found_end_marker {
- proxmox::io_bail!("detected tape block with zero payload size");
- }
-
-
- Ok((size, found_end_marker))
- }
-
- fn read_block_frame(buffer: &mut BlockHeader, reader: &mut R) -> Result<bool, std::io::Error> {
-
- let data = unsafe {
- std::slice::from_raw_parts_mut(
- (buffer as *mut BlockHeader) as *mut u8,
- BlockHeader::SIZE,
- )
- };
-
- tape_device_read_block(reader, data)
- }
-
- fn consume_eof_marker(reader: &mut R) -> Result<(), std::io::Error> {
- let mut tmp_buf = [0u8; 512]; // use a small buffer for testing EOF
- if tape_device_read_block(reader, &mut tmp_buf)? {
- proxmox::io_bail!("detected tape block after stream end marker");
- }
- Ok(())
- }
-
- fn read_block(&mut self) -> Result<usize, std::io::Error> {
-
- if !Self::read_block_frame(&mut self.buffer, &mut self.reader)? {
- self.got_eod = true;
- self.read_pos = self.buffer.payload.len();
- if !self.found_end_marker {
- proxmox::io_bail!("detected tape stream without end marker");
- }
- return Ok(0); // EOD
- }
-
- let (size, found_end_marker) = Self::check_buffer(&self.buffer, self.seq_nr)?;
- self.seq_nr += 1;
-
- if found_end_marker { // consume EOF mark
- self.found_end_marker = true;
- self.incomplete = self.buffer.flags.contains(BlockHeaderFlags::INCOMPLETE);
- Self::consume_eof_marker(&mut self.reader)?;
- self.got_eod = true;
- }
-
- self.read_pos = 0;
-
- Ok(size)
- }
-}
-
-impl <R: Read> TapeRead for BlockedReader<R> {
-
- fn is_incomplete(&self) -> Result<bool, std::io::Error> {
- if !self.got_eod {
- proxmox::io_bail!("is_incomplete failed: EOD not reached");
- }
- if !self.found_end_marker {
- proxmox::io_bail!("is_incomplete failed: no end marker found");
- }
-
- Ok(self.incomplete)
- }
-
- fn has_end_marker(&self) -> Result<bool, std::io::Error> {
- if !self.got_eod {
- proxmox::io_bail!("has_end_marker failed: EOD not reached");
- }
-
- Ok(self.found_end_marker)
- }
-}
-
-impl <R: Read> Read for BlockedReader<R> {
-
- fn read(&mut self, buffer: &mut [u8]) -> Result<usize, std::io::Error> {
-
- if self.read_error {
- proxmox::io_bail!("detected read after error - internal error");
- }
-
- let mut buffer_size = self.buffer.size();
- let mut rest = (buffer_size as isize) - (self.read_pos as isize);
-
- if rest <= 0 && !self.got_eod { // try to refill buffer
- buffer_size = match self.read_block() {
- Ok(len) => len,
- err => {
- self.read_error = true;
- return err;
- }
- };
- rest = buffer_size as isize;
- }
-
- if rest <= 0 {
- Ok(0)
- } else {
- let copy_len = if (buffer.len() as isize) < rest {
- buffer.len()
- } else {
- rest as usize
- };
- buffer[..copy_len].copy_from_slice(
- &self.buffer.payload[self.read_pos..(self.read_pos + copy_len)]);
- self.read_pos += copy_len;
- Ok(copy_len)
- }
- }
-}
-
-#[cfg(test)]
-mod test {
- use std::io::Read;
- use anyhow::Error;
- use crate::tape::{
- TapeWrite,
- file_formats::PROXMOX_TAPE_BLOCK_SIZE,
- helpers::{
- BlockedReader,
- BlockedWriter,
- },
- };
-
- fn write_and_verify(data: &[u8]) -> Result<(), Error> {
-
- let mut tape_data = Vec::new();
-
- let mut writer = BlockedWriter::new(&mut tape_data);
-
- writer.write_all(data)?;
-
- writer.finish(false)?;
-
- assert_eq!(
- tape_data.len(),
- ((data.len() + PROXMOX_TAPE_BLOCK_SIZE)/PROXMOX_TAPE_BLOCK_SIZE)
- *PROXMOX_TAPE_BLOCK_SIZE
- );
-
- let reader = &mut &tape_data[..];
- let mut reader = BlockedReader::open(reader)?.unwrap();
-
- let mut read_data = Vec::with_capacity(PROXMOX_TAPE_BLOCK_SIZE);
- reader.read_to_end(&mut read_data)?;
-
- assert_eq!(data.len(), read_data.len());
-
- assert_eq!(data, &read_data[..]);
-
- Ok(())
- }
-
- #[test]
- fn empty_stream() -> Result<(), Error> {
- write_and_verify(b"")
- }
-
- #[test]
- fn small_data() -> Result<(), Error> {
- write_and_verify(b"ABC")
- }
-
- #[test]
- fn large_data() -> Result<(), Error> {
- let data = proxmox::sys::linux::random_data(1024*1024*5)?;
- write_and_verify(&data)
- }
-
- #[test]
- fn no_data() -> Result<(), Error> {
- let tape_data = Vec::new();
- let reader = &mut &tape_data[..];
- let reader = BlockedReader::open(reader)?;
- assert!(reader.is_none());
-
- Ok(())
- }
-
- #[test]
- fn no_end_marker() -> Result<(), Error> {
- let mut tape_data = Vec::new();
- {
- let mut writer = BlockedWriter::new(&mut tape_data);
- // write at least one block
- let data = proxmox::sys::linux::random_data(PROXMOX_TAPE_BLOCK_SIZE)?;
- writer.write_all(&data)?;
- // but do not call finish here
- }
- let reader = &mut &tape_data[..];
- let mut reader = BlockedReader::open(reader)?.unwrap();
-
- let mut data = Vec::with_capacity(PROXMOX_TAPE_BLOCK_SIZE);
- assert!(reader.read_to_end(&mut data).is_err());
-
- Ok(())
- }
-
- #[test]
- fn small_read_buffer() -> Result<(), Error> {
- let mut tape_data = Vec::new();
-
- let mut writer = BlockedWriter::new(&mut tape_data);
-
- writer.write_all(b"ABC")?;
-
- writer.finish(false)?;
-
- let reader = &mut &tape_data[..];
- let mut reader = BlockedReader::open(reader)?.unwrap();
-
- let mut buf = [0u8; 1];
- assert_eq!(reader.read(&mut buf)?, 1, "wrong byte count");
- assert_eq!(&buf, b"A");
- assert_eq!(reader.read(&mut buf)?, 1, "wrong byte count");
- assert_eq!(&buf, b"B");
- assert_eq!(reader.read(&mut buf)?, 1, "wrong byte count");
- assert_eq!(&buf, b"C");
- assert_eq!(reader.read(&mut buf)?, 0, "wrong byte count");
- assert_eq!(reader.read(&mut buf)?, 0, "wrong byte count");
-
- Ok(())
- }
-}
+++ /dev/null
-use std::io::Write;
-
-use proxmox::tools::vec;
-
-use crate::tape::{
- TapeWrite,
- tape_device_write_block,
- file_formats::{
- BlockHeader,
- BlockHeaderFlags,
- },
-};
-
-/// Assemble and write blocks of data
-///
-/// This type implement 'TapeWrite'. Data written is assembled to
-/// equally sized blocks (see 'BlockHeader'), which are then written
-/// to the underlying writer.
-pub struct BlockedWriter<W> {
- writer: W,
- buffer: Box<BlockHeader>,
- buffer_pos: usize,
- seq_nr: u32,
- logical_end_of_media: bool,
- bytes_written: usize,
-}
-
-impl <W: Write> BlockedWriter<W> {
-
- /// Allow access to underlying writer
- pub fn writer_ref_mut(&mut self) -> &mut W {
- &mut self.writer
- }
-
- /// Creates a new instance.
- pub fn new(writer: W) -> Self {
- Self {
- writer,
- buffer: BlockHeader::new(),
- buffer_pos: 0,
- seq_nr: 0,
- logical_end_of_media: false,
- bytes_written: 0,
- }
- }
-
- fn write_block(buffer: &BlockHeader, writer: &mut W) -> Result<bool, std::io::Error> {
-
- let data = unsafe {
- std::slice::from_raw_parts(
- (buffer as *const BlockHeader) as *const u8,
- BlockHeader::SIZE,
- )
- };
- tape_device_write_block(writer, data)
- }
-
- fn write(&mut self, data: &[u8]) -> Result<usize, std::io::Error> {
-
- if data.is_empty() { return Ok(0); }
-
- let rest = self.buffer.payload.len() - self.buffer_pos;
- let bytes = if data.len() < rest { data.len() } else { rest };
- self.buffer.payload[self.buffer_pos..(self.buffer_pos+bytes)]
- .copy_from_slice(&data[..bytes]);
-
- let rest = rest - bytes;
-
- if rest == 0 {
- self.buffer.flags = BlockHeaderFlags::empty();
- self.buffer.set_size(self.buffer.payload.len());
- self.buffer.set_seq_nr(self.seq_nr);
- self.seq_nr += 1;
- let leom = Self::write_block(&self.buffer, &mut self.writer)?;
- if leom { self.logical_end_of_media = true; }
- self.buffer_pos = 0;
- self.bytes_written += BlockHeader::SIZE;
-
- } else {
- self.buffer_pos += bytes;
- }
-
- Ok(bytes)
- }
-
-}
-
-impl <W: Write> TapeWrite for BlockedWriter<W> {
-
- fn write_all(&mut self, mut data: &[u8]) -> Result<bool, std::io::Error> {
- while !data.is_empty() {
- match self.write(data) {
- Ok(n) => data = &data[n..],
- Err(e) => return Err(e),
- }
- }
- Ok(self.logical_end_of_media)
- }
-
- fn bytes_written(&self) -> usize {
- self.bytes_written
- }
-
- /// flush last block, set END_OF_STREAM flag
- ///
- /// Note: This may write an empty block just including the
- /// END_OF_STREAM flag.
- fn finish(&mut self, incomplete: bool) -> Result<bool, std::io::Error> {
- vec::clear(&mut self.buffer.payload[self.buffer_pos..]);
- self.buffer.flags = BlockHeaderFlags::END_OF_STREAM;
- if incomplete { self.buffer.flags |= BlockHeaderFlags::INCOMPLETE; }
- self.buffer.set_size(self.buffer_pos);
- self.buffer.set_seq_nr(self.seq_nr);
- self.seq_nr += 1;
- self.bytes_written += BlockHeader::SIZE;
- Self::write_block(&self.buffer, &mut self.writer)
- }
-
- /// Returns if the writer already detected the logical end of media
- fn logical_end_of_media(&self) -> bool {
- self.logical_end_of_media
- }
-
-}
mod emulate_tape_reader;
pub use emulate_tape_reader::*;
-mod blocked_reader;
-pub use blocked_reader::*;
-
-mod blocked_writer;
-pub use blocked_writer::*;
-
mod snapshot_reader;
pub use snapshot_reader::*;
mod media_catalog;
pub use media_catalog::*;
-mod chunk_archive;
-pub use chunk_archive::*;
-
-mod snapshot_archive;
-pub use snapshot_archive::*;
-
mod pool_writer;
pub use pool_writer::*;
MAX_CHUNK_ARCHIVE_SIZE,
COMMIT_BLOCK_SIZE,
TapeWrite,
- ChunkArchiveWriter,
SnapshotReader,
SnapshotChunkIterator,
MediaPool,
MediaId,
MediaCatalog,
MediaSetCatalog,
- tape_write_snapshot_archive,
- file_formats::MediaSetLabel,
+ file_formats::{
+ MediaSetLabel,
+ ChunkArchiveWriter,
+ tape_write_snapshot_archive,
+ },
drive::{
TapeDriver,
request_and_load_media,
+++ /dev/null
-use std::io::{Read, Write};
-use std::pin::Pin;
-use std::task::{Context, Poll};
-
-use proxmox::{
- sys::error::SysError,
- tools::Uuid,
-};
-
-use crate::tape::{
- TapeWrite,
- SnapshotReader,
- file_formats::{
- PROXMOX_TAPE_BLOCK_SIZE,
- PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0,
- MediaContentHeader,
- },
-};
-
-/// Write a set of files as `pxar` archive to the tape
-///
-/// This ignores file attributes like ACLs and xattrs.
-///
-/// Returns `Ok(Some(content_uuid))` on succees, and `Ok(None)` if
-/// `LEOM` was detected before all data was written. The stream is
-/// marked inclomplete in that case and does not contain all data (The
-/// backup task must rewrite the whole file on the next media).
-pub fn tape_write_snapshot_archive<'a>(
- writer: &mut (dyn TapeWrite + 'a),
- snapshot_reader: &SnapshotReader,
-) -> Result<Option<Uuid>, std::io::Error> {
-
- let snapshot = snapshot_reader.snapshot().to_string();
- let file_list = snapshot_reader.file_list();
-
- let header_data = snapshot.as_bytes().to_vec();
-
- let header = MediaContentHeader::new(
- PROXMOX_BACKUP_SNAPSHOT_ARCHIVE_MAGIC_1_0, header_data.len() as u32);
- let content_uuid = header.uuid.into();
-
- let root_metadata = pxar::Metadata::dir_builder(0o0664).build();
-
- let mut file_copy_buffer = proxmox::tools::vec::undefined(PROXMOX_TAPE_BLOCK_SIZE);
-
- let result: Result<(), std::io::Error> = proxmox::try_block!({
-
- let leom = writer.write_header(&header, &header_data)?;
- if leom {
- return Err(std::io::Error::from_raw_os_error(nix::errno::Errno::ENOSPC as i32));
- }
-
- let mut encoder = pxar::encoder::sync::Encoder::new(PxarTapeWriter::new(writer), &root_metadata)?;
-
- for filename in file_list.iter() {
-
- let mut file = snapshot_reader.open_file(filename)
- .map_err(|err| proxmox::io_format_err!("open file '{}' failed - {}", filename, err))?;
- let metadata = file.metadata()?;
- let file_size = metadata.len();
-
- let metadata: pxar::Metadata = metadata.into();
-
- if !metadata.is_regular_file() {
- proxmox::io_bail!("file '{}' is not a regular file", filename);
- }
-
- let mut remaining = file_size;
- let mut out = encoder.create_file(&metadata, filename, file_size)?;
- while remaining != 0 {
- let got = file.read(&mut file_copy_buffer[..])?;
- if got as u64 > remaining {
- proxmox::io_bail!("file '{}' changed while reading", filename);
- }
- out.write_all(&file_copy_buffer[..got])?;
- remaining -= got as u64;
-
- }
- if remaining > 0 {
- proxmox::io_bail!("file '{}' shrunk while reading", filename);
- }
- }
- encoder.finish()?;
- Ok(())
- });
-
- match result {
- Ok(()) => {
- writer.finish(false)?;
- Ok(Some(content_uuid))
- }
- Err(err) => {
- if err.is_errno(nix::errno::Errno::ENOSPC) && writer.logical_end_of_media() {
- writer.finish(true)?; // mark as incomplete
- Ok(None)
- } else {
- Err(err)
- }
- }
- }
-}
-
-// Helper to create pxar archives on tape
-//
-// We generate and error at LEOM,
-struct PxarTapeWriter<'a, T: TapeWrite + ?Sized> {
- inner: &'a mut T,
-}
-
-impl<'a, T: TapeWrite + ?Sized> PxarTapeWriter<'a, T> {
- pub fn new(inner: &'a mut T) -> Self {
- Self { inner }
- }
-}
-
-impl<'a, T: TapeWrite + ?Sized> pxar::encoder::SeqWrite for PxarTapeWriter<'a, T> {
-
- fn poll_seq_write(
- self: Pin<&mut Self>,
- _cx: &mut Context,
- buf: &[u8],
- ) -> Poll<std::io::Result<usize>> {
- let this = unsafe { self.get_unchecked_mut() };
- Poll::Ready(match this.inner.write_all(buf) {
- Ok(leom) => {
- if leom {
- Err(std::io::Error::from_raw_os_error(nix::errno::Errno::ENOSPC as i32))
- } else {
- Ok(buf.len())
- }
- }
- Err(err) => Err(err),
- })
- }
-
- fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context) -> Poll<std::io::Result<()>> {
- Poll::Ready(Ok(()))
- }
-}
projects / proxmox-backup.git / commitdiff