src/backup/data_blob.rs

   1 use failure::*;
   2 use std::convert::TryInto;
   3
   4 use proxmox::tools::io::ops::ReadExtOps;
   5 use crate::tools::write::WriteUtilOps;
   6
   7 use super::*;
   8
   9 /// Data blob binary storage format
  10 ///
  11 /// Data blobs store arbitrary binary data (< 16MB), and can be
  12 /// compressed and encrypted. A simply binary format is used to store
  13 /// them on disk or transfer them over the network. Please use index
  14 /// files to store large data files (".fidx" of ".didx").
  15 ///
  16 pub struct DataBlob {
  17     raw_data: Vec<u8>, // tagged, compressed, encryped data
  18 }
  19
  20 impl DataBlob {
  21
  22     /// accessor to raw_data field
  23     pub fn raw_data(&self) -> &[u8]  {
  24         &self.raw_data
  25     }
  26
  27     /// accessor to chunk type (magic number)
  28     pub fn magic(&self) -> &[u8; 8] {
  29         self.raw_data[0..8].try_into().unwrap()
  30     }
  31
  32     /// accessor to crc32 checksum
  33     pub fn crc(&self) -> u32 {
  34         let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc);
  35         u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap())
  36     }
  37
  38     // set the CRC checksum field
  39     pub fn set_crc(&mut self, crc: u32) {
  40         let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc);
  41         self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes());
  42     }
  43
  44     /// compute the CRC32 checksum
  45     pub fn compute_crc(&mut self) -> u32 {
  46         let mut hasher = crc32fast::Hasher::new();
  47         let start = std::mem::size_of::<DataBlobHeader>(); // start after HEAD
  48         hasher.update(&self.raw_data[start..]);
  49         hasher.finalize()
  50     }
  51
  52     pub fn encode(
  53         data: &[u8],
  54         config: Option<&CryptConfig>,
  55         compress: bool,
  56     ) -> Result<Self, Error> {
  57
  58         if data.len() > 16*1024*1024 {
  59             bail!("data blob too large ({} bytes).", data.len());
  60         }
  61
  62         if let Some(config) = config {
  63
  64             let compr_data;
  65             let (_compress, data, magic) = if compress {
  66                 compr_data = zstd::block::compress(data, 1)?;
  67                 // Note: We only use compression if result is shorter
  68                 if compr_data.len() < data.len() {
  69                     (true, &compr_data[..], ENCR_COMPR_BLOB_MAGIC_1_0)
  70                 } else {
  71                     (false, data, ENCRYPTED_BLOB_MAGIC_1_0)
  72                 }
  73             } else {
  74                 (false, data, ENCRYPTED_BLOB_MAGIC_1_0)
  75             };
  76
  77             let header_len = std::mem::size_of::<EncryptedDataBlobHeader>();
  78             let mut raw_data = Vec::with_capacity(data.len() + header_len);
  79
  80             let dummy_head = EncryptedDataBlobHeader {
  81                 head: DataBlobHeader { magic: [0u8; 8], crc: [0; 4] },
  82                 iv: [0u8; 16],
  83                 tag: [0u8; 16],
  84             };
  85             raw_data.write_value(&dummy_head)?;
  86
  87             let (iv, tag) = config.encrypt_to(data, &mut raw_data)?;
  88
  89             let head = EncryptedDataBlobHeader {
  90                 head: DataBlobHeader { magic, crc: [0; 4] }, iv, tag,
  91             };
  92
  93             (&mut raw_data[0..header_len]).write_value(&head)?;
  94
  95             return Ok(DataBlob { raw_data });
  96         } else {
  97
  98             let max_data_len = data.len() + std::mem::size_of::<DataBlobHeader>();
  99             if compress {
 100                 let mut comp_data = Vec::with_capacity(max_data_len);
 101
 102                 let head =  DataBlobHeader {
 103                     magic: COMPRESSED_BLOB_MAGIC_1_0,
 104                     crc: [0; 4],
 105                 };
 106                 comp_data.write_value(&head)?;
 107
 108                 zstd::stream::copy_encode(data, &mut comp_data, 1)?;
 109
 110                 if comp_data.len() < max_data_len {
 111                     return Ok(DataBlob { raw_data: comp_data });
 112                 }
 113             }
 114
 115             let mut raw_data = Vec::with_capacity(max_data_len);
 116
 117             let head =  DataBlobHeader {
 118                 magic: UNCOMPRESSED_BLOB_MAGIC_1_0,
 119                 crc: [0; 4],
 120             };
 121             raw_data.write_value(&head)?;
 122             raw_data.extend_from_slice(data);
 123
 124             return Ok(DataBlob { raw_data });
 125         }
 126     }
 127
 128     /// Decode blob data
 129     pub fn decode(self, config: Option<&CryptConfig>) -> Result<Vec<u8>, Error> {
 130
 131         let magic = self.magic();
 132
 133         if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
 134             let data_start = std::mem::size_of::<DataBlobHeader>();
 135             return Ok(self.raw_data[data_start..].to_vec());
 136         } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
 137             let data_start = std::mem::size_of::<DataBlobHeader>();
 138             let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?;
 139             return Ok(data);
 140         } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
 141             let header_len = std::mem::size_of::<EncryptedDataBlobHeader>();
 142             let head = unsafe {
 143                 (&self.raw_data[..header_len]).read_le_value::<EncryptedDataBlobHeader>()?
 144             };
 145
 146             if let Some(config) = config  {
 147                 let data = if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 {
 148                     config.decode_compressed_chunk(&self.raw_data[header_len..], &head.iv, &head.tag)?
 149                 } else {
 150                     config.decode_uncompressed_chunk(&self.raw_data[header_len..], &head.iv, &head.tag)?
 151                 };
 152                 return Ok(data);
 153             } else {
 154                 bail!("unable to decrypt blob - missing CryptConfig");
 155             }
 156         } else {
 157             bail!("Invalid blob magic number.");
 158         }
 159     }
 160
 161     /// Create Instance from raw data
 162     pub fn from_raw(data: Vec<u8>) -> Result<Self, Error> {
 163
 164         if data.len() < std::mem::size_of::<DataBlobHeader>() {
 165             bail!("blob too small ({} bytes).", data.len());
 166         }
 167
 168         let magic = &data[0..8];
 169
 170         if magic == ENCR_COMPR_BLOB_MAGIC_1_0 || magic == ENCRYPTED_BLOB_MAGIC_1_0 {
 171
 172             if data.len() < std::mem::size_of::<EncryptedDataBlobHeader>() {
 173                 bail!("encrypted blob too small ({} bytes).", data.len());
 174             }
 175
 176             let blob = DataBlob { raw_data: data };
 177
 178             Ok(blob)
 179         } else if magic == COMPRESSED_BLOB_MAGIC_1_0 || magic == UNCOMPRESSED_BLOB_MAGIC_1_0 {
 180
 181             let blob = DataBlob { raw_data: data };
 182
 183             Ok(blob)
 184         } else {
 185             bail!("unable to parse raw blob - wrong magic");
 186         }
 187     }
 188
 189 }