src/backup/data_blob.rs

   1 use failure::*;
   2 use std::convert::TryInto;
   3
   4 use proxmox::tools::io::{ReadExt, WriteExt};
   5
   6 use super::*;
   7
   8 /// Data blob binary storage format
   9 ///
  10 /// Data blobs store arbitrary binary data (< 16MB), and can be
  11 /// compressed and encrypted. A simply binary format is used to store
  12 /// them on disk or transfer them over the network. Please use index
  13 /// files to store large data files (".fidx" of ".didx").
  14 ///
  15 pub struct DataBlob {
  16     raw_data: Vec<u8>, // tagged, compressed, encryped data
  17 }
  18
  19 impl DataBlob {
  20
  21     /// accessor to raw_data field
  22     pub fn raw_data(&self) -> &[u8]  {
  23         &self.raw_data
  24     }
  25
  26     /// Consume self and returns raw_data
  27     pub fn into_inner(self) -> Vec<u8> {
  28         self.raw_data
  29     }
  30
  31     /// accessor to chunk type (magic number)
  32     pub fn magic(&self) -> &[u8; 8] {
  33         self.raw_data[0..8].try_into().unwrap()
  34     }
  35
  36     /// accessor to crc32 checksum
  37     pub fn crc(&self) -> u32 {
  38         let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc);
  39         u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap())
  40     }
  41
  42     // set the CRC checksum field
  43     pub fn set_crc(&mut self, crc: u32) {
  44         let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc);
  45         self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes());
  46     }
  47
  48     /// compute the CRC32 checksum
  49     pub fn compute_crc(&self) -> u32 {
  50         let mut hasher = crc32fast::Hasher::new();
  51         let start = std::mem::size_of::<DataBlobHeader>(); // start after HEAD
  52         hasher.update(&self.raw_data[start..]);
  53         hasher.finalize()
  54     }
  55
  56     /// verify the CRC32 checksum
  57     pub fn verify_crc(&self) -> Result<(), Error> {
  58         let expected_crc = self.compute_crc();
  59         if expected_crc != self.crc() {
  60             bail!("Data blob has wrong CRC checksum.");
  61         }
  62         Ok(())
  63     }
  64
  65     /// Create a DataBlob, optionally compressed and/or encrypted
  66     pub fn encode(
  67         data: &[u8],
  68         config: Option<&CryptConfig>,
  69         compress: bool,
  70     ) -> Result<Self, Error> {
  71
  72         if data.len() > 16*1024*1024 {
  73             bail!("data blob too large ({} bytes).", data.len());
  74         }
  75
  76         let mut blob = if let Some(config) = config {
  77
  78             let compr_data;
  79             let (_compress, data, magic) = if compress {
  80                 compr_data = zstd::block::compress(data, 1)?;
  81                 // Note: We only use compression if result is shorter
  82                 if compr_data.len() < data.len() {
  83                     (true, &compr_data[..], ENCR_COMPR_BLOB_MAGIC_1_0)
  84                 } else {
  85                     (false, data, ENCRYPTED_BLOB_MAGIC_1_0)
  86                 }
  87             } else {
  88                 (false, data, ENCRYPTED_BLOB_MAGIC_1_0)
  89             };
  90
  91             let header_len = std::mem::size_of::<EncryptedDataBlobHeader>();
  92             let mut raw_data = Vec::with_capacity(data.len() + header_len);
  93
  94             let dummy_head = EncryptedDataBlobHeader {
  95                 head: DataBlobHeader { magic: [0u8; 8], crc: [0; 4] },
  96                 iv: [0u8; 16],
  97                 tag: [0u8; 16],
  98             };
  99             unsafe {
 100                 raw_data.write_le_value(dummy_head)?;
 101             }
 102
 103             let (iv, tag) = config.encrypt_to(data, &mut raw_data)?;
 104
 105             let head = EncryptedDataBlobHeader {
 106                 head: DataBlobHeader { magic, crc: [0; 4] }, iv, tag,
 107             };
 108
 109             unsafe {
 110                 (&mut raw_data[0..header_len]).write_le_value(head)?;
 111             }
 112
 113             DataBlob { raw_data }
 114         } else {
 115
 116             let max_data_len = data.len() + std::mem::size_of::<DataBlobHeader>();
 117             if compress {
 118                 let mut comp_data = Vec::with_capacity(max_data_len);
 119
 120                 let head =  DataBlobHeader {
 121                     magic: COMPRESSED_BLOB_MAGIC_1_0,
 122                     crc: [0; 4],
 123                 };
 124                 unsafe {
 125                     comp_data.write_le_value(head)?;
 126                 }
 127
 128                 zstd::stream::copy_encode(data, &mut comp_data, 1)?;
 129
 130                 if comp_data.len() < max_data_len {
 131                     return Ok(DataBlob { raw_data: comp_data });
 132                 }
 133             }
 134
 135             let mut raw_data = Vec::with_capacity(max_data_len);
 136
 137             let head =  DataBlobHeader {
 138                 magic: UNCOMPRESSED_BLOB_MAGIC_1_0,
 139                 crc: [0; 4],
 140             };
 141             unsafe {
 142                 raw_data.write_le_value(head)?;
 143             }
 144             raw_data.extend_from_slice(data);
 145
 146             DataBlob { raw_data }
 147         };
 148
 149         blob.set_crc(blob.compute_crc());
 150
 151         Ok(blob)
 152     }
 153
 154     /// Decode blob data
 155     pub fn decode(self, config: Option<&CryptConfig>) -> Result<Vec<u8>, Error> {
 156
 157         let magic = self.magic();
 158
 159         if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
 160             let data_start = std::mem::size_of::<DataBlobHeader>();
 161             return Ok(self.raw_data[data_start..].to_vec());
 162         } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
 163             let data_start = std::mem::size_of::<DataBlobHeader>();
 164             let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?;
 165             return Ok(data);
 166         } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
 167             let header_len = std::mem::size_of::<EncryptedDataBlobHeader>();
 168             let head = unsafe {
 169                 (&self.raw_data[..header_len]).read_le_value::<EncryptedDataBlobHeader>()?
 170             };
 171
 172             if let Some(config) = config  {
 173                 let data = if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 {
 174                     config.decode_compressed_chunk(&self.raw_data[header_len..], &head.iv, &head.tag)?
 175                 } else {
 176                     config.decode_uncompressed_chunk(&self.raw_data[header_len..], &head.iv, &head.tag)?
 177                 };
 178                 return Ok(data);
 179             } else {
 180                 bail!("unable to decrypt blob - missing CryptConfig");
 181             }
 182         } else if magic == &AUTH_COMPR_BLOB_MAGIC_1_0 || magic == &AUTHENTICATED_BLOB_MAGIC_1_0 {
 183             let header_len = std::mem::size_of::<AuthenticatedDataBlobHeader>();
 184             let head = unsafe {
 185                 (&self.raw_data[..header_len]).read_le_value::<AuthenticatedDataBlobHeader>()?
 186             };
 187
 188             let data_start = std::mem::size_of::<AuthenticatedDataBlobHeader>();
 189
 190             // Note: only verify if we have a crypt config
 191             if let Some(config) = config  {
 192                 let signature = config.compute_auth_tag(&self.raw_data[data_start..]);
 193                 if signature != head.tag {
 194                     bail!("verifying blob signature failed");
 195                 }
 196             }
 197
 198             if magic == &AUTH_COMPR_BLOB_MAGIC_1_0 {
 199                 let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?;
 200                 return Ok(data);
 201             } else {
 202                 return Ok(self.raw_data[data_start..].to_vec());
 203             }
 204         } else {
 205             bail!("Invalid blob magic number.");
 206         }
 207     }
 208
 209     /// Create a signed DataBlob, optionally compressed
 210     pub fn create_signed(
 211         data: &[u8],
 212         config: &CryptConfig,
 213         compress: bool,
 214     ) -> Result<Self, Error> {
 215
 216         if data.len() > 16*1024*1024 {
 217             bail!("data blob too large ({} bytes).", data.len());
 218         }
 219
 220         let compr_data;
 221         let (_compress, data, magic) = if compress {
 222             compr_data = zstd::block::compress(data, 1)?;
 223             // Note: We only use compression if result is shorter
 224             if compr_data.len() < data.len() {
 225                 (true, &compr_data[..], AUTH_COMPR_BLOB_MAGIC_1_0)
 226             } else {
 227                 (false, data, AUTHENTICATED_BLOB_MAGIC_1_0)
 228             }
 229         } else {
 230             (false, data, AUTHENTICATED_BLOB_MAGIC_1_0)
 231         };
 232
 233         let header_len = std::mem::size_of::<AuthenticatedDataBlobHeader>();
 234         let mut raw_data = Vec::with_capacity(data.len() + header_len);
 235
 236         let head = AuthenticatedDataBlobHeader {
 237             head: DataBlobHeader { magic, crc: [0; 4] },
 238             tag: config.compute_auth_tag(data),
 239         };
 240         unsafe {
 241             raw_data.write_le_value(head)?;
 242         }
 243         raw_data.extend_from_slice(data);
 244
 245         let mut blob = DataBlob { raw_data };
 246         blob.set_crc(blob.compute_crc());
 247
 248         return Ok(blob);
 249     }
 250
 251     /// Create Instance from raw data
 252     pub fn from_raw(data: Vec<u8>) -> Result<Self, Error> {
 253
 254         if data.len() < std::mem::size_of::<DataBlobHeader>() {
 255             bail!("blob too small ({} bytes).", data.len());
 256         }
 257
 258         let magic = &data[0..8];
 259
 260         if magic == ENCR_COMPR_BLOB_MAGIC_1_0 || magic == ENCRYPTED_BLOB_MAGIC_1_0 {
 261
 262             if data.len() < std::mem::size_of::<EncryptedDataBlobHeader>() {
 263                 bail!("encrypted blob too small ({} bytes).", data.len());
 264             }
 265
 266             let blob = DataBlob { raw_data: data };
 267
 268             Ok(blob)
 269         } else if magic == COMPRESSED_BLOB_MAGIC_1_0 || magic == UNCOMPRESSED_BLOB_MAGIC_1_0 {
 270
 271             let blob = DataBlob { raw_data: data };
 272
 273             Ok(blob)
 274         } else if magic == AUTH_COMPR_BLOB_MAGIC_1_0 || magic == AUTHENTICATED_BLOB_MAGIC_1_0 {
 275             if data.len() < std::mem::size_of::<AuthenticatedDataBlobHeader>() {
 276                 bail!("authenticated blob too small ({} bytes).", data.len());
 277             }
 278
 279             let blob = DataBlob { raw_data: data };
 280
 281             Ok(blob)
 282         } else {
 283             bail!("unable to parse raw blob - wrong magic");
 284         }
 285     }
 286
 287 }