const MAX_BLOB_SIZE: usize = 128*1024*1024;
-use super::*;
+use super::file_formats::*;
+use super::CryptConfig;
+
+/// Encoded data chunk with digest and positional information
+pub struct ChunkInfo {
+ pub chunk: DataBlob,
+ pub digest: [u8; 32],
+ pub chunk_len: u64,
+ pub offset: u64,
+}
/// Data blob binary storage format
///
/// Data blobs store arbitrary binary data (< 128MB), and can be
-/// compressed and encrypted. A simply binary format is used to store
-/// them on disk or transfer them over the network. Please use index
-/// files to store large data files (".fidx" of ".didx").
+/// compressed and encrypted (or just signed). A simply binary format
+/// is used to store them on disk or transfer them over the network.
+///
+/// Please use index files to store large data files (".fidx" of
+/// ".didx").
///
pub struct DataBlob {
raw_data: Vec<u8>, // tagged, compressed, encryped data
/// accessor to crc32 checksum
pub fn crc(&self) -> u32 {
- let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc);
+ let crc_o = proxmox::offsetof!(DataBlobHeader, crc);
u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap())
}
// set the CRC checksum field
pub fn set_crc(&mut self, crc: u32) {
- let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc);
+ let crc_o = proxmox::offsetof!(DataBlobHeader, crc);
self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes());
}
/// compute the CRC32 checksum
pub fn compute_crc(&self) -> u32 {
let mut hasher = crc32fast::Hasher::new();
- let start = std::mem::size_of::<DataBlobHeader>(); // start after HEAD
+ let start = header_size(self.magic()); // start after HEAD
hasher.update(&self.raw_data[start..]);
hasher.finalize()
}
if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
let data_start = std::mem::size_of::<DataBlobHeader>();
- return Ok(self.raw_data[data_start..].to_vec());
+ Ok(self.raw_data[data_start..].to_vec())
} else if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
let data_start = std::mem::size_of::<DataBlobHeader>();
let data = zstd::block::decompress(&self.raw_data[data_start..], MAX_BLOB_SIZE)?;
- return Ok(data);
+ Ok(data)
} else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 {
let header_len = std::mem::size_of::<EncryptedDataBlobHeader>();
let head = unsafe {
} else {
config.decode_uncompressed_chunk(&self.raw_data[header_len..], &head.iv, &head.tag)?
};
- return Ok(data);
+ Ok(data)
} else {
bail!("unable to decrypt blob - missing CryptConfig");
}
if magic == &AUTH_COMPR_BLOB_MAGIC_1_0 {
let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?;
- return Ok(data);
+ Ok(data)
} else {
- return Ok(self.raw_data[data_start..].to_vec());
+ Ok(self.raw_data[data_start..].to_vec())
}
} else {
bail!("Invalid blob magic number.");
let mut blob = DataBlob { raw_data };
blob.set_crc(blob.compute_crc());
- return Ok(blob);
+ Ok(blob)
+ }
+
+ /// Load blob from ``reader``
+ pub fn load(reader: &mut dyn std::io::Read) -> Result<Self, Error> {
+
+ let mut data = Vec::with_capacity(1024*1024);
+ reader.read_to_end(&mut data)?;
+
+ Self::from_raw(data)
}
/// Create Instance from raw data
}
}
+ /// Verify digest and data length for unencrypted chunks.
+ ///
+ /// To do that, we need to decompress data first. Please note that
+ /// this is noth possible for encrypted chunks.
+ pub fn verify_unencrypted(
+ &self,
+ expected_chunk_size: usize,
+ expected_digest: &[u8; 32],
+ ) -> Result<(), Error> {
+
+ let magic = self.magic();
+
+ let verify_raw_data = |data: &[u8]| {
+ if expected_chunk_size != data.len() {
+ bail!("detected chunk with wrong length ({} != {})", expected_chunk_size, data.len());
+ }
+ let digest = openssl::sha::sha256(data);
+ if &digest != expected_digest {
+ bail!("detected chunk with wrong digest.");
+ }
+ Ok(())
+ };
+
+ if magic == &COMPRESSED_BLOB_MAGIC_1_0 {
+ let data = zstd::block::decompress(&self.raw_data[12..], 16*1024*1024)?;
+ verify_raw_data(&data)?;
+ } else if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 {
+ verify_raw_data(&self.raw_data[12..])?;
+ }
+
+ Ok(())
+ }
+}
+
+/// Builder for chunk DataBlobs
+///
+/// Main purpose is to centralize digest computation. Digest
+/// computation differ for encryped chunk, and this interface ensures that
+/// we always compute the correct one.
+pub struct DataChunkBuilder<'a, 'b> {
+ config: Option<&'b CryptConfig>,
+ orig_data: &'a [u8],
+ digest_computed: bool,
+ digest: [u8; 32],
+ compress: bool,
+}
+
+impl <'a, 'b> DataChunkBuilder<'a, 'b> {
+
+ /// Create a new builder instance.
+ pub fn new(orig_data: &'a [u8]) -> Self {
+ Self {
+ orig_data,
+ config: None,
+ digest_computed: false,
+ digest: [0u8; 32],
+ compress: true,
+ }
+ }
+
+ /// Set compression flag.
+ ///
+ /// If true, chunk data is compressed using zstd (level 1).
+ pub fn compress(mut self, value: bool) -> Self {
+ self.compress = value;
+ self
+ }
+
+ /// Set encryption Configuration
+ ///
+ /// If set, chunks are encrypted.
+ pub fn crypt_config(mut self, value: &'b CryptConfig) -> Self {
+ if self.digest_computed {
+ panic!("unable to set crypt_config after compute_digest().");
+ }
+ self.config = Some(value);
+ self
+ }
+
+ fn compute_digest(&mut self) {
+ if !self.digest_computed {
+ if let Some(ref config) = self.config {
+ self.digest = config.compute_digest(self.orig_data);
+ } else {
+ self.digest = openssl::sha::sha256(self.orig_data);
+ }
+ self.digest_computed = true;
+ }
+ }
+
+ /// Returns the chunk Digest
+ ///
+ /// Note: For encrypted chunks, this needs to be called after
+ /// ``crypt_config``.
+ pub fn digest(&mut self) -> &[u8; 32] {
+ if !self.digest_computed {
+ self.compute_digest();
+ }
+ &self.digest
+ }
+
+ /// Consume self and build the ``DataBlob``.
+ ///
+ /// Returns the blob and the computet digest.
+ pub fn build(mut self) -> Result<(DataBlob, [u8; 32]), Error> {
+ if !self.digest_computed {
+ self.compute_digest();
+ }
+
+ let chunk = DataBlob::encode(
+ self.orig_data,
+ self.config,
+ self.compress,
+ )?;
+
+ Ok((chunk, self.digest))
+ }
+
+ /// Create a chunk filled with zeroes
+ pub fn build_zero_chunk(
+ crypt_config: Option<&CryptConfig>,
+ chunk_size: usize,
+ compress: bool,
+ ) -> Result<(DataBlob, [u8; 32]), Error> {
+
+ let mut zero_bytes = Vec::with_capacity(chunk_size);
+ zero_bytes.resize(chunk_size, 0u8);
+ let mut chunk_builder = DataChunkBuilder::new(&zero_bytes).compress(compress);
+ if let Some(ref crypt_config) = crypt_config {
+ chunk_builder = chunk_builder.crypt_config(crypt_config);
+ }
+
+ chunk_builder.build()
+ }
+
}
projects / proxmox-backup.git / blobdiff