1 use anyhow
::{bail, Error}
;
2 use std
::convert
::TryInto
;
4 use proxmox
::tools
::io
::{ReadExt, WriteExt}
;
6 const MAX_BLOB_SIZE
: usize = 128*1024*1024;
8 use super::file_formats
::*;
9 use super::CryptConfig
;
11 /// Encoded data chunk with digest and positional information
12 pub struct ChunkInfo
{
19 /// Data blob binary storage format
21 /// Data blobs store arbitrary binary data (< 128MB), and can be
22 /// compressed and encrypted (or just signed). A simply binary format
23 /// is used to store them on disk or transfer them over the network.
25 /// Please use index files to store large data files (".fidx" of
29 raw_data
: Vec
<u8>, // tagged, compressed, encryped data
34 /// accessor to raw_data field
35 pub fn raw_data(&self) -> &[u8] {
39 /// Consume self and returns raw_data
40 pub fn into_inner(self) -> Vec
<u8> {
44 /// accessor to chunk type (magic number)
45 pub fn magic(&self) -> &[u8; 8] {
46 self.raw_data
[0..8].try_into().unwrap()
49 /// accessor to crc32 checksum
50 pub fn crc(&self) -> u32 {
51 let crc_o
= proxmox
::offsetof!(DataBlobHeader
, crc
);
52 u32::from_le_bytes(self.raw_data
[crc_o
..crc_o
+4].try_into().unwrap())
55 // set the CRC checksum field
56 pub fn set_crc(&mut self, crc
: u32) {
57 let crc_o
= proxmox
::offsetof!(DataBlobHeader
, crc
);
58 self.raw_data
[crc_o
..crc_o
+4].copy_from_slice(&crc
.to_le_bytes());
61 /// compute the CRC32 checksum
62 pub fn compute_crc(&self) -> u32 {
63 let mut hasher
= crc32fast
::Hasher
::new();
64 let start
= header_size(self.magic()); // start after HEAD
65 hasher
.update(&self.raw_data
[start
..]);
69 /// verify the CRC32 checksum
70 pub fn verify_crc(&self) -> Result
<(), Error
> {
71 let expected_crc
= self.compute_crc();
72 if expected_crc
!= self.crc() {
73 bail
!("Data blob has wrong CRC checksum.");
78 /// Create a DataBlob, optionally compressed and/or encrypted
81 config
: Option
<&CryptConfig
>,
83 ) -> Result
<Self, Error
> {
85 if data
.len() > MAX_BLOB_SIZE
{
86 bail
!("data blob too large ({} bytes).", data
.len());
89 let mut blob
= if let Some(config
) = config
{
92 let (_compress
, data
, magic
) = if compress
{
93 compr_data
= zstd
::block
::compress(data
, 1)?
;
94 // Note: We only use compression if result is shorter
95 if compr_data
.len() < data
.len() {
96 (true, &compr_data
[..], ENCR_COMPR_BLOB_MAGIC_1_0
)
98 (false, data
, ENCRYPTED_BLOB_MAGIC_1_0
)
101 (false, data
, ENCRYPTED_BLOB_MAGIC_1_0
)
104 let header_len
= std
::mem
::size_of
::<EncryptedDataBlobHeader
>();
105 let mut raw_data
= Vec
::with_capacity(data
.len() + header_len
);
107 let dummy_head
= EncryptedDataBlobHeader
{
108 head
: DataBlobHeader { magic: [0u8; 8], crc: [0; 4] }
,
113 raw_data
.write_le_value(dummy_head
)?
;
116 let (iv
, tag
) = config
.encrypt_to(data
, &mut raw_data
)?
;
118 let head
= EncryptedDataBlobHeader
{
119 head
: DataBlobHeader { magic, crc: [0; 4] }
, iv
, tag
,
123 (&mut raw_data
[0..header_len
]).write_le_value(head
)?
;
126 DataBlob { raw_data }
129 let max_data_len
= data
.len() + std
::mem
::size_of
::<DataBlobHeader
>();
131 let mut comp_data
= Vec
::with_capacity(max_data_len
);
133 let head
= DataBlobHeader
{
134 magic
: COMPRESSED_BLOB_MAGIC_1_0
,
138 comp_data
.write_le_value(head
)?
;
141 zstd
::stream
::copy_encode(data
, &mut comp_data
, 1)?
;
143 if comp_data
.len() < max_data_len
{
144 let mut blob
= DataBlob { raw_data: comp_data }
;
145 blob
.set_crc(blob
.compute_crc());
150 let mut raw_data
= Vec
::with_capacity(max_data_len
);
152 let head
= DataBlobHeader
{
153 magic
: UNCOMPRESSED_BLOB_MAGIC_1_0
,
157 raw_data
.write_le_value(head
)?
;
159 raw_data
.extend_from_slice(data
);
161 DataBlob { raw_data }
164 blob
.set_crc(blob
.compute_crc());
170 pub fn decode(self, config
: Option
<&CryptConfig
>) -> Result
<Vec
<u8>, Error
> {
172 let magic
= self.magic();
174 if magic
== &UNCOMPRESSED_BLOB_MAGIC_1_0
{
175 let data_start
= std
::mem
::size_of
::<DataBlobHeader
>();
176 Ok(self.raw_data
[data_start
..].to_vec())
177 } else if magic
== &COMPRESSED_BLOB_MAGIC_1_0
{
178 let data_start
= std
::mem
::size_of
::<DataBlobHeader
>();
179 let data
= zstd
::block
::decompress(&self.raw_data
[data_start
..], MAX_BLOB_SIZE
)?
;
181 } else if magic
== &ENCR_COMPR_BLOB_MAGIC_1_0
|| magic
== &ENCRYPTED_BLOB_MAGIC_1_0
{
182 let header_len
= std
::mem
::size_of
::<EncryptedDataBlobHeader
>();
184 (&self.raw_data
[..header_len
]).read_le_value
::<EncryptedDataBlobHeader
>()?
187 if let Some(config
) = config
{
188 let data
= if magic
== &ENCR_COMPR_BLOB_MAGIC_1_0
{
189 config
.decode_compressed_chunk(&self.raw_data
[header_len
..], &head
.iv
, &head
.tag
)?
191 config
.decode_uncompressed_chunk(&self.raw_data
[header_len
..], &head
.iv
, &head
.tag
)?
195 bail
!("unable to decrypt blob - missing CryptConfig");
197 } else if magic
== &AUTH_COMPR_BLOB_MAGIC_1_0
|| magic
== &AUTHENTICATED_BLOB_MAGIC_1_0
{
198 let header_len
= std
::mem
::size_of
::<AuthenticatedDataBlobHeader
>();
200 (&self.raw_data
[..header_len
]).read_le_value
::<AuthenticatedDataBlobHeader
>()?
203 let data_start
= std
::mem
::size_of
::<AuthenticatedDataBlobHeader
>();
205 // Note: only verify if we have a crypt config
206 if let Some(config
) = config
{
207 let signature
= config
.compute_auth_tag(&self.raw_data
[data_start
..]);
208 if signature
!= head
.tag
{
209 bail
!("verifying blob signature failed");
213 if magic
== &AUTH_COMPR_BLOB_MAGIC_1_0
{
214 let data
= zstd
::block
::decompress(&self.raw_data
[data_start
..], 16*1024*1024)?
;
217 Ok(self.raw_data
[data_start
..].to_vec())
220 bail
!("Invalid blob magic number.");
224 /// Create a signed DataBlob, optionally compressed
225 pub fn create_signed(
227 config
: &CryptConfig
,
229 ) -> Result
<Self, Error
> {
231 if data
.len() > MAX_BLOB_SIZE
{
232 bail
!("data blob too large ({} bytes).", data
.len());
236 let (_compress
, data
, magic
) = if compress
{
237 compr_data
= zstd
::block
::compress(data
, 1)?
;
238 // Note: We only use compression if result is shorter
239 if compr_data
.len() < data
.len() {
240 (true, &compr_data
[..], AUTH_COMPR_BLOB_MAGIC_1_0
)
242 (false, data
, AUTHENTICATED_BLOB_MAGIC_1_0
)
245 (false, data
, AUTHENTICATED_BLOB_MAGIC_1_0
)
248 let header_len
= std
::mem
::size_of
::<AuthenticatedDataBlobHeader
>();
249 let mut raw_data
= Vec
::with_capacity(data
.len() + header_len
);
251 let head
= AuthenticatedDataBlobHeader
{
252 head
: DataBlobHeader { magic, crc: [0; 4] }
,
253 tag
: config
.compute_auth_tag(data
),
256 raw_data
.write_le_value(head
)?
;
258 raw_data
.extend_from_slice(data
);
260 let mut blob
= DataBlob { raw_data }
;
261 blob
.set_crc(blob
.compute_crc());
266 /// Load blob from ``reader``
267 pub fn load(reader
: &mut dyn std
::io
::Read
) -> Result
<Self, Error
> {
269 let mut data
= Vec
::with_capacity(1024*1024);
270 reader
.read_to_end(&mut data
)?
;
275 /// Create Instance from raw data
276 pub fn from_raw(data
: Vec
<u8>) -> Result
<Self, Error
> {
278 if data
.len() < std
::mem
::size_of
::<DataBlobHeader
>() {
279 bail
!("blob too small ({} bytes).", data
.len());
282 let magic
= &data
[0..8];
284 if magic
== ENCR_COMPR_BLOB_MAGIC_1_0
|| magic
== ENCRYPTED_BLOB_MAGIC_1_0
{
286 if data
.len() < std
::mem
::size_of
::<EncryptedDataBlobHeader
>() {
287 bail
!("encrypted blob too small ({} bytes).", data
.len());
290 let blob
= DataBlob { raw_data: data }
;
293 } else if magic
== COMPRESSED_BLOB_MAGIC_1_0
|| magic
== UNCOMPRESSED_BLOB_MAGIC_1_0
{
295 let blob
= DataBlob { raw_data: data }
;
298 } else if magic
== AUTH_COMPR_BLOB_MAGIC_1_0
|| magic
== AUTHENTICATED_BLOB_MAGIC_1_0
{
299 if data
.len() < std
::mem
::size_of
::<AuthenticatedDataBlobHeader
>() {
300 bail
!("authenticated blob too small ({} bytes).", data
.len());
303 let blob
= DataBlob { raw_data: data }
;
307 bail
!("unable to parse raw blob - wrong magic");
311 /// Verify digest and data length for unencrypted chunks.
313 /// To do that, we need to decompress data first. Please note that
314 /// this is north possible for encrypted chunks.
315 pub fn verify_unencrypted(
317 expected_chunk_size
: usize,
318 expected_digest
: &[u8; 32],
319 ) -> Result
<(), Error
> {
321 let magic
= self.magic();
323 let verify_raw_data
= |data
: &[u8]| {
324 if expected_chunk_size
!= data
.len() {
325 bail
!("detected chunk with wrong length ({} != {})", expected_chunk_size
, data
.len());
327 let digest
= openssl
::sha
::sha256(data
);
328 if &digest
!= expected_digest
{
329 bail
!("detected chunk with wrong digest.");
334 if magic
== &COMPRESSED_BLOB_MAGIC_1_0
{
335 let data
= zstd
::block
::decompress(&self.raw_data
[12..], 16*1024*1024)?
;
336 verify_raw_data(&data
)?
;
337 } else if magic
== &UNCOMPRESSED_BLOB_MAGIC_1_0
{
338 verify_raw_data(&self.raw_data
[12..])?
;
345 /// Builder for chunk DataBlobs
347 /// Main purpose is to centralize digest computation. Digest
348 /// computation differ for encryped chunk, and this interface ensures that
349 /// we always compute the correct one.
350 pub struct DataChunkBuilder
<'a
, 'b
> {
351 config
: Option
<&'b CryptConfig
>,
353 digest_computed
: bool
,
358 impl <'a
, 'b
> DataChunkBuilder
<'a
, 'b
> {
360 /// Create a new builder instance.
361 pub fn new(orig_data
: &'a
[u8]) -> Self {
365 digest_computed
: false,
371 /// Set compression flag.
373 /// If true, chunk data is compressed using zstd (level 1).
374 pub fn compress(mut self, value
: bool
) -> Self {
375 self.compress
= value
;
379 /// Set encryption Configuration
381 /// If set, chunks are encrypted.
382 pub fn crypt_config(mut self, value
: &'b CryptConfig
) -> Self {
383 if self.digest_computed
{
384 panic
!("unable to set crypt_config after compute_digest().");
386 self.config
= Some(value
);
390 fn compute_digest(&mut self) {
391 if !self.digest_computed
{
392 if let Some(ref config
) = self.config
{
393 self.digest
= config
.compute_digest(self.orig_data
);
395 self.digest
= openssl
::sha
::sha256(self.orig_data
);
397 self.digest_computed
= true;
401 /// Returns the chunk Digest
403 /// Note: For encrypted chunks, this needs to be called after
404 /// ``crypt_config``.
405 pub fn digest(&mut self) -> &[u8; 32] {
406 if !self.digest_computed
{
407 self.compute_digest();
412 /// Consume self and build the ``DataBlob``.
414 /// Returns the blob and the computet digest.
415 pub fn build(mut self) -> Result
<(DataBlob
, [u8; 32]), Error
> {
416 if !self.digest_computed
{
417 self.compute_digest();
420 let chunk
= DataBlob
::encode(
426 Ok((chunk
, self.digest
))
429 /// Create a chunk filled with zeroes
430 pub fn build_zero_chunk(
431 crypt_config
: Option
<&CryptConfig
>,
434 ) -> Result
<(DataBlob
, [u8; 32]), Error
> {
436 let mut zero_bytes
= Vec
::with_capacity(chunk_size
);
437 zero_bytes
.resize(chunk_size
, 0u8);
438 let mut chunk_builder
= DataChunkBuilder
::new(&zero_bytes
).compress(compress
);
439 if let Some(ref crypt_config
) = crypt_config
{
440 chunk_builder
= chunk_builder
.crypt_config(crypt_config
);
443 chunk_builder
.build()