]>
Commit | Line | Data |
---|---|---|
3025b3a5 DM |
1 | use failure::*; |
2 | use std::convert::TryInto; | |
3025b3a5 | 3 | |
5485b579 | 4 | use proxmox::tools::io::{ReadExt, WriteExt}; |
9f83e0f7 | 5 | |
3025b3a5 DM |
6 | use super::*; |
7 | ||
8 | /// Data blob binary storage format | |
9 | /// | |
10 | /// Data blobs store arbitrary binary data (< 16MB), and can be | |
11 | /// compressed and encrypted. A simply binary format is used to store | |
12 | /// them on disk or transfer them over the network. Please use index | |
13 | /// files to store large data files (".fidx" of ".didx"). | |
14 | /// | |
3025b3a5 DM |
15 | pub struct DataBlob { |
16 | raw_data: Vec<u8>, // tagged, compressed, encryped data | |
17 | } | |
18 | ||
19 | impl DataBlob { | |
20 | ||
21 | /// accessor to raw_data field | |
22 | pub fn raw_data(&self) -> &[u8] { | |
23 | &self.raw_data | |
24 | } | |
25 | ||
cb08ac3e DM |
26 | /// Consume self and returns raw_data |
27 | pub fn into_inner(self) -> Vec<u8> { | |
28 | self.raw_data | |
29 | } | |
30 | ||
3025b3a5 DM |
31 | /// accessor to chunk type (magic number) |
32 | pub fn magic(&self) -> &[u8; 8] { | |
33 | self.raw_data[0..8].try_into().unwrap() | |
34 | } | |
35 | ||
b7f4f27d DM |
36 | /// accessor to crc32 checksum |
37 | pub fn crc(&self) -> u32 { | |
991abfa8 DM |
38 | let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc); |
39 | u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap()) | |
b7f4f27d DM |
40 | } |
41 | ||
42 | // set the CRC checksum field | |
43 | pub fn set_crc(&mut self, crc: u32) { | |
991abfa8 DM |
44 | let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc); |
45 | self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes()); | |
b7f4f27d DM |
46 | } |
47 | ||
48 | /// compute the CRC32 checksum | |
cb08ac3e | 49 | pub fn compute_crc(&self) -> u32 { |
b7f4f27d | 50 | let mut hasher = crc32fast::Hasher::new(); |
991abfa8 DM |
51 | let start = std::mem::size_of::<DataBlobHeader>(); // start after HEAD |
52 | hasher.update(&self.raw_data[start..]); | |
b7f4f27d DM |
53 | hasher.finalize() |
54 | } | |
55 | ||
b208da83 DM |
56 | /// verify the CRC32 checksum |
57 | pub fn verify_crc(&self) -> Result<(), Error> { | |
58 | let expected_crc = self.compute_crc(); | |
59 | if expected_crc != self.crc() { | |
60 | bail!("Data blob has wrong CRC checksum."); | |
61 | } | |
62 | Ok(()) | |
63 | } | |
64 | ||
69ecd8d5 | 65 | /// Create a DataBlob, optionally compressed and/or encrypted |
3025b3a5 DM |
66 | pub fn encode( |
67 | data: &[u8], | |
68 | config: Option<&CryptConfig>, | |
69 | compress: bool, | |
70 | ) -> Result<Self, Error> { | |
71 | ||
72 | if data.len() > 16*1024*1024 { | |
73 | bail!("data blob too large ({} bytes).", data.len()); | |
74 | } | |
75 | ||
f889b158 | 76 | let mut blob = if let Some(config) = config { |
3025b3a5 | 77 | |
0066c6d9 DM |
78 | let compr_data; |
79 | let (_compress, data, magic) = if compress { | |
80 | compr_data = zstd::block::compress(data, 1)?; | |
81 | // Note: We only use compression if result is shorter | |
82 | if compr_data.len() < data.len() { | |
83 | (true, &compr_data[..], ENCR_COMPR_BLOB_MAGIC_1_0) | |
84 | } else { | |
85 | (false, data, ENCRYPTED_BLOB_MAGIC_1_0) | |
86 | } | |
87 | } else { | |
88 | (false, data, ENCRYPTED_BLOB_MAGIC_1_0) | |
89 | }; | |
90 | ||
91 | let header_len = std::mem::size_of::<EncryptedDataBlobHeader>(); | |
92 | let mut raw_data = Vec::with_capacity(data.len() + header_len); | |
93 | ||
94 | let dummy_head = EncryptedDataBlobHeader { | |
95 | head: DataBlobHeader { magic: [0u8; 8], crc: [0; 4] }, | |
96 | iv: [0u8; 16], | |
97 | tag: [0u8; 16], | |
98 | }; | |
5485b579 WB |
99 | unsafe { |
100 | raw_data.write_le_value(dummy_head)?; | |
101 | } | |
0066c6d9 DM |
102 | |
103 | let (iv, tag) = config.encrypt_to(data, &mut raw_data)?; | |
104 | ||
105 | let head = EncryptedDataBlobHeader { | |
106 | head: DataBlobHeader { magic, crc: [0; 4] }, iv, tag, | |
107 | }; | |
108 | ||
5485b579 WB |
109 | unsafe { |
110 | (&mut raw_data[0..header_len]).write_le_value(head)?; | |
111 | } | |
0066c6d9 | 112 | |
f889b158 | 113 | DataBlob { raw_data } |
3025b3a5 DM |
114 | } else { |
115 | ||
991abfa8 | 116 | let max_data_len = data.len() + std::mem::size_of::<DataBlobHeader>(); |
3025b3a5 | 117 | if compress { |
991abfa8 | 118 | let mut comp_data = Vec::with_capacity(max_data_len); |
3025b3a5 | 119 | |
991abfa8 DM |
120 | let head = DataBlobHeader { |
121 | magic: COMPRESSED_BLOB_MAGIC_1_0, | |
122 | crc: [0; 4], | |
123 | }; | |
5485b579 WB |
124 | unsafe { |
125 | comp_data.write_le_value(head)?; | |
126 | } | |
b7f4f27d | 127 | |
3025b3a5 DM |
128 | zstd::stream::copy_encode(data, &mut comp_data, 1)?; |
129 | ||
991abfa8 | 130 | if comp_data.len() < max_data_len { |
eecb2356 DM |
131 | let mut blob = DataBlob { raw_data: comp_data }; |
132 | blob.set_crc(blob.compute_crc()); | |
133 | return Ok(blob); | |
3025b3a5 DM |
134 | } |
135 | } | |
136 | ||
991abfa8 | 137 | let mut raw_data = Vec::with_capacity(max_data_len); |
3025b3a5 | 138 | |
991abfa8 DM |
139 | let head = DataBlobHeader { |
140 | magic: UNCOMPRESSED_BLOB_MAGIC_1_0, | |
141 | crc: [0; 4], | |
142 | }; | |
5485b579 WB |
143 | unsafe { |
144 | raw_data.write_le_value(head)?; | |
145 | } | |
3025b3a5 DM |
146 | raw_data.extend_from_slice(data); |
147 | ||
f889b158 DM |
148 | DataBlob { raw_data } |
149 | }; | |
150 | ||
151 | blob.set_crc(blob.compute_crc()); | |
152 | ||
153 | Ok(blob) | |
3025b3a5 DM |
154 | } |
155 | ||
156 | /// Decode blob data | |
157 | pub fn decode(self, config: Option<&CryptConfig>) -> Result<Vec<u8>, Error> { | |
158 | ||
159 | let magic = self.magic(); | |
160 | ||
161 | if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 { | |
991abfa8 DM |
162 | let data_start = std::mem::size_of::<DataBlobHeader>(); |
163 | return Ok(self.raw_data[data_start..].to_vec()); | |
3025b3a5 | 164 | } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 { |
991abfa8 DM |
165 | let data_start = std::mem::size_of::<DataBlobHeader>(); |
166 | let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?; | |
3025b3a5 | 167 | return Ok(data); |
3025b3a5 | 168 | } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 { |
9f83e0f7 | 169 | let header_len = std::mem::size_of::<EncryptedDataBlobHeader>(); |
ba01828d DM |
170 | let head = unsafe { |
171 | (&self.raw_data[..header_len]).read_le_value::<EncryptedDataBlobHeader>()? | |
172 | }; | |
9f83e0f7 | 173 | |
3025b3a5 DM |
174 | if let Some(config) = config { |
175 | let data = if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 { | |
9f83e0f7 | 176 | config.decode_compressed_chunk(&self.raw_data[header_len..], &head.iv, &head.tag)? |
3025b3a5 | 177 | } else { |
9f83e0f7 | 178 | config.decode_uncompressed_chunk(&self.raw_data[header_len..], &head.iv, &head.tag)? |
3025b3a5 DM |
179 | }; |
180 | return Ok(data); | |
181 | } else { | |
182 | bail!("unable to decrypt blob - missing CryptConfig"); | |
183 | } | |
69ecd8d5 DM |
184 | } else if magic == &AUTH_COMPR_BLOB_MAGIC_1_0 || magic == &AUTHENTICATED_BLOB_MAGIC_1_0 { |
185 | let header_len = std::mem::size_of::<AuthenticatedDataBlobHeader>(); | |
186 | let head = unsafe { | |
187 | (&self.raw_data[..header_len]).read_le_value::<AuthenticatedDataBlobHeader>()? | |
188 | }; | |
189 | ||
190 | let data_start = std::mem::size_of::<AuthenticatedDataBlobHeader>(); | |
191 | ||
192 | // Note: only verify if we have a crypt config | |
193 | if let Some(config) = config { | |
194 | let signature = config.compute_auth_tag(&self.raw_data[data_start..]); | |
195 | if signature != head.tag { | |
196 | bail!("verifying blob signature failed"); | |
197 | } | |
198 | } | |
199 | ||
200 | if magic == &AUTH_COMPR_BLOB_MAGIC_1_0 { | |
201 | let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?; | |
202 | return Ok(data); | |
203 | } else { | |
204 | return Ok(self.raw_data[data_start..].to_vec()); | |
205 | } | |
3025b3a5 DM |
206 | } else { |
207 | bail!("Invalid blob magic number."); | |
208 | } | |
209 | } | |
a38c5d4d | 210 | |
69ecd8d5 DM |
211 | /// Create a signed DataBlob, optionally compressed |
212 | pub fn create_signed( | |
213 | data: &[u8], | |
214 | config: &CryptConfig, | |
215 | compress: bool, | |
216 | ) -> Result<Self, Error> { | |
217 | ||
218 | if data.len() > 16*1024*1024 { | |
219 | bail!("data blob too large ({} bytes).", data.len()); | |
220 | } | |
221 | ||
222 | let compr_data; | |
223 | let (_compress, data, magic) = if compress { | |
224 | compr_data = zstd::block::compress(data, 1)?; | |
225 | // Note: We only use compression if result is shorter | |
226 | if compr_data.len() < data.len() { | |
227 | (true, &compr_data[..], AUTH_COMPR_BLOB_MAGIC_1_0) | |
228 | } else { | |
229 | (false, data, AUTHENTICATED_BLOB_MAGIC_1_0) | |
230 | } | |
231 | } else { | |
232 | (false, data, AUTHENTICATED_BLOB_MAGIC_1_0) | |
233 | }; | |
234 | ||
235 | let header_len = std::mem::size_of::<AuthenticatedDataBlobHeader>(); | |
236 | let mut raw_data = Vec::with_capacity(data.len() + header_len); | |
237 | ||
238 | let head = AuthenticatedDataBlobHeader { | |
239 | head: DataBlobHeader { magic, crc: [0; 4] }, | |
240 | tag: config.compute_auth_tag(data), | |
241 | }; | |
242 | unsafe { | |
243 | raw_data.write_le_value(head)?; | |
244 | } | |
245 | raw_data.extend_from_slice(data); | |
246 | ||
f889b158 DM |
247 | let mut blob = DataBlob { raw_data }; |
248 | blob.set_crc(blob.compute_crc()); | |
249 | ||
250 | return Ok(blob); | |
69ecd8d5 DM |
251 | } |
252 | ||
a38c5d4d DM |
253 | /// Create Instance from raw data |
254 | pub fn from_raw(data: Vec<u8>) -> Result<Self, Error> { | |
255 | ||
256 | if data.len() < std::mem::size_of::<DataBlobHeader>() { | |
257 | bail!("blob too small ({} bytes).", data.len()); | |
258 | } | |
259 | ||
260 | let magic = &data[0..8]; | |
261 | ||
262 | if magic == ENCR_COMPR_BLOB_MAGIC_1_0 || magic == ENCRYPTED_BLOB_MAGIC_1_0 { | |
263 | ||
264 | if data.len() < std::mem::size_of::<EncryptedDataBlobHeader>() { | |
265 | bail!("encrypted blob too small ({} bytes).", data.len()); | |
266 | } | |
267 | ||
268 | let blob = DataBlob { raw_data: data }; | |
269 | ||
270 | Ok(blob) | |
271 | } else if magic == COMPRESSED_BLOB_MAGIC_1_0 || magic == UNCOMPRESSED_BLOB_MAGIC_1_0 { | |
272 | ||
273 | let blob = DataBlob { raw_data: data }; | |
274 | ||
69ecd8d5 DM |
275 | Ok(blob) |
276 | } else if magic == AUTH_COMPR_BLOB_MAGIC_1_0 || magic == AUTHENTICATED_BLOB_MAGIC_1_0 { | |
277 | if data.len() < std::mem::size_of::<AuthenticatedDataBlobHeader>() { | |
278 | bail!("authenticated blob too small ({} bytes).", data.len()); | |
279 | } | |
280 | ||
281 | let blob = DataBlob { raw_data: data }; | |
282 | ||
a38c5d4d DM |
283 | Ok(blob) |
284 | } else { | |
285 | bail!("unable to parse raw blob - wrong magic"); | |
286 | } | |
287 | } | |
288 | ||
3025b3a5 | 289 | } |