]>
Commit | Line | Data |
---|---|---|
3025b3a5 DM |
1 | use failure::*; |
2 | use std::convert::TryInto; | |
3025b3a5 | 3 | |
5485b579 | 4 | use proxmox::tools::io::{ReadExt, WriteExt}; |
9f83e0f7 | 5 | |
781ac11c DM |
6 | const MAX_BLOB_SIZE: usize = 128*1024*1024; |
7 | ||
3025b3a5 DM |
8 | use super::*; |
9 | ||
10 | /// Data blob binary storage format | |
11 | /// | |
863be2e6 | 12 | /// Data blobs store arbitrary binary data (< 128MB), and can be |
3025b3a5 DM |
13 | /// compressed and encrypted. A simply binary format is used to store |
14 | /// them on disk or transfer them over the network. Please use index | |
15 | /// files to store large data files (".fidx" of ".didx"). | |
16 | /// | |
3025b3a5 DM |
17 | pub struct DataBlob { |
18 | raw_data: Vec<u8>, // tagged, compressed, encryped data | |
19 | } | |
20 | ||
21 | impl DataBlob { | |
22 | ||
23 | /// accessor to raw_data field | |
24 | pub fn raw_data(&self) -> &[u8] { | |
25 | &self.raw_data | |
26 | } | |
27 | ||
cb08ac3e DM |
28 | /// Consume self and returns raw_data |
29 | pub fn into_inner(self) -> Vec<u8> { | |
30 | self.raw_data | |
31 | } | |
32 | ||
3025b3a5 DM |
33 | /// accessor to chunk type (magic number) |
34 | pub fn magic(&self) -> &[u8; 8] { | |
35 | self.raw_data[0..8].try_into().unwrap() | |
36 | } | |
37 | ||
b7f4f27d DM |
38 | /// accessor to crc32 checksum |
39 | pub fn crc(&self) -> u32 { | |
991abfa8 DM |
40 | let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc); |
41 | u32::from_le_bytes(self.raw_data[crc_o..crc_o+4].try_into().unwrap()) | |
b7f4f27d DM |
42 | } |
43 | ||
44 | // set the CRC checksum field | |
45 | pub fn set_crc(&mut self, crc: u32) { | |
991abfa8 DM |
46 | let crc_o = proxmox::tools::offsetof!(DataBlobHeader, crc); |
47 | self.raw_data[crc_o..crc_o+4].copy_from_slice(&crc.to_le_bytes()); | |
b7f4f27d DM |
48 | } |
49 | ||
50 | /// compute the CRC32 checksum | |
cb08ac3e | 51 | pub fn compute_crc(&self) -> u32 { |
b7f4f27d | 52 | let mut hasher = crc32fast::Hasher::new(); |
c638542b | 53 | let start = header_size(self.magic()); // start after HEAD |
991abfa8 | 54 | hasher.update(&self.raw_data[start..]); |
b7f4f27d DM |
55 | hasher.finalize() |
56 | } | |
57 | ||
b208da83 DM |
58 | /// verify the CRC32 checksum |
59 | pub fn verify_crc(&self) -> Result<(), Error> { | |
60 | let expected_crc = self.compute_crc(); | |
61 | if expected_crc != self.crc() { | |
62 | bail!("Data blob has wrong CRC checksum."); | |
63 | } | |
64 | Ok(()) | |
65 | } | |
66 | ||
69ecd8d5 | 67 | /// Create a DataBlob, optionally compressed and/or encrypted |
3025b3a5 DM |
68 | pub fn encode( |
69 | data: &[u8], | |
70 | config: Option<&CryptConfig>, | |
71 | compress: bool, | |
72 | ) -> Result<Self, Error> { | |
73 | ||
781ac11c | 74 | if data.len() > MAX_BLOB_SIZE { |
3025b3a5 DM |
75 | bail!("data blob too large ({} bytes).", data.len()); |
76 | } | |
77 | ||
f889b158 | 78 | let mut blob = if let Some(config) = config { |
3025b3a5 | 79 | |
0066c6d9 DM |
80 | let compr_data; |
81 | let (_compress, data, magic) = if compress { | |
82 | compr_data = zstd::block::compress(data, 1)?; | |
83 | // Note: We only use compression if result is shorter | |
84 | if compr_data.len() < data.len() { | |
85 | (true, &compr_data[..], ENCR_COMPR_BLOB_MAGIC_1_0) | |
86 | } else { | |
87 | (false, data, ENCRYPTED_BLOB_MAGIC_1_0) | |
88 | } | |
89 | } else { | |
90 | (false, data, ENCRYPTED_BLOB_MAGIC_1_0) | |
91 | }; | |
92 | ||
93 | let header_len = std::mem::size_of::<EncryptedDataBlobHeader>(); | |
94 | let mut raw_data = Vec::with_capacity(data.len() + header_len); | |
95 | ||
96 | let dummy_head = EncryptedDataBlobHeader { | |
97 | head: DataBlobHeader { magic: [0u8; 8], crc: [0; 4] }, | |
98 | iv: [0u8; 16], | |
99 | tag: [0u8; 16], | |
100 | }; | |
5485b579 WB |
101 | unsafe { |
102 | raw_data.write_le_value(dummy_head)?; | |
103 | } | |
0066c6d9 DM |
104 | |
105 | let (iv, tag) = config.encrypt_to(data, &mut raw_data)?; | |
106 | ||
107 | let head = EncryptedDataBlobHeader { | |
108 | head: DataBlobHeader { magic, crc: [0; 4] }, iv, tag, | |
109 | }; | |
110 | ||
5485b579 WB |
111 | unsafe { |
112 | (&mut raw_data[0..header_len]).write_le_value(head)?; | |
113 | } | |
0066c6d9 | 114 | |
f889b158 | 115 | DataBlob { raw_data } |
3025b3a5 DM |
116 | } else { |
117 | ||
991abfa8 | 118 | let max_data_len = data.len() + std::mem::size_of::<DataBlobHeader>(); |
3025b3a5 | 119 | if compress { |
991abfa8 | 120 | let mut comp_data = Vec::with_capacity(max_data_len); |
3025b3a5 | 121 | |
991abfa8 DM |
122 | let head = DataBlobHeader { |
123 | magic: COMPRESSED_BLOB_MAGIC_1_0, | |
124 | crc: [0; 4], | |
125 | }; | |
5485b579 WB |
126 | unsafe { |
127 | comp_data.write_le_value(head)?; | |
128 | } | |
b7f4f27d | 129 | |
3025b3a5 DM |
130 | zstd::stream::copy_encode(data, &mut comp_data, 1)?; |
131 | ||
991abfa8 | 132 | if comp_data.len() < max_data_len { |
eecb2356 DM |
133 | let mut blob = DataBlob { raw_data: comp_data }; |
134 | blob.set_crc(blob.compute_crc()); | |
135 | return Ok(blob); | |
3025b3a5 DM |
136 | } |
137 | } | |
138 | ||
991abfa8 | 139 | let mut raw_data = Vec::with_capacity(max_data_len); |
3025b3a5 | 140 | |
991abfa8 DM |
141 | let head = DataBlobHeader { |
142 | magic: UNCOMPRESSED_BLOB_MAGIC_1_0, | |
143 | crc: [0; 4], | |
144 | }; | |
5485b579 WB |
145 | unsafe { |
146 | raw_data.write_le_value(head)?; | |
147 | } | |
3025b3a5 DM |
148 | raw_data.extend_from_slice(data); |
149 | ||
f889b158 DM |
150 | DataBlob { raw_data } |
151 | }; | |
152 | ||
153 | blob.set_crc(blob.compute_crc()); | |
154 | ||
155 | Ok(blob) | |
3025b3a5 DM |
156 | } |
157 | ||
158 | /// Decode blob data | |
159 | pub fn decode(self, config: Option<&CryptConfig>) -> Result<Vec<u8>, Error> { | |
160 | ||
161 | let magic = self.magic(); | |
162 | ||
163 | if magic == &UNCOMPRESSED_BLOB_MAGIC_1_0 { | |
991abfa8 DM |
164 | let data_start = std::mem::size_of::<DataBlobHeader>(); |
165 | return Ok(self.raw_data[data_start..].to_vec()); | |
3025b3a5 | 166 | } else if magic == &COMPRESSED_BLOB_MAGIC_1_0 { |
991abfa8 | 167 | let data_start = std::mem::size_of::<DataBlobHeader>(); |
781ac11c | 168 | let data = zstd::block::decompress(&self.raw_data[data_start..], MAX_BLOB_SIZE)?; |
3025b3a5 | 169 | return Ok(data); |
3025b3a5 | 170 | } else if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 || magic == &ENCRYPTED_BLOB_MAGIC_1_0 { |
9f83e0f7 | 171 | let header_len = std::mem::size_of::<EncryptedDataBlobHeader>(); |
ba01828d DM |
172 | let head = unsafe { |
173 | (&self.raw_data[..header_len]).read_le_value::<EncryptedDataBlobHeader>()? | |
174 | }; | |
9f83e0f7 | 175 | |
3025b3a5 DM |
176 | if let Some(config) = config { |
177 | let data = if magic == &ENCR_COMPR_BLOB_MAGIC_1_0 { | |
9f83e0f7 | 178 | config.decode_compressed_chunk(&self.raw_data[header_len..], &head.iv, &head.tag)? |
3025b3a5 | 179 | } else { |
9f83e0f7 | 180 | config.decode_uncompressed_chunk(&self.raw_data[header_len..], &head.iv, &head.tag)? |
3025b3a5 DM |
181 | }; |
182 | return Ok(data); | |
183 | } else { | |
184 | bail!("unable to decrypt blob - missing CryptConfig"); | |
185 | } | |
69ecd8d5 DM |
186 | } else if magic == &AUTH_COMPR_BLOB_MAGIC_1_0 || magic == &AUTHENTICATED_BLOB_MAGIC_1_0 { |
187 | let header_len = std::mem::size_of::<AuthenticatedDataBlobHeader>(); | |
188 | let head = unsafe { | |
189 | (&self.raw_data[..header_len]).read_le_value::<AuthenticatedDataBlobHeader>()? | |
190 | }; | |
191 | ||
192 | let data_start = std::mem::size_of::<AuthenticatedDataBlobHeader>(); | |
193 | ||
194 | // Note: only verify if we have a crypt config | |
195 | if let Some(config) = config { | |
196 | let signature = config.compute_auth_tag(&self.raw_data[data_start..]); | |
197 | if signature != head.tag { | |
198 | bail!("verifying blob signature failed"); | |
199 | } | |
200 | } | |
201 | ||
202 | if magic == &AUTH_COMPR_BLOB_MAGIC_1_0 { | |
203 | let data = zstd::block::decompress(&self.raw_data[data_start..], 16*1024*1024)?; | |
204 | return Ok(data); | |
205 | } else { | |
206 | return Ok(self.raw_data[data_start..].to_vec()); | |
207 | } | |
3025b3a5 DM |
208 | } else { |
209 | bail!("Invalid blob magic number."); | |
210 | } | |
211 | } | |
a38c5d4d | 212 | |
69ecd8d5 DM |
213 | /// Create a signed DataBlob, optionally compressed |
214 | pub fn create_signed( | |
215 | data: &[u8], | |
216 | config: &CryptConfig, | |
217 | compress: bool, | |
218 | ) -> Result<Self, Error> { | |
219 | ||
781ac11c | 220 | if data.len() > MAX_BLOB_SIZE { |
69ecd8d5 DM |
221 | bail!("data blob too large ({} bytes).", data.len()); |
222 | } | |
223 | ||
224 | let compr_data; | |
225 | let (_compress, data, magic) = if compress { | |
226 | compr_data = zstd::block::compress(data, 1)?; | |
227 | // Note: We only use compression if result is shorter | |
228 | if compr_data.len() < data.len() { | |
229 | (true, &compr_data[..], AUTH_COMPR_BLOB_MAGIC_1_0) | |
230 | } else { | |
231 | (false, data, AUTHENTICATED_BLOB_MAGIC_1_0) | |
232 | } | |
233 | } else { | |
234 | (false, data, AUTHENTICATED_BLOB_MAGIC_1_0) | |
235 | }; | |
236 | ||
237 | let header_len = std::mem::size_of::<AuthenticatedDataBlobHeader>(); | |
238 | let mut raw_data = Vec::with_capacity(data.len() + header_len); | |
239 | ||
240 | let head = AuthenticatedDataBlobHeader { | |
241 | head: DataBlobHeader { magic, crc: [0; 4] }, | |
242 | tag: config.compute_auth_tag(data), | |
243 | }; | |
244 | unsafe { | |
245 | raw_data.write_le_value(head)?; | |
246 | } | |
247 | raw_data.extend_from_slice(data); | |
248 | ||
f889b158 DM |
249 | let mut blob = DataBlob { raw_data }; |
250 | blob.set_crc(blob.compute_crc()); | |
251 | ||
252 | return Ok(blob); | |
69ecd8d5 DM |
253 | } |
254 | ||
a38c5d4d DM |
255 | /// Create Instance from raw data |
256 | pub fn from_raw(data: Vec<u8>) -> Result<Self, Error> { | |
257 | ||
258 | if data.len() < std::mem::size_of::<DataBlobHeader>() { | |
259 | bail!("blob too small ({} bytes).", data.len()); | |
260 | } | |
261 | ||
262 | let magic = &data[0..8]; | |
263 | ||
264 | if magic == ENCR_COMPR_BLOB_MAGIC_1_0 || magic == ENCRYPTED_BLOB_MAGIC_1_0 { | |
265 | ||
266 | if data.len() < std::mem::size_of::<EncryptedDataBlobHeader>() { | |
267 | bail!("encrypted blob too small ({} bytes).", data.len()); | |
268 | } | |
269 | ||
270 | let blob = DataBlob { raw_data: data }; | |
271 | ||
272 | Ok(blob) | |
273 | } else if magic == COMPRESSED_BLOB_MAGIC_1_0 || magic == UNCOMPRESSED_BLOB_MAGIC_1_0 { | |
274 | ||
275 | let blob = DataBlob { raw_data: data }; | |
276 | ||
69ecd8d5 DM |
277 | Ok(blob) |
278 | } else if magic == AUTH_COMPR_BLOB_MAGIC_1_0 || magic == AUTHENTICATED_BLOB_MAGIC_1_0 { | |
279 | if data.len() < std::mem::size_of::<AuthenticatedDataBlobHeader>() { | |
280 | bail!("authenticated blob too small ({} bytes).", data.len()); | |
281 | } | |
282 | ||
283 | let blob = DataBlob { raw_data: data }; | |
284 | ||
a38c5d4d DM |
285 | Ok(blob) |
286 | } else { | |
287 | bail!("unable to parse raw blob - wrong magic"); | |
288 | } | |
289 | } | |
1f26fdef | 290 | } |