]>
Commit | Line | Data |
---|---|---|
9d12ba86 RR |
1 | /* |
2 | * Copyright 2016 Broadcom | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License, version 2, as | |
6 | * published by the Free Software Foundation (the "GPL"). | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License version 2 (GPLv2) for more details. | |
12 | * | |
13 | * You should have received a copy of the GNU General Public License | |
14 | * version 2 (GPLv2) along with this source code. | |
15 | */ | |
16 | ||
17 | #ifndef _CIPHER_H | |
18 | #define _CIPHER_H | |
19 | ||
20 | #include <linux/atomic.h> | |
21 | #include <linux/mailbox/brcm-message.h> | |
22 | #include <linux/mailbox_client.h> | |
23 | #include <crypto/aes.h> | |
24 | #include <crypto/internal/hash.h> | |
25 | #include <crypto/aead.h> | |
a59851d2 | 26 | #include <crypto/gcm.h> |
9d12ba86 RR |
27 | #include <crypto/sha.h> |
28 | #include <crypto/sha3.h> | |
29 | ||
30 | #include "spu.h" | |
31 | #include "spum.h" | |
32 | #include "spu2.h" | |
33 | ||
34 | /* Driver supports up to MAX_SPUS SPU blocks */ | |
35 | #define MAX_SPUS 16 | |
36 | ||
37 | #define ARC4_MIN_KEY_SIZE 1 | |
38 | #define ARC4_MAX_KEY_SIZE 256 | |
39 | #define ARC4_BLOCK_SIZE 1 | |
40 | #define ARC4_STATE_SIZE 4 | |
41 | ||
42 | #define CCM_AES_IV_SIZE 16 | |
9d12ba86 RR |
43 | #define CCM_ESP_IV_SIZE 8 |
44 | #define RFC4543_ICV_SIZE 16 | |
45 | ||
46 | #define MAX_KEY_SIZE ARC4_MAX_KEY_SIZE | |
47 | #define MAX_IV_SIZE AES_BLOCK_SIZE | |
48 | #define MAX_DIGEST_SIZE SHA3_512_DIGEST_SIZE | |
49 | #define MAX_ASSOC_SIZE 512 | |
50 | ||
51 | /* size of salt value for AES-GCM-ESP and AES-CCM-ESP */ | |
52 | #define GCM_ESP_SALT_SIZE 4 | |
53 | #define CCM_ESP_SALT_SIZE 3 | |
54 | #define MAX_SALT_SIZE GCM_ESP_SALT_SIZE | |
55 | #define GCM_ESP_SALT_OFFSET 0 | |
56 | #define CCM_ESP_SALT_OFFSET 1 | |
57 | ||
58 | #define GCM_ESP_DIGESTSIZE 16 | |
59 | ||
60 | #define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE | |
61 | ||
62 | /* | |
63 | * Maximum number of bytes from a non-final hash request that can be deferred | |
64 | * until more data is available. With new crypto API framework, this | |
65 | * can be no more than one block of data. | |
66 | */ | |
67 | #define HASH_CARRY_MAX MAX_HASH_BLOCK_SIZE | |
68 | ||
69 | /* Force at least 4-byte alignment of all SPU message fields */ | |
70 | #define SPU_MSG_ALIGN 4 | |
71 | ||
72 | /* Number of times to resend mailbox message if mb queue is full */ | |
73 | #define SPU_MB_RETRY_MAX 1000 | |
74 | ||
75 | /* op_counts[] indexes */ | |
76 | enum op_type { | |
77 | SPU_OP_CIPHER, | |
78 | SPU_OP_HASH, | |
79 | SPU_OP_HMAC, | |
80 | SPU_OP_AEAD, | |
81 | SPU_OP_NUM | |
82 | }; | |
83 | ||
84 | enum spu_spu_type { | |
85 | SPU_TYPE_SPUM, | |
86 | SPU_TYPE_SPU2, | |
87 | }; | |
88 | ||
89 | /* | |
90 | * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus, | |
91 | * respectively. | |
92 | */ | |
93 | enum spu_spu_subtype { | |
94 | SPU_SUBTYPE_SPUM_NS2, | |
95 | SPU_SUBTYPE_SPUM_NSP, | |
96 | SPU_SUBTYPE_SPU2_V1, | |
97 | SPU_SUBTYPE_SPU2_V2 | |
98 | }; | |
99 | ||
100 | struct spu_type_subtype { | |
101 | enum spu_spu_type type; | |
102 | enum spu_spu_subtype subtype; | |
103 | }; | |
104 | ||
105 | struct cipher_op { | |
106 | enum spu_cipher_alg alg; | |
107 | enum spu_cipher_mode mode; | |
108 | }; | |
109 | ||
110 | struct auth_op { | |
111 | enum hash_alg alg; | |
112 | enum hash_mode mode; | |
113 | }; | |
114 | ||
115 | struct iproc_alg_s { | |
116 | u32 type; | |
117 | union { | |
118 | struct crypto_alg crypto; | |
119 | struct ahash_alg hash; | |
120 | struct aead_alg aead; | |
121 | } alg; | |
122 | struct cipher_op cipher_info; | |
123 | struct auth_op auth_info; | |
124 | bool auth_first; | |
125 | bool registered; | |
126 | }; | |
127 | ||
128 | /* | |
129 | * Buffers for a SPU request/reply message pair. All part of one structure to | |
130 | * allow a single alloc per request. | |
131 | */ | |
132 | struct spu_msg_buf { | |
133 | /* Request message fragments */ | |
134 | ||
135 | /* | |
136 | * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC, | |
137 | * and BD header. For SPU2, holds FMD, OMD. | |
138 | */ | |
139 | u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; | |
140 | ||
141 | /* IV or counter. Size to include salt. Also used for XTS tweek. */ | |
142 | u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)]; | |
143 | ||
144 | /* Hash digest. request and response. */ | |
145 | u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)]; | |
146 | ||
147 | /* SPU request message padding */ | |
148 | u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)]; | |
149 | ||
150 | /* SPU-M request message STATUS field */ | |
151 | u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)]; | |
152 | ||
153 | /* Response message fragments */ | |
154 | ||
155 | /* SPU response message header */ | |
156 | u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; | |
157 | ||
158 | /* SPU response message STATUS field padding */ | |
159 | u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)]; | |
160 | ||
161 | /* SPU response message STATUS field */ | |
162 | u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)]; | |
163 | ||
164 | union { | |
165 | /* Buffers only used for ablkcipher */ | |
166 | struct { | |
167 | /* | |
168 | * Field used for either SUPDT when RC4 is used | |
169 | * -OR- tweak value when XTS/AES is used | |
170 | */ | |
171 | u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)]; | |
172 | } c; | |
173 | ||
174 | /* Buffers only used for aead */ | |
175 | struct { | |
176 | /* SPU response pad for GCM data */ | |
177 | u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)]; | |
178 | ||
179 | /* SPU request msg padding for GCM AAD */ | |
180 | u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)]; | |
181 | ||
182 | /* SPU response data to be discarded */ | |
183 | u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE, | |
184 | SPU_MSG_ALIGN)]; | |
185 | } a; | |
186 | }; | |
187 | }; | |
188 | ||
189 | struct iproc_ctx_s { | |
190 | u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE]; | |
191 | unsigned int enckeylen; | |
192 | ||
193 | u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE]; | |
194 | unsigned int authkeylen; | |
195 | ||
196 | u8 salt[MAX_SALT_SIZE]; | |
197 | unsigned int salt_len; | |
198 | unsigned int salt_offset; | |
199 | u8 iv[MAX_IV_SIZE]; | |
200 | ||
201 | unsigned int digestsize; | |
202 | ||
203 | struct iproc_alg_s *alg; | |
204 | bool is_esp; | |
205 | ||
206 | struct cipher_op cipher; | |
207 | enum spu_cipher_type cipher_type; | |
208 | ||
209 | struct auth_op auth; | |
210 | bool auth_first; | |
211 | ||
212 | /* | |
213 | * The maximum length in bytes of the payload in a SPU message for this | |
214 | * context. For SPU-M, the payload is the combination of AAD and data. | |
215 | * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF | |
216 | * indicates that there is no limit to the length of the SPU message | |
217 | * payload. | |
218 | */ | |
219 | unsigned int max_payload; | |
220 | ||
221 | struct crypto_aead *fallback_cipher; | |
222 | ||
223 | /* auth_type is determined during processing of request */ | |
224 | ||
225 | u8 ipad[MAX_HASH_BLOCK_SIZE]; | |
226 | u8 opad[MAX_HASH_BLOCK_SIZE]; | |
227 | ||
228 | /* | |
229 | * Buffer to hold SPU message header template. Template is created at | |
230 | * setkey time for ablkcipher requests, since most of the fields in the | |
231 | * header are known at that time. At request time, just fill in a few | |
232 | * missing pieces related to length of data in the request and IVs, etc. | |
233 | */ | |
234 | u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)]; | |
235 | ||
236 | /* Length of SPU request header */ | |
237 | u16 spu_req_hdr_len; | |
238 | ||
239 | /* Expected length of SPU response header */ | |
240 | u16 spu_resp_hdr_len; | |
241 | ||
242 | /* | |
243 | * shash descriptor - needed to perform incremental hashing in | |
244 | * in software, when hw doesn't support it. | |
245 | */ | |
246 | struct shash_desc *shash; | |
247 | ||
248 | bool is_rfc4543; /* RFC 4543 style of GMAC */ | |
249 | }; | |
250 | ||
251 | /* state from iproc_reqctx_s necessary for hash state export/import */ | |
252 | struct spu_hash_export_s { | |
253 | unsigned int total_todo; | |
254 | unsigned int total_sent; | |
255 | u8 hash_carry[HASH_CARRY_MAX]; | |
256 | unsigned int hash_carry_len; | |
257 | u8 incr_hash[MAX_DIGEST_SIZE]; | |
258 | bool is_sw_hmac; | |
259 | }; | |
260 | ||
261 | struct iproc_reqctx_s { | |
262 | /* general context */ | |
263 | struct crypto_async_request *parent; | |
264 | ||
265 | /* only valid after enqueue() */ | |
266 | struct iproc_ctx_s *ctx; | |
267 | ||
268 | u8 chan_idx; /* Mailbox channel to be used to submit this request */ | |
269 | ||
270 | /* total todo, rx'd, and sent for this request */ | |
271 | unsigned int total_todo; | |
272 | unsigned int total_received; /* only valid for ablkcipher */ | |
273 | unsigned int total_sent; | |
274 | ||
275 | /* | |
276 | * num bytes sent to hw from the src sg in this request. This can differ | |
277 | * from total_sent for incremental hashing. total_sent includes previous | |
278 | * init() and update() data. src_sent does not. | |
279 | */ | |
280 | unsigned int src_sent; | |
281 | ||
282 | /* | |
283 | * For AEAD requests, start of associated data. This will typically | |
284 | * point to the beginning of the src scatterlist from the request, | |
285 | * since assoc data is at the beginning of the src scatterlist rather | |
286 | * than in its own sg. | |
287 | */ | |
288 | struct scatterlist *assoc; | |
289 | ||
290 | /* | |
291 | * scatterlist entry and offset to start of data for next chunk. Crypto | |
292 | * API src scatterlist for AEAD starts with AAD, if present. For first | |
293 | * chunk, src_sg is sg entry at beginning of input data (after AAD). | |
294 | * src_skip begins at the offset in that sg entry where data begins. | |
295 | */ | |
296 | struct scatterlist *src_sg; | |
297 | int src_nents; /* Number of src entries with data */ | |
298 | u32 src_skip; /* bytes of current sg entry already used */ | |
299 | ||
300 | /* | |
301 | * Same for destination. For AEAD, if there is AAD, output data must | |
302 | * be written at offset following AAD. | |
303 | */ | |
304 | struct scatterlist *dst_sg; | |
305 | int dst_nents; /* Number of dst entries with data */ | |
306 | u32 dst_skip; /* bytes of current sg entry already written */ | |
307 | ||
308 | /* Mailbox message used to send this request to PDC driver */ | |
309 | struct brcm_message mb_mssg; | |
310 | ||
311 | bool bd_suppress; /* suppress BD field in SPU response? */ | |
312 | ||
313 | /* cipher context */ | |
314 | bool is_encrypt; | |
315 | ||
316 | /* | |
317 | * CBC mode: IV. CTR mode: counter. Else empty. Used as a DMA | |
318 | * buffer for AEAD requests. So allocate as DMAable memory. If IV | |
319 | * concatenated with salt, includes the salt. | |
320 | */ | |
321 | u8 *iv_ctr; | |
322 | /* Length of IV or counter, in bytes */ | |
323 | unsigned int iv_ctr_len; | |
324 | ||
325 | /* | |
326 | * Hash requests can be of any size, whether initial, update, or final. | |
327 | * A non-final request must be submitted to the SPU as an integral | |
328 | * number of blocks. This may leave data at the end of the request | |
329 | * that is not a full block. Since the request is non-final, it cannot | |
330 | * be padded. So, we write the remainder to this hash_carry buffer and | |
331 | * hold it until the next request arrives. The carry data is then | |
332 | * submitted at the beginning of the data in the next SPU msg. | |
333 | * hash_carry_len is the number of bytes currently in hash_carry. These | |
334 | * fields are only used for ahash requests. | |
335 | */ | |
336 | u8 hash_carry[HASH_CARRY_MAX]; | |
337 | unsigned int hash_carry_len; | |
338 | unsigned int is_final; /* is this the final for the hash op? */ | |
339 | ||
340 | /* | |
341 | * Digest from incremental hash is saved here to include in next hash | |
342 | * operation. Cannot be stored in req->result for truncated hashes, | |
343 | * since result may be sized for final digest. Cannot be saved in | |
344 | * msg_buf because that gets deleted between incremental hash ops | |
345 | * and is not saved as part of export(). | |
346 | */ | |
347 | u8 incr_hash[MAX_DIGEST_SIZE]; | |
348 | ||
349 | /* hmac context */ | |
350 | bool is_sw_hmac; | |
351 | ||
352 | /* aead context */ | |
353 | struct crypto_tfm *old_tfm; | |
354 | crypto_completion_t old_complete; | |
355 | void *old_data; | |
356 | ||
357 | gfp_t gfp; | |
358 | ||
359 | /* Buffers used to build SPU request and response messages */ | |
360 | struct spu_msg_buf msg_buf; | |
361 | }; | |
362 | ||
363 | /* | |
364 | * Structure encapsulates a set of function pointers specific to the type of | |
365 | * SPU hardware running. These functions handling creation and parsing of | |
366 | * SPU request messages and SPU response messages. Includes hardware-specific | |
367 | * values read from device tree. | |
368 | */ | |
369 | struct spu_hw { | |
370 | void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len); | |
371 | u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg, | |
372 | enum spu_cipher_mode cipher_mode, | |
373 | unsigned int blocksize); | |
374 | u32 (*spu_payload_length)(u8 *spu_hdr); | |
375 | u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len, | |
376 | bool is_hash); | |
377 | u16 (*spu_hash_pad_len)(enum hash_alg hash_alg, | |
378 | enum hash_mode hash_mode, u32 chunksize, | |
379 | u16 hash_block_size); | |
380 | u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode, | |
381 | unsigned int data_size); | |
382 | u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode, | |
383 | unsigned int assoc_len, | |
384 | unsigned int iv_len, bool is_encrypt); | |
385 | u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode, | |
386 | u16 iv_len); | |
387 | enum hash_type (*spu_hash_type)(u32 src_sent); | |
388 | u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg, | |
389 | enum hash_type); | |
390 | u32 (*spu_create_request)(u8 *spu_hdr, | |
391 | struct spu_request_opts *req_opts, | |
392 | struct spu_cipher_parms *cipher_parms, | |
393 | struct spu_hash_parms *hash_parms, | |
394 | struct spu_aead_parms *aead_parms, | |
395 | unsigned int data_size); | |
396 | u16 (*spu_cipher_req_init)(u8 *spu_hdr, | |
397 | struct spu_cipher_parms *cipher_parms); | |
398 | void (*spu_cipher_req_finish)(u8 *spu_hdr, | |
399 | u16 spu_req_hdr_len, | |
400 | unsigned int is_inbound, | |
401 | struct spu_cipher_parms *cipher_parms, | |
402 | bool update_key, | |
403 | unsigned int data_size); | |
404 | void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding, | |
405 | u32 hash_pad_len, enum hash_alg auth_alg, | |
406 | enum hash_mode auth_mode, | |
407 | unsigned int total_sent, u32 status_padding); | |
408 | u8 (*spu_xts_tweak_in_payload)(void); | |
409 | u8 (*spu_tx_status_len)(void); | |
410 | u8 (*spu_rx_status_len)(void); | |
411 | int (*spu_status_process)(u8 *statp); | |
412 | void (*spu_ccm_update_iv)(unsigned int digestsize, | |
413 | struct spu_cipher_parms *cipher_parms, | |
414 | unsigned int assoclen, unsigned int chunksize, | |
415 | bool is_encrypt, bool is_esp); | |
416 | u32 (*spu_wordalign_padlen)(u32 data_size); | |
417 | ||
418 | /* The base virtual address of the SPU hw registers */ | |
419 | void __iomem *reg_vbase[MAX_SPUS]; | |
420 | ||
421 | /* Version of the SPU hardware */ | |
422 | enum spu_spu_type spu_type; | |
423 | ||
424 | /* Sub-version of the SPU hardware */ | |
425 | enum spu_spu_subtype spu_subtype; | |
426 | ||
427 | /* The number of SPUs on this platform */ | |
428 | u32 num_spu; | |
9166c443 | 429 | |
430 | /* The number of SPU channels on this platform */ | |
431 | u32 num_chan; | |
9d12ba86 RR |
432 | }; |
433 | ||
434 | struct device_private { | |
9166c443 | 435 | struct platform_device *pdev; |
9d12ba86 RR |
436 | |
437 | struct spu_hw spu; | |
438 | ||
439 | atomic_t session_count; /* number of streams active */ | |
440 | atomic_t stream_count; /* monotonic counter for streamID's */ | |
441 | ||
442 | /* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */ | |
443 | u8 bcm_hdr_len; | |
444 | ||
445 | /* The index of the channel to use for the next crypto request */ | |
446 | atomic_t next_chan; | |
447 | ||
448 | struct dentry *debugfs_dir; | |
449 | struct dentry *debugfs_stats; | |
450 | ||
451 | /* Number of request bytes processed and result bytes returned */ | |
452 | atomic64_t bytes_in; | |
453 | atomic64_t bytes_out; | |
454 | ||
455 | /* Number of operations of each type */ | |
456 | atomic_t op_counts[SPU_OP_NUM]; | |
457 | ||
458 | atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST]; | |
459 | atomic_t hash_cnt[HASH_ALG_LAST]; | |
460 | atomic_t hmac_cnt[HASH_ALG_LAST]; | |
461 | atomic_t aead_cnt[AEAD_TYPE_LAST]; | |
462 | ||
463 | /* Number of calls to setkey() for each operation type */ | |
464 | atomic_t setkey_cnt[SPU_OP_NUM]; | |
465 | ||
466 | /* Number of times request was resubmitted because mb was full */ | |
467 | atomic_t mb_no_spc; | |
468 | ||
469 | /* Number of mailbox send failures */ | |
470 | atomic_t mb_send_fail; | |
471 | ||
472 | /* Number of ICV check failures for AEAD messages */ | |
473 | atomic_t bad_icv; | |
474 | ||
9166c443 | 475 | struct mbox_client mcl; |
9d12ba86 | 476 | |
9166c443 | 477 | /* Array of mailbox channel pointers, one for each channel */ |
478 | struct mbox_chan **mbox; | |
9d12ba86 RR |
479 | }; |
480 | ||
481 | extern struct device_private iproc_priv; | |
482 | ||
483 | #endif |