]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/crypto/caam/caamalg.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
ASoC: max98504: Add missing MAX98504 on SND_SOC_ALL_CODECS
[mirror_ubuntu-bionic-kernel.git] / drivers / crypto / caam / caamalg.c
1 /*
2 * caam - Freescale FSL CAAM support for crypto API
3 *
4 * Copyright 2008-2011 Freescale Semiconductor, Inc.
5 *
6 * Based on talitos crypto API driver.
7 *
8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
9 *
10 * --------------- ---------------
11 * | JobDesc #1 |-------------------->| ShareDesc |
12 * | *(packet 1) | | (PDB) |
13 * --------------- |------------->| (hashKey) |
14 * . | | (cipherKey) |
15 * . | |-------->| (operation) |
16 * --------------- | | ---------------
17 * | JobDesc #2 |------| |
18 * | *(packet 2) | |
19 * --------------- |
20 * . |
21 * . |
22 * --------------- |
23 * | JobDesc #3 |------------
24 * | *(packet 3) |
25 * ---------------
26 *
27 * The SharedDesc never changes for a connection unless rekeyed, but
28 * each packet will likely be in a different place. So all we need
29 * to know to process the packet is where the input is, where the
30 * output goes, and what context we want to process with. Context is
31 * in the SharedDesc, packet references in the JobDesc.
32 *
33 * So, a job desc looks like:
34 *
35 * ---------------------
36 * | Header |
37 * | ShareDesc Pointer |
38 * | SEQ_OUT_PTR |
39 * | (output buffer) |
40 * | (output length) |
41 * | SEQ_IN_PTR |
42 * | (input buffer) |
43 * | (input length) |
44 * ---------------------
45 */
46
47 #include "compat.h"
48
49 #include "regs.h"
50 #include "intern.h"
51 #include "desc_constr.h"
52 #include "jr.h"
53 #include "error.h"
54 #include "sg_sw_sec4.h"
55 #include "key_gen.h"
56
57 /*
58 * crypto alg
59 */
60 #define CAAM_CRA_PRIORITY 3000
61 /* max key is sum of AES_MAX_KEY_SIZE, max split key size */
62 #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
63 CTR_RFC3686_NONCE_SIZE + \
64 SHA512_DIGEST_SIZE * 2)
65 /* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
66 #define CAAM_MAX_IV_LENGTH 16
67
68 #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2)
69 #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
70 CAAM_CMD_SZ * 4)
71 #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \
72 CAAM_CMD_SZ * 5)
73
74 /* length of descriptors text */
75 #define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
76 #define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ)
77 #define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
78 #define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ)
79
80 /* Note: Nonce is counted in enckeylen */
81 #define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ)
82
83 #define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ)
84 #define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ)
85 #define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ)
86
87 #define DESC_GCM_BASE (3 * CAAM_CMD_SZ)
88 #define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ)
89 #define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ)
90
91 #define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ)
92 #define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
93 #define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ)
94
95 #define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ)
96 #define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ)
97 #define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ)
98
99 #define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ)
100 #define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \
101 20 * CAAM_CMD_SZ)
102 #define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \
103 15 * CAAM_CMD_SZ)
104
105 #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN)
106 #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
107
108 #ifdef DEBUG
109 /* for print_hex_dumps with line references */
110 #define debug(format, arg...) printk(format, arg)
111 #else
112 #define debug(format, arg...)
113 #endif
114
115 #ifdef DEBUG
116 #include <linux/highmem.h>
117
118 static void dbg_dump_sg(const char *level, const char *prefix_str,
119 int prefix_type, int rowsize, int groupsize,
120 struct scatterlist *sg, size_t tlen, bool ascii,
121 bool may_sleep)
122 {
123 struct scatterlist *it;
124 void *it_page;
125 size_t len;
126 void *buf;
127
128 for (it = sg; it != NULL && tlen > 0 ; it = sg_next(sg)) {
129 /*
130 * make sure the scatterlist's page
131 * has a valid virtual memory mapping
132 */
133 it_page = kmap_atomic(sg_page(it));
134 if (unlikely(!it_page)) {
135 printk(KERN_ERR "dbg_dump_sg: kmap failed\n");
136 return;
137 }
138
139 buf = it_page + it->offset;
140 len = min(tlen, it->length);
141 print_hex_dump(level, prefix_str, prefix_type, rowsize,
142 groupsize, buf, len, ascii);
143 tlen -= len;
144
145 kunmap_atomic(it_page);
146 }
147 }
148 #endif
149
150 static struct list_head alg_list;
151
152 struct caam_alg_entry {
153 int class1_alg_type;
154 int class2_alg_type;
155 int alg_op;
156 bool rfc3686;
157 bool geniv;
158 };
159
160 struct caam_aead_alg {
161 struct aead_alg aead;
162 struct caam_alg_entry caam;
163 bool registered;
164 };
165
166 /* Set DK bit in class 1 operation if shared */
167 static inline void append_dec_op1(u32 *desc, u32 type)
168 {
169 u32 *jump_cmd, *uncond_jump_cmd;
170
171 /* DK bit is valid only for AES */
172 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
173 append_operation(desc, type | OP_ALG_AS_INITFINAL |
174 OP_ALG_DECRYPT);
175 return;
176 }
177
178 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
179 append_operation(desc, type | OP_ALG_AS_INITFINAL |
180 OP_ALG_DECRYPT);
181 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
182 set_jump_tgt_here(desc, jump_cmd);
183 append_operation(desc, type | OP_ALG_AS_INITFINAL |
184 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
185 set_jump_tgt_here(desc, uncond_jump_cmd);
186 }
187
188 /*
189 * For aead functions, read payload and write payload,
190 * both of which are specified in req->src and req->dst
191 */
192 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
193 {
194 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
195 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
196 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
197 }
198
199 /*
200 * For ablkcipher encrypt and decrypt, read from req->src and
201 * write to req->dst
202 */
203 static inline void ablkcipher_append_src_dst(u32 *desc)
204 {
205 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
206 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
207 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
208 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
209 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
210 }
211
212 /*
213 * per-session context
214 */
215 struct caam_ctx {
216 struct device *jrdev;
217 u32 sh_desc_enc[DESC_MAX_USED_LEN];
218 u32 sh_desc_dec[DESC_MAX_USED_LEN];
219 u32 sh_desc_givenc[DESC_MAX_USED_LEN];
220 dma_addr_t sh_desc_enc_dma;
221 dma_addr_t sh_desc_dec_dma;
222 dma_addr_t sh_desc_givenc_dma;
223 u32 class1_alg_type;
224 u32 class2_alg_type;
225 u32 alg_op;
226 u8 key[CAAM_MAX_KEY_SIZE];
227 dma_addr_t key_dma;
228 unsigned int enckeylen;
229 unsigned int split_key_len;
230 unsigned int split_key_pad_len;
231 unsigned int authsize;
232 };
233
234 static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
235 int keys_fit_inline, bool is_rfc3686)
236 {
237 u32 *nonce;
238 unsigned int enckeylen = ctx->enckeylen;
239
240 /*
241 * RFC3686 specific:
242 * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE}
243 * | enckeylen = encryption key size + nonce size
244 */
245 if (is_rfc3686)
246 enckeylen -= CTR_RFC3686_NONCE_SIZE;
247
248 if (keys_fit_inline) {
249 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
250 ctx->split_key_len, CLASS_2 |
251 KEY_DEST_MDHA_SPLIT | KEY_ENC);
252 append_key_as_imm(desc, (void *)ctx->key +
253 ctx->split_key_pad_len, enckeylen,
254 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
255 } else {
256 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
257 KEY_DEST_MDHA_SPLIT | KEY_ENC);
258 append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
259 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
260 }
261
262 /* Load Counter into CONTEXT1 reg */
263 if (is_rfc3686) {
264 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len +
265 enckeylen);
266 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
267 LDST_CLASS_IND_CCB |
268 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
269 append_move(desc,
270 MOVE_SRC_OUTFIFO |
271 MOVE_DEST_CLASS1CTX |
272 (16 << MOVE_OFFSET_SHIFT) |
273 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
274 }
275 }
276
277 static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
278 int keys_fit_inline, bool is_rfc3686)
279 {
280 u32 *key_jump_cmd;
281
282 /* Note: Context registers are saved. */
283 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
284
285 /* Skip if already shared */
286 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
287 JUMP_COND_SHRD);
288
289 append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
290
291 set_jump_tgt_here(desc, key_jump_cmd);
292 }
293
294 static int aead_null_set_sh_desc(struct crypto_aead *aead)
295 {
296 struct caam_ctx *ctx = crypto_aead_ctx(aead);
297 struct device *jrdev = ctx->jrdev;
298 bool keys_fit_inline = false;
299 u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd;
300 u32 *desc;
301
302 /*
303 * Job Descriptor and Shared Descriptors
304 * must all fit into the 64-word Descriptor h/w Buffer
305 */
306 if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN +
307 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
308 keys_fit_inline = true;
309
310 /* aead_encrypt shared descriptor */
311 desc = ctx->sh_desc_enc;
312
313 init_sh_desc(desc, HDR_SHARE_SERIAL);
314
315 /* Skip if already shared */
316 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
317 JUMP_COND_SHRD);
318 if (keys_fit_inline)
319 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
320 ctx->split_key_len, CLASS_2 |
321 KEY_DEST_MDHA_SPLIT | KEY_ENC);
322 else
323 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
324 KEY_DEST_MDHA_SPLIT | KEY_ENC);
325 set_jump_tgt_here(desc, key_jump_cmd);
326
327 /* assoclen + cryptlen = seqinlen */
328 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
329
330 /* Prepare to read and write cryptlen + assoclen bytes */
331 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
332 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
333
334 /*
335 * MOVE_LEN opcode is not available in all SEC HW revisions,
336 * thus need to do some magic, i.e. self-patch the descriptor
337 * buffer.
338 */
339 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
340 MOVE_DEST_MATH3 |
341 (0x6 << MOVE_LEN_SHIFT));
342 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
343 MOVE_DEST_DESCBUF |
344 MOVE_WAITCOMP |
345 (0x8 << MOVE_LEN_SHIFT));
346
347 /* Class 2 operation */
348 append_operation(desc, ctx->class2_alg_type |
349 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
350
351 /* Read and write cryptlen bytes */
352 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
353
354 set_move_tgt_here(desc, read_move_cmd);
355 set_move_tgt_here(desc, write_move_cmd);
356 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
357 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
358 MOVE_AUX_LS);
359
360 /* Write ICV */
361 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
362 LDST_SRCDST_BYTE_CONTEXT);
363
364 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
365 desc_bytes(desc),
366 DMA_TO_DEVICE);
367 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
368 dev_err(jrdev, "unable to map shared descriptor\n");
369 return -ENOMEM;
370 }
371 #ifdef DEBUG
372 print_hex_dump(KERN_ERR,
373 "aead null enc shdesc@"__stringify(__LINE__)": ",
374 DUMP_PREFIX_ADDRESS, 16, 4, desc,
375 desc_bytes(desc), 1);
376 #endif
377
378 /*
379 * Job Descriptor and Shared Descriptors
380 * must all fit into the 64-word Descriptor h/w Buffer
381 */
382 keys_fit_inline = false;
383 if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN +
384 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
385 keys_fit_inline = true;
386
387 desc = ctx->sh_desc_dec;
388
389 /* aead_decrypt shared descriptor */
390 init_sh_desc(desc, HDR_SHARE_SERIAL);
391
392 /* Skip if already shared */
393 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
394 JUMP_COND_SHRD);
395 if (keys_fit_inline)
396 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
397 ctx->split_key_len, CLASS_2 |
398 KEY_DEST_MDHA_SPLIT | KEY_ENC);
399 else
400 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
401 KEY_DEST_MDHA_SPLIT | KEY_ENC);
402 set_jump_tgt_here(desc, key_jump_cmd);
403
404 /* Class 2 operation */
405 append_operation(desc, ctx->class2_alg_type |
406 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
407
408 /* assoclen + cryptlen = seqoutlen */
409 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
410
411 /* Prepare to read and write cryptlen + assoclen bytes */
412 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
413 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
414
415 /*
416 * MOVE_LEN opcode is not available in all SEC HW revisions,
417 * thus need to do some magic, i.e. self-patch the descriptor
418 * buffer.
419 */
420 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
421 MOVE_DEST_MATH2 |
422 (0x6 << MOVE_LEN_SHIFT));
423 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
424 MOVE_DEST_DESCBUF |
425 MOVE_WAITCOMP |
426 (0x8 << MOVE_LEN_SHIFT));
427
428 /* Read and write cryptlen bytes */
429 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
430
431 /*
432 * Insert a NOP here, since we need at least 4 instructions between
433 * code patching the descriptor buffer and the location being patched.
434 */
435 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
436 set_jump_tgt_here(desc, jump_cmd);
437
438 set_move_tgt_here(desc, read_move_cmd);
439 set_move_tgt_here(desc, write_move_cmd);
440 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
441 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
442 MOVE_AUX_LS);
443 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
444
445 /* Load ICV */
446 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
447 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
448
449 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
450 desc_bytes(desc),
451 DMA_TO_DEVICE);
452 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
453 dev_err(jrdev, "unable to map shared descriptor\n");
454 return -ENOMEM;
455 }
456 #ifdef DEBUG
457 print_hex_dump(KERN_ERR,
458 "aead null dec shdesc@"__stringify(__LINE__)": ",
459 DUMP_PREFIX_ADDRESS, 16, 4, desc,
460 desc_bytes(desc), 1);
461 #endif
462
463 return 0;
464 }
465
466 static int aead_set_sh_desc(struct crypto_aead *aead)
467 {
468 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
469 struct caam_aead_alg, aead);
470 unsigned int ivsize = crypto_aead_ivsize(aead);
471 struct caam_ctx *ctx = crypto_aead_ctx(aead);
472 struct device *jrdev = ctx->jrdev;
473 bool keys_fit_inline;
474 u32 geniv, moveiv;
475 u32 ctx1_iv_off = 0;
476 u32 *desc;
477 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
478 OP_ALG_AAI_CTR_MOD128);
479 const bool is_rfc3686 = alg->caam.rfc3686;
480
481 if (!ctx->authsize)
482 return 0;
483
484 /* NULL encryption / decryption */
485 if (!ctx->enckeylen)
486 return aead_null_set_sh_desc(aead);
487
488 /*
489 * AES-CTR needs to load IV in CONTEXT1 reg
490 * at an offset of 128bits (16bytes)
491 * CONTEXT1[255:128] = IV
492 */
493 if (ctr_mode)
494 ctx1_iv_off = 16;
495
496 /*
497 * RFC3686 specific:
498 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
499 */
500 if (is_rfc3686)
501 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
502
503 if (alg->caam.geniv)
504 goto skip_enc;
505
506 /*
507 * Job Descriptor and Shared Descriptors
508 * must all fit into the 64-word Descriptor h/w Buffer
509 */
510 keys_fit_inline = false;
511 if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN +
512 ctx->split_key_pad_len + ctx->enckeylen +
513 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
514 CAAM_DESC_BYTES_MAX)
515 keys_fit_inline = true;
516
517 /* aead_encrypt shared descriptor */
518 desc = ctx->sh_desc_enc;
519
520 /* Note: Context registers are saved. */
521 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
522
523 /* Class 2 operation */
524 append_operation(desc, ctx->class2_alg_type |
525 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
526
527 /* Read and write assoclen bytes */
528 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
529 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
530
531 /* Skip assoc data */
532 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
533
534 /* read assoc before reading payload */
535 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
536 FIFOLDST_VLF);
537
538 /* Load Counter into CONTEXT1 reg */
539 if (is_rfc3686)
540 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
541 LDST_SRCDST_BYTE_CONTEXT |
542 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
543 LDST_OFFSET_SHIFT));
544
545 /* Class 1 operation */
546 append_operation(desc, ctx->class1_alg_type |
547 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
548
549 /* Read and write cryptlen bytes */
550 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
551 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
552 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
553
554 /* Write ICV */
555 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
556 LDST_SRCDST_BYTE_CONTEXT);
557
558 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
559 desc_bytes(desc),
560 DMA_TO_DEVICE);
561 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
562 dev_err(jrdev, "unable to map shared descriptor\n");
563 return -ENOMEM;
564 }
565 #ifdef DEBUG
566 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
567 DUMP_PREFIX_ADDRESS, 16, 4, desc,
568 desc_bytes(desc), 1);
569 #endif
570
571 skip_enc:
572 /*
573 * Job Descriptor and Shared Descriptors
574 * must all fit into the 64-word Descriptor h/w Buffer
575 */
576 keys_fit_inline = false;
577 if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN +
578 ctx->split_key_pad_len + ctx->enckeylen +
579 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
580 CAAM_DESC_BYTES_MAX)
581 keys_fit_inline = true;
582
583 /* aead_decrypt shared descriptor */
584 desc = ctx->sh_desc_dec;
585
586 /* Note: Context registers are saved. */
587 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
588
589 /* Class 2 operation */
590 append_operation(desc, ctx->class2_alg_type |
591 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
592
593 /* Read and write assoclen bytes */
594 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
595 if (alg->caam.geniv)
596 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize);
597 else
598 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
599
600 /* Skip assoc data */
601 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
602
603 /* read assoc before reading payload */
604 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
605 KEY_VLF);
606
607 if (alg->caam.geniv) {
608 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
609 LDST_SRCDST_BYTE_CONTEXT |
610 (ctx1_iv_off << LDST_OFFSET_SHIFT));
611 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
612 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
613 }
614
615 /* Load Counter into CONTEXT1 reg */
616 if (is_rfc3686)
617 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
618 LDST_SRCDST_BYTE_CONTEXT |
619 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
620 LDST_OFFSET_SHIFT));
621
622 /* Choose operation */
623 if (ctr_mode)
624 append_operation(desc, ctx->class1_alg_type |
625 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
626 else
627 append_dec_op1(desc, ctx->class1_alg_type);
628
629 /* Read and write cryptlen bytes */
630 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
631 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
632 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
633
634 /* Load ICV */
635 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
636 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
637
638 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
639 desc_bytes(desc),
640 DMA_TO_DEVICE);
641 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
642 dev_err(jrdev, "unable to map shared descriptor\n");
643 return -ENOMEM;
644 }
645 #ifdef DEBUG
646 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
647 DUMP_PREFIX_ADDRESS, 16, 4, desc,
648 desc_bytes(desc), 1);
649 #endif
650
651 if (!alg->caam.geniv)
652 goto skip_givenc;
653
654 /*
655 * Job Descriptor and Shared Descriptors
656 * must all fit into the 64-word Descriptor h/w Buffer
657 */
658 keys_fit_inline = false;
659 if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN +
660 ctx->split_key_pad_len + ctx->enckeylen +
661 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
662 CAAM_DESC_BYTES_MAX)
663 keys_fit_inline = true;
664
665 /* aead_givencrypt shared descriptor */
666 desc = ctx->sh_desc_enc;
667
668 /* Note: Context registers are saved. */
669 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
670
671 if (is_rfc3686)
672 goto copy_iv;
673
674 /* Generate IV */
675 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
676 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
677 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
678 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
679 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
680 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
681 append_move(desc, MOVE_WAITCOMP |
682 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
683 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
684 (ivsize << MOVE_LEN_SHIFT));
685 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
686
687 copy_iv:
688 /* Copy IV to class 1 context */
689 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
690 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
691 (ivsize << MOVE_LEN_SHIFT));
692
693 /* Return to encryption */
694 append_operation(desc, ctx->class2_alg_type |
695 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
696
697 /* Read and write assoclen bytes */
698 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
699 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
700
701 /* ivsize + cryptlen = seqoutlen - authsize */
702 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
703
704 /* Skip assoc data */
705 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
706
707 /* read assoc before reading payload */
708 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
709 KEY_VLF);
710
711 /* Copy iv from outfifo to class 2 fifo */
712 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
713 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
714 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
715 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
716 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
717 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
718
719 /* Load Counter into CONTEXT1 reg */
720 if (is_rfc3686)
721 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
722 LDST_SRCDST_BYTE_CONTEXT |
723 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
724 LDST_OFFSET_SHIFT));
725
726 /* Class 1 operation */
727 append_operation(desc, ctx->class1_alg_type |
728 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
729
730 /* Will write ivsize + cryptlen */
731 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
732
733 /* Not need to reload iv */
734 append_seq_fifo_load(desc, ivsize,
735 FIFOLD_CLASS_SKIP);
736
737 /* Will read cryptlen */
738 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
739 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
740
741 /* Write ICV */
742 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
743 LDST_SRCDST_BYTE_CONTEXT);
744
745 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
746 desc_bytes(desc),
747 DMA_TO_DEVICE);
748 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
749 dev_err(jrdev, "unable to map shared descriptor\n");
750 return -ENOMEM;
751 }
752 #ifdef DEBUG
753 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
754 DUMP_PREFIX_ADDRESS, 16, 4, desc,
755 desc_bytes(desc), 1);
756 #endif
757
758 skip_givenc:
759 return 0;
760 }
761
762 static int aead_setauthsize(struct crypto_aead *authenc,
763 unsigned int authsize)
764 {
765 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
766
767 ctx->authsize = authsize;
768 aead_set_sh_desc(authenc);
769
770 return 0;
771 }
772
773 static int gcm_set_sh_desc(struct crypto_aead *aead)
774 {
775 struct caam_ctx *ctx = crypto_aead_ctx(aead);
776 struct device *jrdev = ctx->jrdev;
777 bool keys_fit_inline = false;
778 u32 *key_jump_cmd, *zero_payload_jump_cmd,
779 *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2;
780 u32 *desc;
781
782 if (!ctx->enckeylen || !ctx->authsize)
783 return 0;
784
785 /*
786 * AES GCM encrypt shared descriptor
787 * Job Descriptor and Shared Descriptor
788 * must fit into the 64-word Descriptor h/w Buffer
789 */
790 if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN +
791 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
792 keys_fit_inline = true;
793
794 desc = ctx->sh_desc_enc;
795
796 init_sh_desc(desc, HDR_SHARE_SERIAL);
797
798 /* skip key loading if they are loaded due to sharing */
799 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
800 JUMP_COND_SHRD | JUMP_COND_SELF);
801 if (keys_fit_inline)
802 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
803 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
804 else
805 append_key(desc, ctx->key_dma, ctx->enckeylen,
806 CLASS_1 | KEY_DEST_CLASS_REG);
807 set_jump_tgt_here(desc, key_jump_cmd);
808
809 /* class 1 operation */
810 append_operation(desc, ctx->class1_alg_type |
811 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
812
813 /* if assoclen + cryptlen is ZERO, skip to ICV write */
814 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
815 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
816 JUMP_COND_MATH_Z);
817
818 /* if assoclen is ZERO, skip reading the assoc data */
819 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
820 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
821 JUMP_COND_MATH_Z);
822
823 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
824
825 /* skip assoc data */
826 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
827
828 /* cryptlen = seqinlen - assoclen */
829 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
830
831 /* if cryptlen is ZERO jump to zero-payload commands */
832 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
833 JUMP_COND_MATH_Z);
834
835 /* read assoc data */
836 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
837 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
838 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
839
840 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
841
842 /* write encrypted data */
843 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
844
845 /* read payload data */
846 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
847 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
848
849 /* jump the zero-payload commands */
850 append_jump(desc, JUMP_TEST_ALL | 2);
851
852 /* zero-payload commands */
853 set_jump_tgt_here(desc, zero_payload_jump_cmd);
854
855 /* read assoc data */
856 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
857 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
858
859 /* There is no input data */
860 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
861
862 /* write ICV */
863 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
864 LDST_SRCDST_BYTE_CONTEXT);
865
866 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
867 desc_bytes(desc),
868 DMA_TO_DEVICE);
869 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
870 dev_err(jrdev, "unable to map shared descriptor\n");
871 return -ENOMEM;
872 }
873 #ifdef DEBUG
874 print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ",
875 DUMP_PREFIX_ADDRESS, 16, 4, desc,
876 desc_bytes(desc), 1);
877 #endif
878
879 /*
880 * Job Descriptor and Shared Descriptors
881 * must all fit into the 64-word Descriptor h/w Buffer
882 */
883 keys_fit_inline = false;
884 if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN +
885 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
886 keys_fit_inline = true;
887
888 desc = ctx->sh_desc_dec;
889
890 init_sh_desc(desc, HDR_SHARE_SERIAL);
891
892 /* skip key loading if they are loaded due to sharing */
893 key_jump_cmd = append_jump(desc, JUMP_JSL |
894 JUMP_TEST_ALL | JUMP_COND_SHRD |
895 JUMP_COND_SELF);
896 if (keys_fit_inline)
897 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
898 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
899 else
900 append_key(desc, ctx->key_dma, ctx->enckeylen,
901 CLASS_1 | KEY_DEST_CLASS_REG);
902 set_jump_tgt_here(desc, key_jump_cmd);
903
904 /* class 1 operation */
905 append_operation(desc, ctx->class1_alg_type |
906 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
907
908 /* if assoclen is ZERO, skip reading the assoc data */
909 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
910 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
911 JUMP_COND_MATH_Z);
912
913 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
914
915 /* skip assoc data */
916 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
917
918 /* read assoc data */
919 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
920 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
921
922 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
923
924 /* cryptlen = seqoutlen - assoclen */
925 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
926
927 /* jump to zero-payload command if cryptlen is zero */
928 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
929 JUMP_COND_MATH_Z);
930
931 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
932
933 /* store encrypted data */
934 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
935
936 /* read payload data */
937 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
938 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
939
940 /* zero-payload command */
941 set_jump_tgt_here(desc, zero_payload_jump_cmd);
942
943 /* read ICV */
944 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
945 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
946
947 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
948 desc_bytes(desc),
949 DMA_TO_DEVICE);
950 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
951 dev_err(jrdev, "unable to map shared descriptor\n");
952 return -ENOMEM;
953 }
954 #ifdef DEBUG
955 print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ",
956 DUMP_PREFIX_ADDRESS, 16, 4, desc,
957 desc_bytes(desc), 1);
958 #endif
959
960 return 0;
961 }
962
963 static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
964 {
965 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
966
967 ctx->authsize = authsize;
968 gcm_set_sh_desc(authenc);
969
970 return 0;
971 }
972
973 static int rfc4106_set_sh_desc(struct crypto_aead *aead)
974 {
975 struct caam_ctx *ctx = crypto_aead_ctx(aead);
976 struct device *jrdev = ctx->jrdev;
977 bool keys_fit_inline = false;
978 u32 *key_jump_cmd;
979 u32 *desc;
980
981 if (!ctx->enckeylen || !ctx->authsize)
982 return 0;
983
984 /*
985 * RFC4106 encrypt shared descriptor
986 * Job Descriptor and Shared Descriptor
987 * must fit into the 64-word Descriptor h/w Buffer
988 */
989 if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN +
990 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
991 keys_fit_inline = true;
992
993 desc = ctx->sh_desc_enc;
994
995 init_sh_desc(desc, HDR_SHARE_SERIAL);
996
997 /* Skip key loading if it is loaded due to sharing */
998 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
999 JUMP_COND_SHRD);
1000 if (keys_fit_inline)
1001 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1002 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1003 else
1004 append_key(desc, ctx->key_dma, ctx->enckeylen,
1005 CLASS_1 | KEY_DEST_CLASS_REG);
1006 set_jump_tgt_here(desc, key_jump_cmd);
1007
1008 /* Class 1 operation */
1009 append_operation(desc, ctx->class1_alg_type |
1010 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1011
1012 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
1013 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1014
1015 /* Read assoc data */
1016 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1017 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1018
1019 /* Skip IV */
1020 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1021
1022 /* Will read cryptlen bytes */
1023 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1024
1025 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1026 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1027
1028 /* Skip assoc data */
1029 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1030
1031 /* cryptlen = seqoutlen - assoclen */
1032 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
1033
1034 /* Write encrypted data */
1035 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1036
1037 /* Read payload data */
1038 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1039 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1040
1041 /* Write ICV */
1042 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1043 LDST_SRCDST_BYTE_CONTEXT);
1044
1045 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1046 desc_bytes(desc),
1047 DMA_TO_DEVICE);
1048 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1049 dev_err(jrdev, "unable to map shared descriptor\n");
1050 return -ENOMEM;
1051 }
1052 #ifdef DEBUG
1053 print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ",
1054 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1055 desc_bytes(desc), 1);
1056 #endif
1057
1058 /*
1059 * Job Descriptor and Shared Descriptors
1060 * must all fit into the 64-word Descriptor h/w Buffer
1061 */
1062 keys_fit_inline = false;
1063 if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN +
1064 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1065 keys_fit_inline = true;
1066
1067 desc = ctx->sh_desc_dec;
1068
1069 init_sh_desc(desc, HDR_SHARE_SERIAL);
1070
1071 /* Skip key loading if it is loaded due to sharing */
1072 key_jump_cmd = append_jump(desc, JUMP_JSL |
1073 JUMP_TEST_ALL | JUMP_COND_SHRD);
1074 if (keys_fit_inline)
1075 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1076 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1077 else
1078 append_key(desc, ctx->key_dma, ctx->enckeylen,
1079 CLASS_1 | KEY_DEST_CLASS_REG);
1080 set_jump_tgt_here(desc, key_jump_cmd);
1081
1082 /* Class 1 operation */
1083 append_operation(desc, ctx->class1_alg_type |
1084 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1085
1086 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
1087 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1088
1089 /* Read assoc data */
1090 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1091 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1092
1093 /* Skip IV */
1094 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1095
1096 /* Will read cryptlen bytes */
1097 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1098
1099 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
1100 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1101
1102 /* Skip assoc data */
1103 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1104
1105 /* Will write cryptlen bytes */
1106 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1107
1108 /* Store payload data */
1109 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1110
1111 /* Read encrypted data */
1112 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1113 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1114
1115 /* Read ICV */
1116 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1117 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1118
1119 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1120 desc_bytes(desc),
1121 DMA_TO_DEVICE);
1122 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1123 dev_err(jrdev, "unable to map shared descriptor\n");
1124 return -ENOMEM;
1125 }
1126 #ifdef DEBUG
1127 print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ",
1128 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1129 desc_bytes(desc), 1);
1130 #endif
1131
1132 return 0;
1133 }
1134
1135 static int rfc4106_setauthsize(struct crypto_aead *authenc,
1136 unsigned int authsize)
1137 {
1138 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1139
1140 ctx->authsize = authsize;
1141 rfc4106_set_sh_desc(authenc);
1142
1143 return 0;
1144 }
1145
1146 static int rfc4543_set_sh_desc(struct crypto_aead *aead)
1147 {
1148 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1149 struct device *jrdev = ctx->jrdev;
1150 bool keys_fit_inline = false;
1151 u32 *key_jump_cmd;
1152 u32 *read_move_cmd, *write_move_cmd;
1153 u32 *desc;
1154
1155 if (!ctx->enckeylen || !ctx->authsize)
1156 return 0;
1157
1158 /*
1159 * RFC4543 encrypt shared descriptor
1160 * Job Descriptor and Shared Descriptor
1161 * must fit into the 64-word Descriptor h/w Buffer
1162 */
1163 if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN +
1164 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1165 keys_fit_inline = true;
1166
1167 desc = ctx->sh_desc_enc;
1168
1169 init_sh_desc(desc, HDR_SHARE_SERIAL);
1170
1171 /* Skip key loading if it is loaded due to sharing */
1172 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1173 JUMP_COND_SHRD);
1174 if (keys_fit_inline)
1175 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1176 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1177 else
1178 append_key(desc, ctx->key_dma, ctx->enckeylen,
1179 CLASS_1 | KEY_DEST_CLASS_REG);
1180 set_jump_tgt_here(desc, key_jump_cmd);
1181
1182 /* Class 1 operation */
1183 append_operation(desc, ctx->class1_alg_type |
1184 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1185
1186 /* assoclen + cryptlen = seqinlen */
1187 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1188
1189 /*
1190 * MOVE_LEN opcode is not available in all SEC HW revisions,
1191 * thus need to do some magic, i.e. self-patch the descriptor
1192 * buffer.
1193 */
1194 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1195 (0x6 << MOVE_LEN_SHIFT));
1196 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1197 (0x8 << MOVE_LEN_SHIFT));
1198
1199 /* Will read assoclen + cryptlen bytes */
1200 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1201
1202 /* Will write assoclen + cryptlen bytes */
1203 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1204
1205 /* Read and write assoclen + cryptlen bytes */
1206 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1207
1208 set_move_tgt_here(desc, read_move_cmd);
1209 set_move_tgt_here(desc, write_move_cmd);
1210 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1211 /* Move payload data to OFIFO */
1212 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1213
1214 /* Write ICV */
1215 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1216 LDST_SRCDST_BYTE_CONTEXT);
1217
1218 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1219 desc_bytes(desc),
1220 DMA_TO_DEVICE);
1221 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1222 dev_err(jrdev, "unable to map shared descriptor\n");
1223 return -ENOMEM;
1224 }
1225 #ifdef DEBUG
1226 print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ",
1227 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1228 desc_bytes(desc), 1);
1229 #endif
1230
1231 /*
1232 * Job Descriptor and Shared Descriptors
1233 * must all fit into the 64-word Descriptor h/w Buffer
1234 */
1235 keys_fit_inline = false;
1236 if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN +
1237 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1238 keys_fit_inline = true;
1239
1240 desc = ctx->sh_desc_dec;
1241
1242 init_sh_desc(desc, HDR_SHARE_SERIAL);
1243
1244 /* Skip key loading if it is loaded due to sharing */
1245 key_jump_cmd = append_jump(desc, JUMP_JSL |
1246 JUMP_TEST_ALL | JUMP_COND_SHRD);
1247 if (keys_fit_inline)
1248 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1249 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1250 else
1251 append_key(desc, ctx->key_dma, ctx->enckeylen,
1252 CLASS_1 | KEY_DEST_CLASS_REG);
1253 set_jump_tgt_here(desc, key_jump_cmd);
1254
1255 /* Class 1 operation */
1256 append_operation(desc, ctx->class1_alg_type |
1257 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1258
1259 /* assoclen + cryptlen = seqoutlen */
1260 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1261
1262 /*
1263 * MOVE_LEN opcode is not available in all SEC HW revisions,
1264 * thus need to do some magic, i.e. self-patch the descriptor
1265 * buffer.
1266 */
1267 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1268 (0x6 << MOVE_LEN_SHIFT));
1269 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1270 (0x8 << MOVE_LEN_SHIFT));
1271
1272 /* Will read assoclen + cryptlen bytes */
1273 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1274
1275 /* Will write assoclen + cryptlen bytes */
1276 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1277
1278 /* Store payload data */
1279 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1280
1281 /* In-snoop assoclen + cryptlen data */
1282 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1283 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1284
1285 set_move_tgt_here(desc, read_move_cmd);
1286 set_move_tgt_here(desc, write_move_cmd);
1287 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1288 /* Move payload data to OFIFO */
1289 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1290 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1291
1292 /* Read ICV */
1293 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1294 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1295
1296 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1297 desc_bytes(desc),
1298 DMA_TO_DEVICE);
1299 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1300 dev_err(jrdev, "unable to map shared descriptor\n");
1301 return -ENOMEM;
1302 }
1303 #ifdef DEBUG
1304 print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ",
1305 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1306 desc_bytes(desc), 1);
1307 #endif
1308
1309 return 0;
1310 }
1311
1312 static int rfc4543_setauthsize(struct crypto_aead *authenc,
1313 unsigned int authsize)
1314 {
1315 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1316
1317 ctx->authsize = authsize;
1318 rfc4543_set_sh_desc(authenc);
1319
1320 return 0;
1321 }
1322
1323 static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
1324 u32 authkeylen)
1325 {
1326 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
1327 ctx->split_key_pad_len, key_in, authkeylen,
1328 ctx->alg_op);
1329 }
1330
1331 static int aead_setkey(struct crypto_aead *aead,
1332 const u8 *key, unsigned int keylen)
1333 {
1334 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
1335 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
1336 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1337 struct device *jrdev = ctx->jrdev;
1338 struct crypto_authenc_keys keys;
1339 int ret = 0;
1340
1341 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1342 goto badkey;
1343
1344 /* Pick class 2 key length from algorithm submask */
1345 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
1346 OP_ALG_ALGSEL_SHIFT] * 2;
1347 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
1348
1349 if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE)
1350 goto badkey;
1351
1352 #ifdef DEBUG
1353 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
1354 keys.authkeylen + keys.enckeylen, keys.enckeylen,
1355 keys.authkeylen);
1356 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
1357 ctx->split_key_len, ctx->split_key_pad_len);
1358 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1359 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1360 #endif
1361
1362 ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
1363 if (ret) {
1364 goto badkey;
1365 }
1366
1367 /* postpend encryption key to auth split key */
1368 memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen);
1369
1370 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
1371 keys.enckeylen, DMA_TO_DEVICE);
1372 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1373 dev_err(jrdev, "unable to map key i/o memory\n");
1374 return -ENOMEM;
1375 }
1376 #ifdef DEBUG
1377 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
1378 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
1379 ctx->split_key_pad_len + keys.enckeylen, 1);
1380 #endif
1381
1382 ctx->enckeylen = keys.enckeylen;
1383
1384 ret = aead_set_sh_desc(aead);
1385 if (ret) {
1386 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
1387 keys.enckeylen, DMA_TO_DEVICE);
1388 }
1389
1390 return ret;
1391 badkey:
1392 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
1393 return -EINVAL;
1394 }
1395
1396 static int gcm_setkey(struct crypto_aead *aead,
1397 const u8 *key, unsigned int keylen)
1398 {
1399 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1400 struct device *jrdev = ctx->jrdev;
1401 int ret = 0;
1402
1403 #ifdef DEBUG
1404 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1405 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1406 #endif
1407
1408 memcpy(ctx->key, key, keylen);
1409 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1410 DMA_TO_DEVICE);
1411 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1412 dev_err(jrdev, "unable to map key i/o memory\n");
1413 return -ENOMEM;
1414 }
1415 ctx->enckeylen = keylen;
1416
1417 ret = gcm_set_sh_desc(aead);
1418 if (ret) {
1419 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1420 DMA_TO_DEVICE);
1421 }
1422
1423 return ret;
1424 }
1425
1426 static int rfc4106_setkey(struct crypto_aead *aead,
1427 const u8 *key, unsigned int keylen)
1428 {
1429 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1430 struct device *jrdev = ctx->jrdev;
1431 int ret = 0;
1432
1433 if (keylen < 4)
1434 return -EINVAL;
1435
1436 #ifdef DEBUG
1437 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1438 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1439 #endif
1440
1441 memcpy(ctx->key, key, keylen);
1442
1443 /*
1444 * The last four bytes of the key material are used as the salt value
1445 * in the nonce. Update the AES key length.
1446 */
1447 ctx->enckeylen = keylen - 4;
1448
1449 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1450 DMA_TO_DEVICE);
1451 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1452 dev_err(jrdev, "unable to map key i/o memory\n");
1453 return -ENOMEM;
1454 }
1455
1456 ret = rfc4106_set_sh_desc(aead);
1457 if (ret) {
1458 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1459 DMA_TO_DEVICE);
1460 }
1461
1462 return ret;
1463 }
1464
1465 static int rfc4543_setkey(struct crypto_aead *aead,
1466 const u8 *key, unsigned int keylen)
1467 {
1468 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1469 struct device *jrdev = ctx->jrdev;
1470 int ret = 0;
1471
1472 if (keylen < 4)
1473 return -EINVAL;
1474
1475 #ifdef DEBUG
1476 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1477 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1478 #endif
1479
1480 memcpy(ctx->key, key, keylen);
1481
1482 /*
1483 * The last four bytes of the key material are used as the salt value
1484 * in the nonce. Update the AES key length.
1485 */
1486 ctx->enckeylen = keylen - 4;
1487
1488 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1489 DMA_TO_DEVICE);
1490 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1491 dev_err(jrdev, "unable to map key i/o memory\n");
1492 return -ENOMEM;
1493 }
1494
1495 ret = rfc4543_set_sh_desc(aead);
1496 if (ret) {
1497 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1498 DMA_TO_DEVICE);
1499 }
1500
1501 return ret;
1502 }
1503
1504 static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
1505 const u8 *key, unsigned int keylen)
1506 {
1507 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1508 struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher;
1509 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
1510 const char *alg_name = crypto_tfm_alg_name(tfm);
1511 struct device *jrdev = ctx->jrdev;
1512 int ret = 0;
1513 u32 *key_jump_cmd;
1514 u32 *desc;
1515 u8 *nonce;
1516 u32 geniv;
1517 u32 ctx1_iv_off = 0;
1518 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
1519 OP_ALG_AAI_CTR_MOD128);
1520 const bool is_rfc3686 = (ctr_mode &&
1521 (strstr(alg_name, "rfc3686") != NULL));
1522
1523 #ifdef DEBUG
1524 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1525 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1526 #endif
1527 /*
1528 * AES-CTR needs to load IV in CONTEXT1 reg
1529 * at an offset of 128bits (16bytes)
1530 * CONTEXT1[255:128] = IV
1531 */
1532 if (ctr_mode)
1533 ctx1_iv_off = 16;
1534
1535 /*
1536 * RFC3686 specific:
1537 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1538 * | *key = {KEY, NONCE}
1539 */
1540 if (is_rfc3686) {
1541 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1542 keylen -= CTR_RFC3686_NONCE_SIZE;
1543 }
1544
1545 memcpy(ctx->key, key, keylen);
1546 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1547 DMA_TO_DEVICE);
1548 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1549 dev_err(jrdev, "unable to map key i/o memory\n");
1550 return -ENOMEM;
1551 }
1552 ctx->enckeylen = keylen;
1553
1554 /* ablkcipher_encrypt shared descriptor */
1555 desc = ctx->sh_desc_enc;
1556 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1557 /* Skip if already shared */
1558 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1559 JUMP_COND_SHRD);
1560
1561 /* Load class1 key only */
1562 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1563 ctx->enckeylen, CLASS_1 |
1564 KEY_DEST_CLASS_REG);
1565
1566 /* Load nonce into CONTEXT1 reg */
1567 if (is_rfc3686) {
1568 nonce = (u8 *)key + keylen;
1569 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1570 LDST_CLASS_IND_CCB |
1571 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1572 append_move(desc, MOVE_WAITCOMP |
1573 MOVE_SRC_OUTFIFO |
1574 MOVE_DEST_CLASS1CTX |
1575 (16 << MOVE_OFFSET_SHIFT) |
1576 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1577 }
1578
1579 set_jump_tgt_here(desc, key_jump_cmd);
1580
1581 /* Load iv */
1582 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
1583 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1584
1585 /* Load counter into CONTEXT1 reg */
1586 if (is_rfc3686)
1587 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1588 LDST_SRCDST_BYTE_CONTEXT |
1589 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1590 LDST_OFFSET_SHIFT));
1591
1592 /* Load operation */
1593 append_operation(desc, ctx->class1_alg_type |
1594 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1595
1596 /* Perform operation */
1597 ablkcipher_append_src_dst(desc);
1598
1599 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1600 desc_bytes(desc),
1601 DMA_TO_DEVICE);
1602 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1603 dev_err(jrdev, "unable to map shared descriptor\n");
1604 return -ENOMEM;
1605 }
1606 #ifdef DEBUG
1607 print_hex_dump(KERN_ERR,
1608 "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
1609 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1610 desc_bytes(desc), 1);
1611 #endif
1612 /* ablkcipher_decrypt shared descriptor */
1613 desc = ctx->sh_desc_dec;
1614
1615 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1616 /* Skip if already shared */
1617 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1618 JUMP_COND_SHRD);
1619
1620 /* Load class1 key only */
1621 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1622 ctx->enckeylen, CLASS_1 |
1623 KEY_DEST_CLASS_REG);
1624
1625 /* Load nonce into CONTEXT1 reg */
1626 if (is_rfc3686) {
1627 nonce = (u8 *)key + keylen;
1628 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1629 LDST_CLASS_IND_CCB |
1630 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1631 append_move(desc, MOVE_WAITCOMP |
1632 MOVE_SRC_OUTFIFO |
1633 MOVE_DEST_CLASS1CTX |
1634 (16 << MOVE_OFFSET_SHIFT) |
1635 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1636 }
1637
1638 set_jump_tgt_here(desc, key_jump_cmd);
1639
1640 /* load IV */
1641 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
1642 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1643
1644 /* Load counter into CONTEXT1 reg */
1645 if (is_rfc3686)
1646 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1647 LDST_SRCDST_BYTE_CONTEXT |
1648 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1649 LDST_OFFSET_SHIFT));
1650
1651 /* Choose operation */
1652 if (ctr_mode)
1653 append_operation(desc, ctx->class1_alg_type |
1654 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
1655 else
1656 append_dec_op1(desc, ctx->class1_alg_type);
1657
1658 /* Perform operation */
1659 ablkcipher_append_src_dst(desc);
1660
1661 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1662 desc_bytes(desc),
1663 DMA_TO_DEVICE);
1664 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1665 dev_err(jrdev, "unable to map shared descriptor\n");
1666 return -ENOMEM;
1667 }
1668
1669 #ifdef DEBUG
1670 print_hex_dump(KERN_ERR,
1671 "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
1672 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1673 desc_bytes(desc), 1);
1674 #endif
1675 /* ablkcipher_givencrypt shared descriptor */
1676 desc = ctx->sh_desc_givenc;
1677
1678 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1679 /* Skip if already shared */
1680 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1681 JUMP_COND_SHRD);
1682
1683 /* Load class1 key only */
1684 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1685 ctx->enckeylen, CLASS_1 |
1686 KEY_DEST_CLASS_REG);
1687
1688 /* Load Nonce into CONTEXT1 reg */
1689 if (is_rfc3686) {
1690 nonce = (u8 *)key + keylen;
1691 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1692 LDST_CLASS_IND_CCB |
1693 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1694 append_move(desc, MOVE_WAITCOMP |
1695 MOVE_SRC_OUTFIFO |
1696 MOVE_DEST_CLASS1CTX |
1697 (16 << MOVE_OFFSET_SHIFT) |
1698 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1699 }
1700 set_jump_tgt_here(desc, key_jump_cmd);
1701
1702 /* Generate IV */
1703 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1704 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
1705 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT);
1706 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1707 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1708 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1709 append_move(desc, MOVE_WAITCOMP |
1710 MOVE_SRC_INFIFO |
1711 MOVE_DEST_CLASS1CTX |
1712 (crt->ivsize << MOVE_LEN_SHIFT) |
1713 (ctx1_iv_off << MOVE_OFFSET_SHIFT));
1714 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1715
1716 /* Copy generated IV to memory */
1717 append_seq_store(desc, crt->ivsize,
1718 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1719 (ctx1_iv_off << LDST_OFFSET_SHIFT));
1720
1721 /* Load Counter into CONTEXT1 reg */
1722 if (is_rfc3686)
1723 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1724 LDST_SRCDST_BYTE_CONTEXT |
1725 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1726 LDST_OFFSET_SHIFT));
1727
1728 if (ctx1_iv_off)
1729 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
1730 (1 << JUMP_OFFSET_SHIFT));
1731
1732 /* Load operation */
1733 append_operation(desc, ctx->class1_alg_type |
1734 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1735
1736 /* Perform operation */
1737 ablkcipher_append_src_dst(desc);
1738
1739 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
1740 desc_bytes(desc),
1741 DMA_TO_DEVICE);
1742 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
1743 dev_err(jrdev, "unable to map shared descriptor\n");
1744 return -ENOMEM;
1745 }
1746 #ifdef DEBUG
1747 print_hex_dump(KERN_ERR,
1748 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
1749 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1750 desc_bytes(desc), 1);
1751 #endif
1752
1753 return ret;
1754 }
1755
1756 static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
1757 const u8 *key, unsigned int keylen)
1758 {
1759 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
1760 struct device *jrdev = ctx->jrdev;
1761 u32 *key_jump_cmd, *desc;
1762 __be64 sector_size = cpu_to_be64(512);
1763
1764 if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) {
1765 crypto_ablkcipher_set_flags(ablkcipher,
1766 CRYPTO_TFM_RES_BAD_KEY_LEN);
1767 dev_err(jrdev, "key size mismatch\n");
1768 return -EINVAL;
1769 }
1770
1771 memcpy(ctx->key, key, keylen);
1772 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE);
1773 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1774 dev_err(jrdev, "unable to map key i/o memory\n");
1775 return -ENOMEM;
1776 }
1777 ctx->enckeylen = keylen;
1778
1779 /* xts_ablkcipher_encrypt shared descriptor */
1780 desc = ctx->sh_desc_enc;
1781 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1782 /* Skip if already shared */
1783 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1784 JUMP_COND_SHRD);
1785
1786 /* Load class1 keys only */
1787 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1788 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1789
1790 /* Load sector size with index 40 bytes (0x28) */
1791 append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT |
1792 LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8);
1793 append_data(desc, (void *)&sector_size, 8);
1794
1795 set_jump_tgt_here(desc, key_jump_cmd);
1796
1797 /*
1798 * create sequence for loading the sector index
1799 * Upper 8B of IV - will be used as sector index
1800 * Lower 8B of IV - will be discarded
1801 */
1802 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
1803 LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8);
1804 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1805
1806 /* Load operation */
1807 append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL |
1808 OP_ALG_ENCRYPT);
1809
1810 /* Perform operation */
1811 ablkcipher_append_src_dst(desc);
1812
1813 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
1814 DMA_TO_DEVICE);
1815 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1816 dev_err(jrdev, "unable to map shared descriptor\n");
1817 return -ENOMEM;
1818 }
1819 #ifdef DEBUG
1820 print_hex_dump(KERN_ERR,
1821 "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
1822 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1823 #endif
1824
1825 /* xts_ablkcipher_decrypt shared descriptor */
1826 desc = ctx->sh_desc_dec;
1827
1828 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1829 /* Skip if already shared */
1830 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1831 JUMP_COND_SHRD);
1832
1833 /* Load class1 key only */
1834 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1835 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1836
1837 /* Load sector size with index 40 bytes (0x28) */
1838 append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT |
1839 LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8);
1840 append_data(desc, (void *)&sector_size, 8);
1841
1842 set_jump_tgt_here(desc, key_jump_cmd);
1843
1844 /*
1845 * create sequence for loading the sector index
1846 * Upper 8B of IV - will be used as sector index
1847 * Lower 8B of IV - will be discarded
1848 */
1849 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT |
1850 LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8);
1851 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1852
1853 /* Load operation */
1854 append_dec_op1(desc, ctx->class1_alg_type);
1855
1856 /* Perform operation */
1857 ablkcipher_append_src_dst(desc);
1858
1859 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc),
1860 DMA_TO_DEVICE);
1861 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1862 dma_unmap_single(jrdev, ctx->sh_desc_enc_dma,
1863 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
1864 dev_err(jrdev, "unable to map shared descriptor\n");
1865 return -ENOMEM;
1866 }
1867 #ifdef DEBUG
1868 print_hex_dump(KERN_ERR,
1869 "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
1870 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1871 #endif
1872
1873 return 0;
1874 }
1875
1876 /*
1877 * aead_edesc - s/w-extended aead descriptor
1878 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
1879 * @src_nents: number of segments in input scatterlist
1880 * @dst_nents: number of segments in output scatterlist
1881 * @iv_dma: dma address of iv for checking continuity and link table
1882 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
1883 * @sec4_sg_bytes: length of dma mapped sec4_sg space
1884 * @sec4_sg_dma: bus physical mapped address of h/w link table
1885 * @hw_desc: the h/w job descriptor followed by any referenced link tables
1886 */
1887 struct aead_edesc {
1888 int assoc_nents;
1889 int src_nents;
1890 int dst_nents;
1891 dma_addr_t iv_dma;
1892 int sec4_sg_bytes;
1893 dma_addr_t sec4_sg_dma;
1894 struct sec4_sg_entry *sec4_sg;
1895 u32 hw_desc[];
1896 };
1897
1898 /*
1899 * ablkcipher_edesc - s/w-extended ablkcipher descriptor
1900 * @src_nents: number of segments in input scatterlist
1901 * @dst_nents: number of segments in output scatterlist
1902 * @iv_dma: dma address of iv for checking continuity and link table
1903 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
1904 * @sec4_sg_bytes: length of dma mapped sec4_sg space
1905 * @sec4_sg_dma: bus physical mapped address of h/w link table
1906 * @hw_desc: the h/w job descriptor followed by any referenced link tables
1907 */
1908 struct ablkcipher_edesc {
1909 int src_nents;
1910 int dst_nents;
1911 dma_addr_t iv_dma;
1912 int sec4_sg_bytes;
1913 dma_addr_t sec4_sg_dma;
1914 struct sec4_sg_entry *sec4_sg;
1915 u32 hw_desc[0];
1916 };
1917
1918 static void caam_unmap(struct device *dev, struct scatterlist *src,
1919 struct scatterlist *dst, int src_nents,
1920 int dst_nents,
1921 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
1922 int sec4_sg_bytes)
1923 {
1924 if (dst != src) {
1925 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE);
1926 dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE);
1927 } else {
1928 dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL);
1929 }
1930
1931 if (iv_dma)
1932 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1933 if (sec4_sg_bytes)
1934 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
1935 DMA_TO_DEVICE);
1936 }
1937
1938 static void aead_unmap(struct device *dev,
1939 struct aead_edesc *edesc,
1940 struct aead_request *req)
1941 {
1942 caam_unmap(dev, req->src, req->dst,
1943 edesc->src_nents, edesc->dst_nents, 0, 0,
1944 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
1945 }
1946
1947 static void ablkcipher_unmap(struct device *dev,
1948 struct ablkcipher_edesc *edesc,
1949 struct ablkcipher_request *req)
1950 {
1951 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
1952 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
1953
1954 caam_unmap(dev, req->src, req->dst,
1955 edesc->src_nents, edesc->dst_nents,
1956 edesc->iv_dma, ivsize,
1957 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
1958 }
1959
1960 static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
1961 void *context)
1962 {
1963 struct aead_request *req = context;
1964 struct aead_edesc *edesc;
1965
1966 #ifdef DEBUG
1967 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1968 #endif
1969
1970 edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
1971
1972 if (err)
1973 caam_jr_strstatus(jrdev, err);
1974
1975 aead_unmap(jrdev, edesc, req);
1976
1977 kfree(edesc);
1978
1979 aead_request_complete(req, err);
1980 }
1981
1982 static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
1983 void *context)
1984 {
1985 struct aead_request *req = context;
1986 struct aead_edesc *edesc;
1987
1988 #ifdef DEBUG
1989 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
1990 #endif
1991
1992 edesc = container_of(desc, struct aead_edesc, hw_desc[0]);
1993
1994 if (err)
1995 caam_jr_strstatus(jrdev, err);
1996
1997 aead_unmap(jrdev, edesc, req);
1998
1999 /*
2000 * verify hw auth check passed else return -EBADMSG
2001 */
2002 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
2003 err = -EBADMSG;
2004
2005 kfree(edesc);
2006
2007 aead_request_complete(req, err);
2008 }
2009
2010 static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
2011 void *context)
2012 {
2013 struct ablkcipher_request *req = context;
2014 struct ablkcipher_edesc *edesc;
2015 #ifdef DEBUG
2016 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2017 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2018
2019 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2020 #endif
2021
2022 edesc = (struct ablkcipher_edesc *)((char *)desc -
2023 offsetof(struct ablkcipher_edesc, hw_desc));
2024
2025 if (err)
2026 caam_jr_strstatus(jrdev, err);
2027
2028 #ifdef DEBUG
2029 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
2030 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2031 edesc->src_nents > 1 ? 100 : ivsize, 1);
2032 dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ",
2033 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
2034 edesc->dst_nents > 1 ? 100 : req->nbytes, 1, true);
2035 #endif
2036
2037 ablkcipher_unmap(jrdev, edesc, req);
2038 kfree(edesc);
2039
2040 ablkcipher_request_complete(req, err);
2041 }
2042
2043 static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
2044 void *context)
2045 {
2046 struct ablkcipher_request *req = context;
2047 struct ablkcipher_edesc *edesc;
2048 #ifdef DEBUG
2049 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2050 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2051
2052 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2053 #endif
2054
2055 edesc = (struct ablkcipher_edesc *)((char *)desc -
2056 offsetof(struct ablkcipher_edesc, hw_desc));
2057 if (err)
2058 caam_jr_strstatus(jrdev, err);
2059
2060 #ifdef DEBUG
2061 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
2062 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2063 ivsize, 1);
2064 dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ",
2065 DUMP_PREFIX_ADDRESS, 16, 4, req->dst,
2066 edesc->dst_nents > 1 ? 100 : req->nbytes, 1, true);
2067 #endif
2068
2069 ablkcipher_unmap(jrdev, edesc, req);
2070 kfree(edesc);
2071
2072 ablkcipher_request_complete(req, err);
2073 }
2074
2075 /*
2076 * Fill in aead job descriptor
2077 */
2078 static void init_aead_job(struct aead_request *req,
2079 struct aead_edesc *edesc,
2080 bool all_contig, bool encrypt)
2081 {
2082 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2083 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2084 int authsize = ctx->authsize;
2085 u32 *desc = edesc->hw_desc;
2086 u32 out_options, in_options;
2087 dma_addr_t dst_dma, src_dma;
2088 int len, sec4_sg_index = 0;
2089 dma_addr_t ptr;
2090 u32 *sh_desc;
2091
2092 sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec;
2093 ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma;
2094
2095 len = desc_len(sh_desc);
2096 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2097
2098 if (all_contig) {
2099 src_dma = sg_dma_address(req->src);
2100 in_options = 0;
2101 } else {
2102 src_dma = edesc->sec4_sg_dma;
2103 sec4_sg_index += edesc->src_nents;
2104 in_options = LDST_SGF;
2105 }
2106
2107 append_seq_in_ptr(desc, src_dma, req->assoclen + req->cryptlen,
2108 in_options);
2109
2110 dst_dma = src_dma;
2111 out_options = in_options;
2112
2113 if (unlikely(req->src != req->dst)) {
2114 if (!edesc->dst_nents) {
2115 dst_dma = sg_dma_address(req->dst);
2116 } else {
2117 dst_dma = edesc->sec4_sg_dma +
2118 sec4_sg_index *
2119 sizeof(struct sec4_sg_entry);
2120 out_options = LDST_SGF;
2121 }
2122 }
2123
2124 if (encrypt)
2125 append_seq_out_ptr(desc, dst_dma,
2126 req->assoclen + req->cryptlen + authsize,
2127 out_options);
2128 else
2129 append_seq_out_ptr(desc, dst_dma,
2130 req->assoclen + req->cryptlen - authsize,
2131 out_options);
2132
2133 /* REG3 = assoclen */
2134 append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen);
2135 }
2136
2137 static void init_gcm_job(struct aead_request *req,
2138 struct aead_edesc *edesc,
2139 bool all_contig, bool encrypt)
2140 {
2141 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2142 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2143 unsigned int ivsize = crypto_aead_ivsize(aead);
2144 u32 *desc = edesc->hw_desc;
2145 bool generic_gcm = (ivsize == 12);
2146 unsigned int last;
2147
2148 init_aead_job(req, edesc, all_contig, encrypt);
2149
2150 /* BUG This should not be specific to generic GCM. */
2151 last = 0;
2152 if (encrypt && generic_gcm && !(req->assoclen + req->cryptlen))
2153 last = FIFOLD_TYPE_LAST1;
2154
2155 /* Read GCM IV */
2156 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
2157 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last);
2158 /* Append Salt */
2159 if (!generic_gcm)
2160 append_data(desc, ctx->key + ctx->enckeylen, 4);
2161 /* Append IV */
2162 append_data(desc, req->iv, ivsize);
2163 /* End of blank commands */
2164 }
2165
2166 static void init_authenc_job(struct aead_request *req,
2167 struct aead_edesc *edesc,
2168 bool all_contig, bool encrypt)
2169 {
2170 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2171 struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead),
2172 struct caam_aead_alg, aead);
2173 unsigned int ivsize = crypto_aead_ivsize(aead);
2174 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2175 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
2176 OP_ALG_AAI_CTR_MOD128);
2177 const bool is_rfc3686 = alg->caam.rfc3686;
2178 u32 *desc = edesc->hw_desc;
2179 u32 ivoffset = 0;
2180
2181 /*
2182 * AES-CTR needs to load IV in CONTEXT1 reg
2183 * at an offset of 128bits (16bytes)
2184 * CONTEXT1[255:128] = IV
2185 */
2186 if (ctr_mode)
2187 ivoffset = 16;
2188
2189 /*
2190 * RFC3686 specific:
2191 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
2192 */
2193 if (is_rfc3686)
2194 ivoffset = 16 + CTR_RFC3686_NONCE_SIZE;
2195
2196 init_aead_job(req, edesc, all_contig, encrypt);
2197
2198 if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv))
2199 append_load_as_imm(desc, req->iv, ivsize,
2200 LDST_CLASS_1_CCB |
2201 LDST_SRCDST_BYTE_CONTEXT |
2202 (ivoffset << LDST_OFFSET_SHIFT));
2203 }
2204
2205 /*
2206 * Fill in ablkcipher job descriptor
2207 */
2208 static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
2209 struct ablkcipher_edesc *edesc,
2210 struct ablkcipher_request *req,
2211 bool iv_contig)
2212 {
2213 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2214 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2215 u32 *desc = edesc->hw_desc;
2216 u32 out_options = 0, in_options;
2217 dma_addr_t dst_dma, src_dma;
2218 int len, sec4_sg_index = 0;
2219
2220 #ifdef DEBUG
2221 bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2222 CRYPTO_TFM_REQ_MAY_SLEEP)) != 0);
2223 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
2224 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2225 ivsize, 1);
2226 printk(KERN_ERR "asked=%d, nbytes%d\n", (int)edesc->src_nents ? 100 : req->nbytes, req->nbytes);
2227 dbg_dump_sg(KERN_ERR, "src @"__stringify(__LINE__)": ",
2228 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
2229 edesc->src_nents ? 100 : req->nbytes, 1, may_sleep);
2230 #endif
2231
2232 len = desc_len(sh_desc);
2233 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2234
2235 if (iv_contig) {
2236 src_dma = edesc->iv_dma;
2237 in_options = 0;
2238 } else {
2239 src_dma = edesc->sec4_sg_dma;
2240 sec4_sg_index += edesc->src_nents + 1;
2241 in_options = LDST_SGF;
2242 }
2243 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
2244
2245 if (likely(req->src == req->dst)) {
2246 if (!edesc->src_nents && iv_contig) {
2247 dst_dma = sg_dma_address(req->src);
2248 } else {
2249 dst_dma = edesc->sec4_sg_dma +
2250 sizeof(struct sec4_sg_entry);
2251 out_options = LDST_SGF;
2252 }
2253 } else {
2254 if (!edesc->dst_nents) {
2255 dst_dma = sg_dma_address(req->dst);
2256 } else {
2257 dst_dma = edesc->sec4_sg_dma +
2258 sec4_sg_index * sizeof(struct sec4_sg_entry);
2259 out_options = LDST_SGF;
2260 }
2261 }
2262 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
2263 }
2264
2265 /*
2266 * Fill in ablkcipher givencrypt job descriptor
2267 */
2268 static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
2269 struct ablkcipher_edesc *edesc,
2270 struct ablkcipher_request *req,
2271 bool iv_contig)
2272 {
2273 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2274 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2275 u32 *desc = edesc->hw_desc;
2276 u32 out_options, in_options;
2277 dma_addr_t dst_dma, src_dma;
2278 int len, sec4_sg_index = 0;
2279
2280 #ifdef DEBUG
2281 bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2282 CRYPTO_TFM_REQ_MAY_SLEEP)) != 0);
2283 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
2284 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2285 ivsize, 1);
2286 dbg_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ",
2287 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
2288 edesc->src_nents ? 100 : req->nbytes, 1, may_sleep);
2289 #endif
2290
2291 len = desc_len(sh_desc);
2292 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2293
2294 if (!edesc->src_nents) {
2295 src_dma = sg_dma_address(req->src);
2296 in_options = 0;
2297 } else {
2298 src_dma = edesc->sec4_sg_dma;
2299 sec4_sg_index += edesc->src_nents;
2300 in_options = LDST_SGF;
2301 }
2302 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
2303
2304 if (iv_contig) {
2305 dst_dma = edesc->iv_dma;
2306 out_options = 0;
2307 } else {
2308 dst_dma = edesc->sec4_sg_dma +
2309 sec4_sg_index * sizeof(struct sec4_sg_entry);
2310 out_options = LDST_SGF;
2311 }
2312 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options);
2313 }
2314
2315 /*
2316 * allocate and map the aead extended descriptor
2317 */
2318 static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
2319 int desc_bytes, bool *all_contig_ptr,
2320 bool encrypt)
2321 {
2322 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2323 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2324 struct device *jrdev = ctx->jrdev;
2325 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2326 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2327 int src_nents, dst_nents = 0;
2328 struct aead_edesc *edesc;
2329 int sgc;
2330 bool all_contig = true;
2331 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
2332 unsigned int authsize = ctx->authsize;
2333
2334 if (unlikely(req->dst != req->src)) {
2335 src_nents = sg_count(req->src, req->assoclen + req->cryptlen);
2336 dst_nents = sg_count(req->dst,
2337 req->assoclen + req->cryptlen +
2338 (encrypt ? authsize : (-authsize)));
2339 } else {
2340 src_nents = sg_count(req->src,
2341 req->assoclen + req->cryptlen +
2342 (encrypt ? authsize : 0));
2343 }
2344
2345 /* Check if data are contiguous. */
2346 all_contig = !src_nents;
2347 if (!all_contig) {
2348 src_nents = src_nents ? : 1;
2349 sec4_sg_len = src_nents;
2350 }
2351
2352 sec4_sg_len += dst_nents;
2353
2354 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
2355
2356 /* allocate space for base edesc and hw desc commands, link tables */
2357 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2358 GFP_DMA | flags);
2359 if (!edesc) {
2360 dev_err(jrdev, "could not allocate extended descriptor\n");
2361 return ERR_PTR(-ENOMEM);
2362 }
2363
2364 if (likely(req->src == req->dst)) {
2365 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2366 DMA_BIDIRECTIONAL);
2367 if (unlikely(!sgc)) {
2368 dev_err(jrdev, "unable to map source\n");
2369 kfree(edesc);
2370 return ERR_PTR(-ENOMEM);
2371 }
2372 } else {
2373 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2374 DMA_TO_DEVICE);
2375 if (unlikely(!sgc)) {
2376 dev_err(jrdev, "unable to map source\n");
2377 kfree(edesc);
2378 return ERR_PTR(-ENOMEM);
2379 }
2380
2381 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2382 DMA_FROM_DEVICE);
2383 if (unlikely(!sgc)) {
2384 dev_err(jrdev, "unable to map destination\n");
2385 dma_unmap_sg(jrdev, req->src, src_nents ? : 1,
2386 DMA_TO_DEVICE);
2387 kfree(edesc);
2388 return ERR_PTR(-ENOMEM);
2389 }
2390 }
2391
2392 edesc->src_nents = src_nents;
2393 edesc->dst_nents = dst_nents;
2394 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
2395 desc_bytes;
2396 *all_contig_ptr = all_contig;
2397
2398 sec4_sg_index = 0;
2399 if (!all_contig) {
2400 sg_to_sec4_sg_last(req->src, src_nents,
2401 edesc->sec4_sg + sec4_sg_index, 0);
2402 sec4_sg_index += src_nents;
2403 }
2404 if (dst_nents) {
2405 sg_to_sec4_sg_last(req->dst, dst_nents,
2406 edesc->sec4_sg + sec4_sg_index, 0);
2407 }
2408
2409 if (!sec4_sg_bytes)
2410 return edesc;
2411
2412 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2413 sec4_sg_bytes, DMA_TO_DEVICE);
2414 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2415 dev_err(jrdev, "unable to map S/G table\n");
2416 aead_unmap(jrdev, edesc, req);
2417 kfree(edesc);
2418 return ERR_PTR(-ENOMEM);
2419 }
2420
2421 edesc->sec4_sg_bytes = sec4_sg_bytes;
2422
2423 return edesc;
2424 }
2425
2426 static int gcm_encrypt(struct aead_request *req)
2427 {
2428 struct aead_edesc *edesc;
2429 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2430 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2431 struct device *jrdev = ctx->jrdev;
2432 bool all_contig;
2433 u32 *desc;
2434 int ret = 0;
2435
2436 /* allocate extended descriptor */
2437 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, true);
2438 if (IS_ERR(edesc))
2439 return PTR_ERR(edesc);
2440
2441 /* Create and submit job descriptor */
2442 init_gcm_job(req, edesc, all_contig, true);
2443 #ifdef DEBUG
2444 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2445 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2446 desc_bytes(edesc->hw_desc), 1);
2447 #endif
2448
2449 desc = edesc->hw_desc;
2450 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2451 if (!ret) {
2452 ret = -EINPROGRESS;
2453 } else {
2454 aead_unmap(jrdev, edesc, req);
2455 kfree(edesc);
2456 }
2457
2458 return ret;
2459 }
2460
2461 static int ipsec_gcm_encrypt(struct aead_request *req)
2462 {
2463 if (req->assoclen < 8)
2464 return -EINVAL;
2465
2466 return gcm_encrypt(req);
2467 }
2468
2469 static int aead_encrypt(struct aead_request *req)
2470 {
2471 struct aead_edesc *edesc;
2472 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2473 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2474 struct device *jrdev = ctx->jrdev;
2475 bool all_contig;
2476 u32 *desc;
2477 int ret = 0;
2478
2479 /* allocate extended descriptor */
2480 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
2481 &all_contig, true);
2482 if (IS_ERR(edesc))
2483 return PTR_ERR(edesc);
2484
2485 /* Create and submit job descriptor */
2486 init_authenc_job(req, edesc, all_contig, true);
2487 #ifdef DEBUG
2488 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2489 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2490 desc_bytes(edesc->hw_desc), 1);
2491 #endif
2492
2493 desc = edesc->hw_desc;
2494 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2495 if (!ret) {
2496 ret = -EINPROGRESS;
2497 } else {
2498 aead_unmap(jrdev, edesc, req);
2499 kfree(edesc);
2500 }
2501
2502 return ret;
2503 }
2504
2505 static int gcm_decrypt(struct aead_request *req)
2506 {
2507 struct aead_edesc *edesc;
2508 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2509 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2510 struct device *jrdev = ctx->jrdev;
2511 bool all_contig;
2512 u32 *desc;
2513 int ret = 0;
2514
2515 /* allocate extended descriptor */
2516 edesc = aead_edesc_alloc(req, GCM_DESC_JOB_IO_LEN, &all_contig, false);
2517 if (IS_ERR(edesc))
2518 return PTR_ERR(edesc);
2519
2520 /* Create and submit job descriptor*/
2521 init_gcm_job(req, edesc, all_contig, false);
2522 #ifdef DEBUG
2523 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2524 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2525 desc_bytes(edesc->hw_desc), 1);
2526 #endif
2527
2528 desc = edesc->hw_desc;
2529 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2530 if (!ret) {
2531 ret = -EINPROGRESS;
2532 } else {
2533 aead_unmap(jrdev, edesc, req);
2534 kfree(edesc);
2535 }
2536
2537 return ret;
2538 }
2539
2540 static int ipsec_gcm_decrypt(struct aead_request *req)
2541 {
2542 if (req->assoclen < 8)
2543 return -EINVAL;
2544
2545 return gcm_decrypt(req);
2546 }
2547
2548 static int aead_decrypt(struct aead_request *req)
2549 {
2550 struct aead_edesc *edesc;
2551 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2552 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2553 struct device *jrdev = ctx->jrdev;
2554 bool all_contig;
2555 u32 *desc;
2556 int ret = 0;
2557
2558 #ifdef DEBUG
2559 bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2560 CRYPTO_TFM_REQ_MAY_SLEEP)) != 0);
2561 dbg_dump_sg(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
2562 DUMP_PREFIX_ADDRESS, 16, 4, req->src,
2563 req->assoclen + req->cryptlen, 1, may_sleep);
2564 #endif
2565
2566 /* allocate extended descriptor */
2567 edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN,
2568 &all_contig, false);
2569 if (IS_ERR(edesc))
2570 return PTR_ERR(edesc);
2571
2572 /* Create and submit job descriptor*/
2573 init_authenc_job(req, edesc, all_contig, false);
2574 #ifdef DEBUG
2575 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
2576 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2577 desc_bytes(edesc->hw_desc), 1);
2578 #endif
2579
2580 desc = edesc->hw_desc;
2581 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2582 if (!ret) {
2583 ret = -EINPROGRESS;
2584 } else {
2585 aead_unmap(jrdev, edesc, req);
2586 kfree(edesc);
2587 }
2588
2589 return ret;
2590 }
2591
2592 /*
2593 * allocate and map the ablkcipher extended descriptor for ablkcipher
2594 */
2595 static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
2596 *req, int desc_bytes,
2597 bool *iv_contig_out)
2598 {
2599 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2600 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2601 struct device *jrdev = ctx->jrdev;
2602 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2603 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
2604 GFP_KERNEL : GFP_ATOMIC;
2605 int src_nents, dst_nents = 0, sec4_sg_bytes;
2606 struct ablkcipher_edesc *edesc;
2607 dma_addr_t iv_dma = 0;
2608 bool iv_contig = false;
2609 int sgc;
2610 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2611 int sec4_sg_index;
2612
2613 src_nents = sg_count(req->src, req->nbytes);
2614
2615 if (req->dst != req->src)
2616 dst_nents = sg_count(req->dst, req->nbytes);
2617
2618 if (likely(req->src == req->dst)) {
2619 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2620 DMA_BIDIRECTIONAL);
2621 } else {
2622 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2623 DMA_TO_DEVICE);
2624 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2625 DMA_FROM_DEVICE);
2626 }
2627
2628 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
2629 if (dma_mapping_error(jrdev, iv_dma)) {
2630 dev_err(jrdev, "unable to map IV\n");
2631 return ERR_PTR(-ENOMEM);
2632 }
2633
2634 /*
2635 * Check if iv can be contiguous with source and destination.
2636 * If so, include it. If not, create scatterlist.
2637 */
2638 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
2639 iv_contig = true;
2640 else
2641 src_nents = src_nents ? : 1;
2642 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
2643 sizeof(struct sec4_sg_entry);
2644
2645 /* allocate space for base edesc and hw desc commands, link tables */
2646 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2647 GFP_DMA | flags);
2648 if (!edesc) {
2649 dev_err(jrdev, "could not allocate extended descriptor\n");
2650 return ERR_PTR(-ENOMEM);
2651 }
2652
2653 edesc->src_nents = src_nents;
2654 edesc->dst_nents = dst_nents;
2655 edesc->sec4_sg_bytes = sec4_sg_bytes;
2656 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
2657 desc_bytes;
2658
2659 sec4_sg_index = 0;
2660 if (!iv_contig) {
2661 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
2662 sg_to_sec4_sg_last(req->src, src_nents,
2663 edesc->sec4_sg + 1, 0);
2664 sec4_sg_index += 1 + src_nents;
2665 }
2666
2667 if (dst_nents) {
2668 sg_to_sec4_sg_last(req->dst, dst_nents,
2669 edesc->sec4_sg + sec4_sg_index, 0);
2670 }
2671
2672 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2673 sec4_sg_bytes, DMA_TO_DEVICE);
2674 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2675 dev_err(jrdev, "unable to map S/G table\n");
2676 return ERR_PTR(-ENOMEM);
2677 }
2678
2679 edesc->iv_dma = iv_dma;
2680
2681 #ifdef DEBUG
2682 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
2683 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
2684 sec4_sg_bytes, 1);
2685 #endif
2686
2687 *iv_contig_out = iv_contig;
2688 return edesc;
2689 }
2690
2691 static int ablkcipher_encrypt(struct ablkcipher_request *req)
2692 {
2693 struct ablkcipher_edesc *edesc;
2694 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2695 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2696 struct device *jrdev = ctx->jrdev;
2697 bool iv_contig;
2698 u32 *desc;
2699 int ret = 0;
2700
2701 /* allocate extended descriptor */
2702 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
2703 CAAM_CMD_SZ, &iv_contig);
2704 if (IS_ERR(edesc))
2705 return PTR_ERR(edesc);
2706
2707 /* Create and submit job descriptor*/
2708 init_ablkcipher_job(ctx->sh_desc_enc,
2709 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
2710 #ifdef DEBUG
2711 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
2712 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2713 desc_bytes(edesc->hw_desc), 1);
2714 #endif
2715 desc = edesc->hw_desc;
2716 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
2717
2718 if (!ret) {
2719 ret = -EINPROGRESS;
2720 } else {
2721 ablkcipher_unmap(jrdev, edesc, req);
2722 kfree(edesc);
2723 }
2724
2725 return ret;
2726 }
2727
2728 static int ablkcipher_decrypt(struct ablkcipher_request *req)
2729 {
2730 struct ablkcipher_edesc *edesc;
2731 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2732 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2733 struct device *jrdev = ctx->jrdev;
2734 bool iv_contig;
2735 u32 *desc;
2736 int ret = 0;
2737
2738 /* allocate extended descriptor */
2739 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
2740 CAAM_CMD_SZ, &iv_contig);
2741 if (IS_ERR(edesc))
2742 return PTR_ERR(edesc);
2743
2744 /* Create and submit job descriptor*/
2745 init_ablkcipher_job(ctx->sh_desc_dec,
2746 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
2747 desc = edesc->hw_desc;
2748 #ifdef DEBUG
2749 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
2750 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2751 desc_bytes(edesc->hw_desc), 1);
2752 #endif
2753
2754 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
2755 if (!ret) {
2756 ret = -EINPROGRESS;
2757 } else {
2758 ablkcipher_unmap(jrdev, edesc, req);
2759 kfree(edesc);
2760 }
2761
2762 return ret;
2763 }
2764
2765 /*
2766 * allocate and map the ablkcipher extended descriptor
2767 * for ablkcipher givencrypt
2768 */
2769 static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
2770 struct skcipher_givcrypt_request *greq,
2771 int desc_bytes,
2772 bool *iv_contig_out)
2773 {
2774 struct ablkcipher_request *req = &greq->creq;
2775 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2776 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2777 struct device *jrdev = ctx->jrdev;
2778 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2779 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
2780 GFP_KERNEL : GFP_ATOMIC;
2781 int src_nents, dst_nents = 0, sec4_sg_bytes;
2782 struct ablkcipher_edesc *edesc;
2783 dma_addr_t iv_dma = 0;
2784 bool iv_contig = false;
2785 int sgc;
2786 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2787 int sec4_sg_index;
2788
2789 src_nents = sg_count(req->src, req->nbytes);
2790
2791 if (unlikely(req->dst != req->src))
2792 dst_nents = sg_count(req->dst, req->nbytes);
2793
2794 if (likely(req->src == req->dst)) {
2795 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2796 DMA_BIDIRECTIONAL);
2797 } else {
2798 sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1,
2799 DMA_TO_DEVICE);
2800 sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1,
2801 DMA_FROM_DEVICE);
2802 }
2803
2804 /*
2805 * Check if iv can be contiguous with source and destination.
2806 * If so, include it. If not, create scatterlist.
2807 */
2808 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
2809 if (dma_mapping_error(jrdev, iv_dma)) {
2810 dev_err(jrdev, "unable to map IV\n");
2811 return ERR_PTR(-ENOMEM);
2812 }
2813
2814 if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst))
2815 iv_contig = true;
2816 else
2817 dst_nents = dst_nents ? : 1;
2818 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
2819 sizeof(struct sec4_sg_entry);
2820
2821 /* allocate space for base edesc and hw desc commands, link tables */
2822 edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes,
2823 GFP_DMA | flags);
2824 if (!edesc) {
2825 dev_err(jrdev, "could not allocate extended descriptor\n");
2826 return ERR_PTR(-ENOMEM);
2827 }
2828
2829 edesc->src_nents = src_nents;
2830 edesc->dst_nents = dst_nents;
2831 edesc->sec4_sg_bytes = sec4_sg_bytes;
2832 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
2833 desc_bytes;
2834
2835 sec4_sg_index = 0;
2836 if (src_nents) {
2837 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
2838 sec4_sg_index += src_nents;
2839 }
2840
2841 if (!iv_contig) {
2842 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
2843 iv_dma, ivsize, 0);
2844 sec4_sg_index += 1;
2845 sg_to_sec4_sg_last(req->dst, dst_nents,
2846 edesc->sec4_sg + sec4_sg_index, 0);
2847 }
2848
2849 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2850 sec4_sg_bytes, DMA_TO_DEVICE);
2851 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2852 dev_err(jrdev, "unable to map S/G table\n");
2853 return ERR_PTR(-ENOMEM);
2854 }
2855 edesc->iv_dma = iv_dma;
2856
2857 #ifdef DEBUG
2858 print_hex_dump(KERN_ERR,
2859 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ",
2860 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
2861 sec4_sg_bytes, 1);
2862 #endif
2863
2864 *iv_contig_out = iv_contig;
2865 return edesc;
2866 }
2867
2868 static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
2869 {
2870 struct ablkcipher_request *req = &creq->creq;
2871 struct ablkcipher_edesc *edesc;
2872 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2873 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
2874 struct device *jrdev = ctx->jrdev;
2875 bool iv_contig;
2876 u32 *desc;
2877 int ret = 0;
2878
2879 /* allocate extended descriptor */
2880 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *
2881 CAAM_CMD_SZ, &iv_contig);
2882 if (IS_ERR(edesc))
2883 return PTR_ERR(edesc);
2884
2885 /* Create and submit job descriptor*/
2886 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
2887 edesc, req, iv_contig);
2888 #ifdef DEBUG
2889 print_hex_dump(KERN_ERR,
2890 "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
2891 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2892 desc_bytes(edesc->hw_desc), 1);
2893 #endif
2894 desc = edesc->hw_desc;
2895 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
2896
2897 if (!ret) {
2898 ret = -EINPROGRESS;
2899 } else {
2900 ablkcipher_unmap(jrdev, edesc, req);
2901 kfree(edesc);
2902 }
2903
2904 return ret;
2905 }
2906
2907 #define template_aead template_u.aead
2908 #define template_ablkcipher template_u.ablkcipher
2909 struct caam_alg_template {
2910 char name[CRYPTO_MAX_ALG_NAME];
2911 char driver_name[CRYPTO_MAX_ALG_NAME];
2912 unsigned int blocksize;
2913 u32 type;
2914 union {
2915 struct ablkcipher_alg ablkcipher;
2916 } template_u;
2917 u32 class1_alg_type;
2918 u32 class2_alg_type;
2919 u32 alg_op;
2920 };
2921
2922 static struct caam_alg_template driver_algs[] = {
2923 /* ablkcipher descriptor */
2924 {
2925 .name = "cbc(aes)",
2926 .driver_name = "cbc-aes-caam",
2927 .blocksize = AES_BLOCK_SIZE,
2928 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2929 .template_ablkcipher = {
2930 .setkey = ablkcipher_setkey,
2931 .encrypt = ablkcipher_encrypt,
2932 .decrypt = ablkcipher_decrypt,
2933 .givencrypt = ablkcipher_givencrypt,
2934 .geniv = "<built-in>",
2935 .min_keysize = AES_MIN_KEY_SIZE,
2936 .max_keysize = AES_MAX_KEY_SIZE,
2937 .ivsize = AES_BLOCK_SIZE,
2938 },
2939 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
2940 },
2941 {
2942 .name = "cbc(des3_ede)",
2943 .driver_name = "cbc-3des-caam",
2944 .blocksize = DES3_EDE_BLOCK_SIZE,
2945 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2946 .template_ablkcipher = {
2947 .setkey = ablkcipher_setkey,
2948 .encrypt = ablkcipher_encrypt,
2949 .decrypt = ablkcipher_decrypt,
2950 .givencrypt = ablkcipher_givencrypt,
2951 .geniv = "<built-in>",
2952 .min_keysize = DES3_EDE_KEY_SIZE,
2953 .max_keysize = DES3_EDE_KEY_SIZE,
2954 .ivsize = DES3_EDE_BLOCK_SIZE,
2955 },
2956 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
2957 },
2958 {
2959 .name = "cbc(des)",
2960 .driver_name = "cbc-des-caam",
2961 .blocksize = DES_BLOCK_SIZE,
2962 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2963 .template_ablkcipher = {
2964 .setkey = ablkcipher_setkey,
2965 .encrypt = ablkcipher_encrypt,
2966 .decrypt = ablkcipher_decrypt,
2967 .givencrypt = ablkcipher_givencrypt,
2968 .geniv = "<built-in>",
2969 .min_keysize = DES_KEY_SIZE,
2970 .max_keysize = DES_KEY_SIZE,
2971 .ivsize = DES_BLOCK_SIZE,
2972 },
2973 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2974 },
2975 {
2976 .name = "ctr(aes)",
2977 .driver_name = "ctr-aes-caam",
2978 .blocksize = 1,
2979 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2980 .template_ablkcipher = {
2981 .setkey = ablkcipher_setkey,
2982 .encrypt = ablkcipher_encrypt,
2983 .decrypt = ablkcipher_decrypt,
2984 .geniv = "chainiv",
2985 .min_keysize = AES_MIN_KEY_SIZE,
2986 .max_keysize = AES_MAX_KEY_SIZE,
2987 .ivsize = AES_BLOCK_SIZE,
2988 },
2989 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
2990 },
2991 {
2992 .name = "rfc3686(ctr(aes))",
2993 .driver_name = "rfc3686-ctr-aes-caam",
2994 .blocksize = 1,
2995 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
2996 .template_ablkcipher = {
2997 .setkey = ablkcipher_setkey,
2998 .encrypt = ablkcipher_encrypt,
2999 .decrypt = ablkcipher_decrypt,
3000 .givencrypt = ablkcipher_givencrypt,
3001 .geniv = "<built-in>",
3002 .min_keysize = AES_MIN_KEY_SIZE +
3003 CTR_RFC3686_NONCE_SIZE,
3004 .max_keysize = AES_MAX_KEY_SIZE +
3005 CTR_RFC3686_NONCE_SIZE,
3006 .ivsize = CTR_RFC3686_IV_SIZE,
3007 },
3008 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3009 },
3010 {
3011 .name = "xts(aes)",
3012 .driver_name = "xts-aes-caam",
3013 .blocksize = AES_BLOCK_SIZE,
3014 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
3015 .template_ablkcipher = {
3016 .setkey = xts_ablkcipher_setkey,
3017 .encrypt = ablkcipher_encrypt,
3018 .decrypt = ablkcipher_decrypt,
3019 .geniv = "eseqiv",
3020 .min_keysize = 2 * AES_MIN_KEY_SIZE,
3021 .max_keysize = 2 * AES_MAX_KEY_SIZE,
3022 .ivsize = AES_BLOCK_SIZE,
3023 },
3024 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS,
3025 },
3026 };
3027
3028 static struct caam_aead_alg driver_aeads[] = {
3029 {
3030 .aead = {
3031 .base = {
3032 .cra_name = "rfc4106(gcm(aes))",
3033 .cra_driver_name = "rfc4106-gcm-aes-caam",
3034 .cra_blocksize = 1,
3035 },
3036 .setkey = rfc4106_setkey,
3037 .setauthsize = rfc4106_setauthsize,
3038 .encrypt = ipsec_gcm_encrypt,
3039 .decrypt = ipsec_gcm_decrypt,
3040 .ivsize = 8,
3041 .maxauthsize = AES_BLOCK_SIZE,
3042 },
3043 .caam = {
3044 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
3045 },
3046 },
3047 {
3048 .aead = {
3049 .base = {
3050 .cra_name = "rfc4543(gcm(aes))",
3051 .cra_driver_name = "rfc4543-gcm-aes-caam",
3052 .cra_blocksize = 1,
3053 },
3054 .setkey = rfc4543_setkey,
3055 .setauthsize = rfc4543_setauthsize,
3056 .encrypt = ipsec_gcm_encrypt,
3057 .decrypt = ipsec_gcm_decrypt,
3058 .ivsize = 8,
3059 .maxauthsize = AES_BLOCK_SIZE,
3060 },
3061 .caam = {
3062 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
3063 },
3064 },
3065 /* Galois Counter Mode */
3066 {
3067 .aead = {
3068 .base = {
3069 .cra_name = "gcm(aes)",
3070 .cra_driver_name = "gcm-aes-caam",
3071 .cra_blocksize = 1,
3072 },
3073 .setkey = gcm_setkey,
3074 .setauthsize = gcm_setauthsize,
3075 .encrypt = gcm_encrypt,
3076 .decrypt = gcm_decrypt,
3077 .ivsize = 12,
3078 .maxauthsize = AES_BLOCK_SIZE,
3079 },
3080 .caam = {
3081 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
3082 },
3083 },
3084 /* single-pass ipsec_esp descriptor */
3085 {
3086 .aead = {
3087 .base = {
3088 .cra_name = "authenc(hmac(md5),"
3089 "ecb(cipher_null))",
3090 .cra_driver_name = "authenc-hmac-md5-"
3091 "ecb-cipher_null-caam",
3092 .cra_blocksize = NULL_BLOCK_SIZE,
3093 },
3094 .setkey = aead_setkey,
3095 .setauthsize = aead_setauthsize,
3096 .encrypt = aead_encrypt,
3097 .decrypt = aead_decrypt,
3098 .ivsize = NULL_IV_SIZE,
3099 .maxauthsize = MD5_DIGEST_SIZE,
3100 },
3101 .caam = {
3102 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3103 OP_ALG_AAI_HMAC_PRECOMP,
3104 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3105 },
3106 },
3107 {
3108 .aead = {
3109 .base = {
3110 .cra_name = "authenc(hmac(sha1),"
3111 "ecb(cipher_null))",
3112 .cra_driver_name = "authenc-hmac-sha1-"
3113 "ecb-cipher_null-caam",
3114 .cra_blocksize = NULL_BLOCK_SIZE,
3115 },
3116 .setkey = aead_setkey,
3117 .setauthsize = aead_setauthsize,
3118 .encrypt = aead_encrypt,
3119 .decrypt = aead_decrypt,
3120 .ivsize = NULL_IV_SIZE,
3121 .maxauthsize = SHA1_DIGEST_SIZE,
3122 },
3123 .caam = {
3124 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3125 OP_ALG_AAI_HMAC_PRECOMP,
3126 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3127 },
3128 },
3129 {
3130 .aead = {
3131 .base = {
3132 .cra_name = "authenc(hmac(sha224),"
3133 "ecb(cipher_null))",
3134 .cra_driver_name = "authenc-hmac-sha224-"
3135 "ecb-cipher_null-caam",
3136 .cra_blocksize = NULL_BLOCK_SIZE,
3137 },
3138 .setkey = aead_setkey,
3139 .setauthsize = aead_setauthsize,
3140 .encrypt = aead_encrypt,
3141 .decrypt = aead_decrypt,
3142 .ivsize = NULL_IV_SIZE,
3143 .maxauthsize = SHA224_DIGEST_SIZE,
3144 },
3145 .caam = {
3146 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3147 OP_ALG_AAI_HMAC_PRECOMP,
3148 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3149 },
3150 },
3151 {
3152 .aead = {
3153 .base = {
3154 .cra_name = "authenc(hmac(sha256),"
3155 "ecb(cipher_null))",
3156 .cra_driver_name = "authenc-hmac-sha256-"
3157 "ecb-cipher_null-caam",
3158 .cra_blocksize = NULL_BLOCK_SIZE,
3159 },
3160 .setkey = aead_setkey,
3161 .setauthsize = aead_setauthsize,
3162 .encrypt = aead_encrypt,
3163 .decrypt = aead_decrypt,
3164 .ivsize = NULL_IV_SIZE,
3165 .maxauthsize = SHA256_DIGEST_SIZE,
3166 },
3167 .caam = {
3168 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3169 OP_ALG_AAI_HMAC_PRECOMP,
3170 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3171 },
3172 },
3173 {
3174 .aead = {
3175 .base = {
3176 .cra_name = "authenc(hmac(sha384),"
3177 "ecb(cipher_null))",
3178 .cra_driver_name = "authenc-hmac-sha384-"
3179 "ecb-cipher_null-caam",
3180 .cra_blocksize = NULL_BLOCK_SIZE,
3181 },
3182 .setkey = aead_setkey,
3183 .setauthsize = aead_setauthsize,
3184 .encrypt = aead_encrypt,
3185 .decrypt = aead_decrypt,
3186 .ivsize = NULL_IV_SIZE,
3187 .maxauthsize = SHA384_DIGEST_SIZE,
3188 },
3189 .caam = {
3190 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3191 OP_ALG_AAI_HMAC_PRECOMP,
3192 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3193 },
3194 },
3195 {
3196 .aead = {
3197 .base = {
3198 .cra_name = "authenc(hmac(sha512),"
3199 "ecb(cipher_null))",
3200 .cra_driver_name = "authenc-hmac-sha512-"
3201 "ecb-cipher_null-caam",
3202 .cra_blocksize = NULL_BLOCK_SIZE,
3203 },
3204 .setkey = aead_setkey,
3205 .setauthsize = aead_setauthsize,
3206 .encrypt = aead_encrypt,
3207 .decrypt = aead_decrypt,
3208 .ivsize = NULL_IV_SIZE,
3209 .maxauthsize = SHA512_DIGEST_SIZE,
3210 },
3211 .caam = {
3212 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3213 OP_ALG_AAI_HMAC_PRECOMP,
3214 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3215 },
3216 },
3217 {
3218 .aead = {
3219 .base = {
3220 .cra_name = "authenc(hmac(md5),cbc(aes))",
3221 .cra_driver_name = "authenc-hmac-md5-"
3222 "cbc-aes-caam",
3223 .cra_blocksize = AES_BLOCK_SIZE,
3224 },
3225 .setkey = aead_setkey,
3226 .setauthsize = aead_setauthsize,
3227 .encrypt = aead_encrypt,
3228 .decrypt = aead_decrypt,
3229 .ivsize = AES_BLOCK_SIZE,
3230 .maxauthsize = MD5_DIGEST_SIZE,
3231 },
3232 .caam = {
3233 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3234 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3235 OP_ALG_AAI_HMAC_PRECOMP,
3236 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3237 },
3238 },
3239 {
3240 .aead = {
3241 .base = {
3242 .cra_name = "echainiv(authenc(hmac(md5),"
3243 "cbc(aes)))",
3244 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3245 "cbc-aes-caam",
3246 .cra_blocksize = AES_BLOCK_SIZE,
3247 },
3248 .setkey = aead_setkey,
3249 .setauthsize = aead_setauthsize,
3250 .encrypt = aead_encrypt,
3251 .decrypt = aead_decrypt,
3252 .ivsize = AES_BLOCK_SIZE,
3253 .maxauthsize = MD5_DIGEST_SIZE,
3254 },
3255 .caam = {
3256 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3257 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3258 OP_ALG_AAI_HMAC_PRECOMP,
3259 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3260 .geniv = true,
3261 },
3262 },
3263 {
3264 .aead = {
3265 .base = {
3266 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3267 .cra_driver_name = "authenc-hmac-sha1-"
3268 "cbc-aes-caam",
3269 .cra_blocksize = AES_BLOCK_SIZE,
3270 },
3271 .setkey = aead_setkey,
3272 .setauthsize = aead_setauthsize,
3273 .encrypt = aead_encrypt,
3274 .decrypt = aead_decrypt,
3275 .ivsize = AES_BLOCK_SIZE,
3276 .maxauthsize = SHA1_DIGEST_SIZE,
3277 },
3278 .caam = {
3279 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3280 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3281 OP_ALG_AAI_HMAC_PRECOMP,
3282 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3283 },
3284 },
3285 {
3286 .aead = {
3287 .base = {
3288 .cra_name = "echainiv(authenc(hmac(sha1),"
3289 "cbc(aes)))",
3290 .cra_driver_name = "echainiv-authenc-"
3291 "hmac-sha1-cbc-aes-caam",
3292 .cra_blocksize = AES_BLOCK_SIZE,
3293 },
3294 .setkey = aead_setkey,
3295 .setauthsize = aead_setauthsize,
3296 .encrypt = aead_encrypt,
3297 .decrypt = aead_decrypt,
3298 .ivsize = AES_BLOCK_SIZE,
3299 .maxauthsize = SHA1_DIGEST_SIZE,
3300 },
3301 .caam = {
3302 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3303 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3304 OP_ALG_AAI_HMAC_PRECOMP,
3305 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3306 .geniv = true,
3307 },
3308 },
3309 {
3310 .aead = {
3311 .base = {
3312 .cra_name = "authenc(hmac(sha224),cbc(aes))",
3313 .cra_driver_name = "authenc-hmac-sha224-"
3314 "cbc-aes-caam",
3315 .cra_blocksize = AES_BLOCK_SIZE,
3316 },
3317 .setkey = aead_setkey,
3318 .setauthsize = aead_setauthsize,
3319 .encrypt = aead_encrypt,
3320 .decrypt = aead_decrypt,
3321 .ivsize = AES_BLOCK_SIZE,
3322 .maxauthsize = SHA224_DIGEST_SIZE,
3323 },
3324 .caam = {
3325 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3326 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3327 OP_ALG_AAI_HMAC_PRECOMP,
3328 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3329 },
3330 },
3331 {
3332 .aead = {
3333 .base = {
3334 .cra_name = "echainiv(authenc(hmac(sha224),"
3335 "cbc(aes)))",
3336 .cra_driver_name = "echainiv-authenc-"
3337 "hmac-sha224-cbc-aes-caam",
3338 .cra_blocksize = AES_BLOCK_SIZE,
3339 },
3340 .setkey = aead_setkey,
3341 .setauthsize = aead_setauthsize,
3342 .encrypt = aead_encrypt,
3343 .decrypt = aead_decrypt,
3344 .ivsize = AES_BLOCK_SIZE,
3345 .maxauthsize = SHA224_DIGEST_SIZE,
3346 },
3347 .caam = {
3348 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3349 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3350 OP_ALG_AAI_HMAC_PRECOMP,
3351 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3352 .geniv = true,
3353 },
3354 },
3355 {
3356 .aead = {
3357 .base = {
3358 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3359 .cra_driver_name = "authenc-hmac-sha256-"
3360 "cbc-aes-caam",
3361 .cra_blocksize = AES_BLOCK_SIZE,
3362 },
3363 .setkey = aead_setkey,
3364 .setauthsize = aead_setauthsize,
3365 .encrypt = aead_encrypt,
3366 .decrypt = aead_decrypt,
3367 .ivsize = AES_BLOCK_SIZE,
3368 .maxauthsize = SHA256_DIGEST_SIZE,
3369 },
3370 .caam = {
3371 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3372 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3373 OP_ALG_AAI_HMAC_PRECOMP,
3374 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3375 },
3376 },
3377 {
3378 .aead = {
3379 .base = {
3380 .cra_name = "echainiv(authenc(hmac(sha256),"
3381 "cbc(aes)))",
3382 .cra_driver_name = "echainiv-authenc-"
3383 "hmac-sha256-cbc-aes-caam",
3384 .cra_blocksize = AES_BLOCK_SIZE,
3385 },
3386 .setkey = aead_setkey,
3387 .setauthsize = aead_setauthsize,
3388 .encrypt = aead_encrypt,
3389 .decrypt = aead_decrypt,
3390 .ivsize = AES_BLOCK_SIZE,
3391 .maxauthsize = SHA256_DIGEST_SIZE,
3392 },
3393 .caam = {
3394 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3395 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3396 OP_ALG_AAI_HMAC_PRECOMP,
3397 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3398 .geniv = true,
3399 },
3400 },
3401 {
3402 .aead = {
3403 .base = {
3404 .cra_name = "authenc(hmac(sha384),cbc(aes))",
3405 .cra_driver_name = "authenc-hmac-sha384-"
3406 "cbc-aes-caam",
3407 .cra_blocksize = AES_BLOCK_SIZE,
3408 },
3409 .setkey = aead_setkey,
3410 .setauthsize = aead_setauthsize,
3411 .encrypt = aead_encrypt,
3412 .decrypt = aead_decrypt,
3413 .ivsize = AES_BLOCK_SIZE,
3414 .maxauthsize = SHA384_DIGEST_SIZE,
3415 },
3416 .caam = {
3417 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3418 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3419 OP_ALG_AAI_HMAC_PRECOMP,
3420 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3421 },
3422 },
3423 {
3424 .aead = {
3425 .base = {
3426 .cra_name = "echainiv(authenc(hmac(sha384),"
3427 "cbc(aes)))",
3428 .cra_driver_name = "echainiv-authenc-"
3429 "hmac-sha384-cbc-aes-caam",
3430 .cra_blocksize = AES_BLOCK_SIZE,
3431 },
3432 .setkey = aead_setkey,
3433 .setauthsize = aead_setauthsize,
3434 .encrypt = aead_encrypt,
3435 .decrypt = aead_decrypt,
3436 .ivsize = AES_BLOCK_SIZE,
3437 .maxauthsize = SHA384_DIGEST_SIZE,
3438 },
3439 .caam = {
3440 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3441 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3442 OP_ALG_AAI_HMAC_PRECOMP,
3443 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3444 .geniv = true,
3445 },
3446 },
3447 {
3448 .aead = {
3449 .base = {
3450 .cra_name = "authenc(hmac(sha512),cbc(aes))",
3451 .cra_driver_name = "authenc-hmac-sha512-"
3452 "cbc-aes-caam",
3453 .cra_blocksize = AES_BLOCK_SIZE,
3454 },
3455 .setkey = aead_setkey,
3456 .setauthsize = aead_setauthsize,
3457 .encrypt = aead_encrypt,
3458 .decrypt = aead_decrypt,
3459 .ivsize = AES_BLOCK_SIZE,
3460 .maxauthsize = SHA512_DIGEST_SIZE,
3461 },
3462 .caam = {
3463 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3464 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3465 OP_ALG_AAI_HMAC_PRECOMP,
3466 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3467 },
3468 },
3469 {
3470 .aead = {
3471 .base = {
3472 .cra_name = "echainiv(authenc(hmac(sha512),"
3473 "cbc(aes)))",
3474 .cra_driver_name = "echainiv-authenc-"
3475 "hmac-sha512-cbc-aes-caam",
3476 .cra_blocksize = AES_BLOCK_SIZE,
3477 },
3478 .setkey = aead_setkey,
3479 .setauthsize = aead_setauthsize,
3480 .encrypt = aead_encrypt,
3481 .decrypt = aead_decrypt,
3482 .ivsize = AES_BLOCK_SIZE,
3483 .maxauthsize = SHA512_DIGEST_SIZE,
3484 },
3485 .caam = {
3486 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3487 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3488 OP_ALG_AAI_HMAC_PRECOMP,
3489 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3490 .geniv = true,
3491 },
3492 },
3493 {
3494 .aead = {
3495 .base = {
3496 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
3497 .cra_driver_name = "authenc-hmac-md5-"
3498 "cbc-des3_ede-caam",
3499 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3500 },
3501 .setkey = aead_setkey,
3502 .setauthsize = aead_setauthsize,
3503 .encrypt = aead_encrypt,
3504 .decrypt = aead_decrypt,
3505 .ivsize = DES3_EDE_BLOCK_SIZE,
3506 .maxauthsize = MD5_DIGEST_SIZE,
3507 },
3508 .caam = {
3509 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3510 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3511 OP_ALG_AAI_HMAC_PRECOMP,
3512 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3513 }
3514 },
3515 {
3516 .aead = {
3517 .base = {
3518 .cra_name = "echainiv(authenc(hmac(md5),"
3519 "cbc(des3_ede)))",
3520 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3521 "cbc-des3_ede-caam",
3522 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3523 },
3524 .setkey = aead_setkey,
3525 .setauthsize = aead_setauthsize,
3526 .encrypt = aead_encrypt,
3527 .decrypt = aead_decrypt,
3528 .ivsize = DES3_EDE_BLOCK_SIZE,
3529 .maxauthsize = MD5_DIGEST_SIZE,
3530 },
3531 .caam = {
3532 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3533 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3534 OP_ALG_AAI_HMAC_PRECOMP,
3535 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3536 .geniv = true,
3537 }
3538 },
3539 {
3540 .aead = {
3541 .base = {
3542 .cra_name = "authenc(hmac(sha1),"
3543 "cbc(des3_ede))",
3544 .cra_driver_name = "authenc-hmac-sha1-"
3545 "cbc-des3_ede-caam",
3546 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3547 },
3548 .setkey = aead_setkey,
3549 .setauthsize = aead_setauthsize,
3550 .encrypt = aead_encrypt,
3551 .decrypt = aead_decrypt,
3552 .ivsize = DES3_EDE_BLOCK_SIZE,
3553 .maxauthsize = SHA1_DIGEST_SIZE,
3554 },
3555 .caam = {
3556 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3557 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3558 OP_ALG_AAI_HMAC_PRECOMP,
3559 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3560 },
3561 },
3562 {
3563 .aead = {
3564 .base = {
3565 .cra_name = "echainiv(authenc(hmac(sha1),"
3566 "cbc(des3_ede)))",
3567 .cra_driver_name = "echainiv-authenc-"
3568 "hmac-sha1-"
3569 "cbc-des3_ede-caam",
3570 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3571 },
3572 .setkey = aead_setkey,
3573 .setauthsize = aead_setauthsize,
3574 .encrypt = aead_encrypt,
3575 .decrypt = aead_decrypt,
3576 .ivsize = DES3_EDE_BLOCK_SIZE,
3577 .maxauthsize = SHA1_DIGEST_SIZE,
3578 },
3579 .caam = {
3580 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3581 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3582 OP_ALG_AAI_HMAC_PRECOMP,
3583 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3584 .geniv = true,
3585 },
3586 },
3587 {
3588 .aead = {
3589 .base = {
3590 .cra_name = "authenc(hmac(sha224),"
3591 "cbc(des3_ede))",
3592 .cra_driver_name = "authenc-hmac-sha224-"
3593 "cbc-des3_ede-caam",
3594 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3595 },
3596 .setkey = aead_setkey,
3597 .setauthsize = aead_setauthsize,
3598 .encrypt = aead_encrypt,
3599 .decrypt = aead_decrypt,
3600 .ivsize = DES3_EDE_BLOCK_SIZE,
3601 .maxauthsize = SHA224_DIGEST_SIZE,
3602 },
3603 .caam = {
3604 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3605 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3606 OP_ALG_AAI_HMAC_PRECOMP,
3607 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3608 },
3609 },
3610 {
3611 .aead = {
3612 .base = {
3613 .cra_name = "echainiv(authenc(hmac(sha224),"
3614 "cbc(des3_ede)))",
3615 .cra_driver_name = "echainiv-authenc-"
3616 "hmac-sha224-"
3617 "cbc-des3_ede-caam",
3618 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3619 },
3620 .setkey = aead_setkey,
3621 .setauthsize = aead_setauthsize,
3622 .encrypt = aead_encrypt,
3623 .decrypt = aead_decrypt,
3624 .ivsize = DES3_EDE_BLOCK_SIZE,
3625 .maxauthsize = SHA224_DIGEST_SIZE,
3626 },
3627 .caam = {
3628 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3629 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3630 OP_ALG_AAI_HMAC_PRECOMP,
3631 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3632 .geniv = true,
3633 },
3634 },
3635 {
3636 .aead = {
3637 .base = {
3638 .cra_name = "authenc(hmac(sha256),"
3639 "cbc(des3_ede))",
3640 .cra_driver_name = "authenc-hmac-sha256-"
3641 "cbc-des3_ede-caam",
3642 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3643 },
3644 .setkey = aead_setkey,
3645 .setauthsize = aead_setauthsize,
3646 .encrypt = aead_encrypt,
3647 .decrypt = aead_decrypt,
3648 .ivsize = DES3_EDE_BLOCK_SIZE,
3649 .maxauthsize = SHA256_DIGEST_SIZE,
3650 },
3651 .caam = {
3652 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3653 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3654 OP_ALG_AAI_HMAC_PRECOMP,
3655 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3656 },
3657 },
3658 {
3659 .aead = {
3660 .base = {
3661 .cra_name = "echainiv(authenc(hmac(sha256),"
3662 "cbc(des3_ede)))",
3663 .cra_driver_name = "echainiv-authenc-"
3664 "hmac-sha256-"
3665 "cbc-des3_ede-caam",
3666 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3667 },
3668 .setkey = aead_setkey,
3669 .setauthsize = aead_setauthsize,
3670 .encrypt = aead_encrypt,
3671 .decrypt = aead_decrypt,
3672 .ivsize = DES3_EDE_BLOCK_SIZE,
3673 .maxauthsize = SHA256_DIGEST_SIZE,
3674 },
3675 .caam = {
3676 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3677 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3678 OP_ALG_AAI_HMAC_PRECOMP,
3679 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3680 .geniv = true,
3681 },
3682 },
3683 {
3684 .aead = {
3685 .base = {
3686 .cra_name = "authenc(hmac(sha384),"
3687 "cbc(des3_ede))",
3688 .cra_driver_name = "authenc-hmac-sha384-"
3689 "cbc-des3_ede-caam",
3690 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3691 },
3692 .setkey = aead_setkey,
3693 .setauthsize = aead_setauthsize,
3694 .encrypt = aead_encrypt,
3695 .decrypt = aead_decrypt,
3696 .ivsize = DES3_EDE_BLOCK_SIZE,
3697 .maxauthsize = SHA384_DIGEST_SIZE,
3698 },
3699 .caam = {
3700 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3701 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3702 OP_ALG_AAI_HMAC_PRECOMP,
3703 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3704 },
3705 },
3706 {
3707 .aead = {
3708 .base = {
3709 .cra_name = "echainiv(authenc(hmac(sha384),"
3710 "cbc(des3_ede)))",
3711 .cra_driver_name = "echainiv-authenc-"
3712 "hmac-sha384-"
3713 "cbc-des3_ede-caam",
3714 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3715 },
3716 .setkey = aead_setkey,
3717 .setauthsize = aead_setauthsize,
3718 .encrypt = aead_encrypt,
3719 .decrypt = aead_decrypt,
3720 .ivsize = DES3_EDE_BLOCK_SIZE,
3721 .maxauthsize = SHA384_DIGEST_SIZE,
3722 },
3723 .caam = {
3724 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3725 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3726 OP_ALG_AAI_HMAC_PRECOMP,
3727 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3728 .geniv = true,
3729 },
3730 },
3731 {
3732 .aead = {
3733 .base = {
3734 .cra_name = "authenc(hmac(sha512),"
3735 "cbc(des3_ede))",
3736 .cra_driver_name = "authenc-hmac-sha512-"
3737 "cbc-des3_ede-caam",
3738 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3739 },
3740 .setkey = aead_setkey,
3741 .setauthsize = aead_setauthsize,
3742 .encrypt = aead_encrypt,
3743 .decrypt = aead_decrypt,
3744 .ivsize = DES3_EDE_BLOCK_SIZE,
3745 .maxauthsize = SHA512_DIGEST_SIZE,
3746 },
3747 .caam = {
3748 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3749 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3750 OP_ALG_AAI_HMAC_PRECOMP,
3751 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3752 },
3753 },
3754 {
3755 .aead = {
3756 .base = {
3757 .cra_name = "echainiv(authenc(hmac(sha512),"
3758 "cbc(des3_ede)))",
3759 .cra_driver_name = "echainiv-authenc-"
3760 "hmac-sha512-"
3761 "cbc-des3_ede-caam",
3762 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
3763 },
3764 .setkey = aead_setkey,
3765 .setauthsize = aead_setauthsize,
3766 .encrypt = aead_encrypt,
3767 .decrypt = aead_decrypt,
3768 .ivsize = DES3_EDE_BLOCK_SIZE,
3769 .maxauthsize = SHA512_DIGEST_SIZE,
3770 },
3771 .caam = {
3772 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3773 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3774 OP_ALG_AAI_HMAC_PRECOMP,
3775 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3776 .geniv = true,
3777 },
3778 },
3779 {
3780 .aead = {
3781 .base = {
3782 .cra_name = "authenc(hmac(md5),cbc(des))",
3783 .cra_driver_name = "authenc-hmac-md5-"
3784 "cbc-des-caam",
3785 .cra_blocksize = DES_BLOCK_SIZE,
3786 },
3787 .setkey = aead_setkey,
3788 .setauthsize = aead_setauthsize,
3789 .encrypt = aead_encrypt,
3790 .decrypt = aead_decrypt,
3791 .ivsize = DES_BLOCK_SIZE,
3792 .maxauthsize = MD5_DIGEST_SIZE,
3793 },
3794 .caam = {
3795 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3796 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3797 OP_ALG_AAI_HMAC_PRECOMP,
3798 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3799 },
3800 },
3801 {
3802 .aead = {
3803 .base = {
3804 .cra_name = "echainiv(authenc(hmac(md5),"
3805 "cbc(des)))",
3806 .cra_driver_name = "echainiv-authenc-hmac-md5-"
3807 "cbc-des-caam",
3808 .cra_blocksize = DES_BLOCK_SIZE,
3809 },
3810 .setkey = aead_setkey,
3811 .setauthsize = aead_setauthsize,
3812 .encrypt = aead_encrypt,
3813 .decrypt = aead_decrypt,
3814 .ivsize = DES_BLOCK_SIZE,
3815 .maxauthsize = MD5_DIGEST_SIZE,
3816 },
3817 .caam = {
3818 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3819 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
3820 OP_ALG_AAI_HMAC_PRECOMP,
3821 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3822 .geniv = true,
3823 },
3824 },
3825 {
3826 .aead = {
3827 .base = {
3828 .cra_name = "authenc(hmac(sha1),cbc(des))",
3829 .cra_driver_name = "authenc-hmac-sha1-"
3830 "cbc-des-caam",
3831 .cra_blocksize = DES_BLOCK_SIZE,
3832 },
3833 .setkey = aead_setkey,
3834 .setauthsize = aead_setauthsize,
3835 .encrypt = aead_encrypt,
3836 .decrypt = aead_decrypt,
3837 .ivsize = DES_BLOCK_SIZE,
3838 .maxauthsize = SHA1_DIGEST_SIZE,
3839 },
3840 .caam = {
3841 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3842 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3843 OP_ALG_AAI_HMAC_PRECOMP,
3844 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3845 },
3846 },
3847 {
3848 .aead = {
3849 .base = {
3850 .cra_name = "echainiv(authenc(hmac(sha1),"
3851 "cbc(des)))",
3852 .cra_driver_name = "echainiv-authenc-"
3853 "hmac-sha1-cbc-des-caam",
3854 .cra_blocksize = DES_BLOCK_SIZE,
3855 },
3856 .setkey = aead_setkey,
3857 .setauthsize = aead_setauthsize,
3858 .encrypt = aead_encrypt,
3859 .decrypt = aead_decrypt,
3860 .ivsize = DES_BLOCK_SIZE,
3861 .maxauthsize = SHA1_DIGEST_SIZE,
3862 },
3863 .caam = {
3864 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3865 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
3866 OP_ALG_AAI_HMAC_PRECOMP,
3867 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3868 .geniv = true,
3869 },
3870 },
3871 {
3872 .aead = {
3873 .base = {
3874 .cra_name = "authenc(hmac(sha224),cbc(des))",
3875 .cra_driver_name = "authenc-hmac-sha224-"
3876 "cbc-des-caam",
3877 .cra_blocksize = DES_BLOCK_SIZE,
3878 },
3879 .setkey = aead_setkey,
3880 .setauthsize = aead_setauthsize,
3881 .encrypt = aead_encrypt,
3882 .decrypt = aead_decrypt,
3883 .ivsize = DES_BLOCK_SIZE,
3884 .maxauthsize = SHA224_DIGEST_SIZE,
3885 },
3886 .caam = {
3887 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3888 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3889 OP_ALG_AAI_HMAC_PRECOMP,
3890 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3891 },
3892 },
3893 {
3894 .aead = {
3895 .base = {
3896 .cra_name = "echainiv(authenc(hmac(sha224),"
3897 "cbc(des)))",
3898 .cra_driver_name = "echainiv-authenc-"
3899 "hmac-sha224-cbc-des-caam",
3900 .cra_blocksize = DES_BLOCK_SIZE,
3901 },
3902 .setkey = aead_setkey,
3903 .setauthsize = aead_setauthsize,
3904 .encrypt = aead_encrypt,
3905 .decrypt = aead_decrypt,
3906 .ivsize = DES_BLOCK_SIZE,
3907 .maxauthsize = SHA224_DIGEST_SIZE,
3908 },
3909 .caam = {
3910 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3911 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3912 OP_ALG_AAI_HMAC_PRECOMP,
3913 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3914 .geniv = true,
3915 },
3916 },
3917 {
3918 .aead = {
3919 .base = {
3920 .cra_name = "authenc(hmac(sha256),cbc(des))",
3921 .cra_driver_name = "authenc-hmac-sha256-"
3922 "cbc-des-caam",
3923 .cra_blocksize = DES_BLOCK_SIZE,
3924 },
3925 .setkey = aead_setkey,
3926 .setauthsize = aead_setauthsize,
3927 .encrypt = aead_encrypt,
3928 .decrypt = aead_decrypt,
3929 .ivsize = DES_BLOCK_SIZE,
3930 .maxauthsize = SHA256_DIGEST_SIZE,
3931 },
3932 .caam = {
3933 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3934 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3935 OP_ALG_AAI_HMAC_PRECOMP,
3936 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3937 },
3938 },
3939 {
3940 .aead = {
3941 .base = {
3942 .cra_name = "echainiv(authenc(hmac(sha256),"
3943 "cbc(des)))",
3944 .cra_driver_name = "echainiv-authenc-"
3945 "hmac-sha256-cbc-des-caam",
3946 .cra_blocksize = DES_BLOCK_SIZE,
3947 },
3948 .setkey = aead_setkey,
3949 .setauthsize = aead_setauthsize,
3950 .encrypt = aead_encrypt,
3951 .decrypt = aead_decrypt,
3952 .ivsize = DES_BLOCK_SIZE,
3953 .maxauthsize = SHA256_DIGEST_SIZE,
3954 },
3955 .caam = {
3956 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3957 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3958 OP_ALG_AAI_HMAC_PRECOMP,
3959 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3960 .geniv = true,
3961 },
3962 },
3963 {
3964 .aead = {
3965 .base = {
3966 .cra_name = "authenc(hmac(sha384),cbc(des))",
3967 .cra_driver_name = "authenc-hmac-sha384-"
3968 "cbc-des-caam",
3969 .cra_blocksize = DES_BLOCK_SIZE,
3970 },
3971 .setkey = aead_setkey,
3972 .setauthsize = aead_setauthsize,
3973 .encrypt = aead_encrypt,
3974 .decrypt = aead_decrypt,
3975 .ivsize = DES_BLOCK_SIZE,
3976 .maxauthsize = SHA384_DIGEST_SIZE,
3977 },
3978 .caam = {
3979 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3980 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3981 OP_ALG_AAI_HMAC_PRECOMP,
3982 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3983 },
3984 },
3985 {
3986 .aead = {
3987 .base = {
3988 .cra_name = "echainiv(authenc(hmac(sha384),"
3989 "cbc(des)))",
3990 .cra_driver_name = "echainiv-authenc-"
3991 "hmac-sha384-cbc-des-caam",
3992 .cra_blocksize = DES_BLOCK_SIZE,
3993 },
3994 .setkey = aead_setkey,
3995 .setauthsize = aead_setauthsize,
3996 .encrypt = aead_encrypt,
3997 .decrypt = aead_decrypt,
3998 .ivsize = DES_BLOCK_SIZE,
3999 .maxauthsize = SHA384_DIGEST_SIZE,
4000 },
4001 .caam = {
4002 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
4003 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
4004 OP_ALG_AAI_HMAC_PRECOMP,
4005 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
4006 .geniv = true,
4007 },
4008 },
4009 {
4010 .aead = {
4011 .base = {
4012 .cra_name = "authenc(hmac(sha512),cbc(des))",
4013 .cra_driver_name = "authenc-hmac-sha512-"
4014 "cbc-des-caam",
4015 .cra_blocksize = DES_BLOCK_SIZE,
4016 },
4017 .setkey = aead_setkey,
4018 .setauthsize = aead_setauthsize,
4019 .encrypt = aead_encrypt,
4020 .decrypt = aead_decrypt,
4021 .ivsize = DES_BLOCK_SIZE,
4022 .maxauthsize = SHA512_DIGEST_SIZE,
4023 },
4024 .caam = {
4025 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
4026 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4027 OP_ALG_AAI_HMAC_PRECOMP,
4028 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4029 },
4030 },
4031 {
4032 .aead = {
4033 .base = {
4034 .cra_name = "echainiv(authenc(hmac(sha512),"
4035 "cbc(des)))",
4036 .cra_driver_name = "echainiv-authenc-"
4037 "hmac-sha512-cbc-des-caam",
4038 .cra_blocksize = DES_BLOCK_SIZE,
4039 },
4040 .setkey = aead_setkey,
4041 .setauthsize = aead_setauthsize,
4042 .encrypt = aead_encrypt,
4043 .decrypt = aead_decrypt,
4044 .ivsize = DES_BLOCK_SIZE,
4045 .maxauthsize = SHA512_DIGEST_SIZE,
4046 },
4047 .caam = {
4048 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
4049 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4050 OP_ALG_AAI_HMAC_PRECOMP,
4051 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4052 .geniv = true,
4053 },
4054 },
4055 {
4056 .aead = {
4057 .base = {
4058 .cra_name = "authenc(hmac(md5),"
4059 "rfc3686(ctr(aes)))",
4060 .cra_driver_name = "authenc-hmac-md5-"
4061 "rfc3686-ctr-aes-caam",
4062 .cra_blocksize = 1,
4063 },
4064 .setkey = aead_setkey,
4065 .setauthsize = aead_setauthsize,
4066 .encrypt = aead_encrypt,
4067 .decrypt = aead_decrypt,
4068 .ivsize = CTR_RFC3686_IV_SIZE,
4069 .maxauthsize = MD5_DIGEST_SIZE,
4070 },
4071 .caam = {
4072 .class1_alg_type = OP_ALG_ALGSEL_AES |
4073 OP_ALG_AAI_CTR_MOD128,
4074 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
4075 OP_ALG_AAI_HMAC_PRECOMP,
4076 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
4077 .rfc3686 = true,
4078 },
4079 },
4080 {
4081 .aead = {
4082 .base = {
4083 .cra_name = "seqiv(authenc("
4084 "hmac(md5),rfc3686(ctr(aes))))",
4085 .cra_driver_name = "seqiv-authenc-hmac-md5-"
4086 "rfc3686-ctr-aes-caam",
4087 .cra_blocksize = 1,
4088 },
4089 .setkey = aead_setkey,
4090 .setauthsize = aead_setauthsize,
4091 .encrypt = aead_encrypt,
4092 .decrypt = aead_decrypt,
4093 .ivsize = CTR_RFC3686_IV_SIZE,
4094 .maxauthsize = MD5_DIGEST_SIZE,
4095 },
4096 .caam = {
4097 .class1_alg_type = OP_ALG_ALGSEL_AES |
4098 OP_ALG_AAI_CTR_MOD128,
4099 .class2_alg_type = OP_ALG_ALGSEL_MD5 |
4100 OP_ALG_AAI_HMAC_PRECOMP,
4101 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
4102 .rfc3686 = true,
4103 .geniv = true,
4104 },
4105 },
4106 {
4107 .aead = {
4108 .base = {
4109 .cra_name = "authenc(hmac(sha1),"
4110 "rfc3686(ctr(aes)))",
4111 .cra_driver_name = "authenc-hmac-sha1-"
4112 "rfc3686-ctr-aes-caam",
4113 .cra_blocksize = 1,
4114 },
4115 .setkey = aead_setkey,
4116 .setauthsize = aead_setauthsize,
4117 .encrypt = aead_encrypt,
4118 .decrypt = aead_decrypt,
4119 .ivsize = CTR_RFC3686_IV_SIZE,
4120 .maxauthsize = SHA1_DIGEST_SIZE,
4121 },
4122 .caam = {
4123 .class1_alg_type = OP_ALG_ALGSEL_AES |
4124 OP_ALG_AAI_CTR_MOD128,
4125 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
4126 OP_ALG_AAI_HMAC_PRECOMP,
4127 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
4128 .rfc3686 = true,
4129 },
4130 },
4131 {
4132 .aead = {
4133 .base = {
4134 .cra_name = "seqiv(authenc("
4135 "hmac(sha1),rfc3686(ctr(aes))))",
4136 .cra_driver_name = "seqiv-authenc-hmac-sha1-"
4137 "rfc3686-ctr-aes-caam",
4138 .cra_blocksize = 1,
4139 },
4140 .setkey = aead_setkey,
4141 .setauthsize = aead_setauthsize,
4142 .encrypt = aead_encrypt,
4143 .decrypt = aead_decrypt,
4144 .ivsize = CTR_RFC3686_IV_SIZE,
4145 .maxauthsize = SHA1_DIGEST_SIZE,
4146 },
4147 .caam = {
4148 .class1_alg_type = OP_ALG_ALGSEL_AES |
4149 OP_ALG_AAI_CTR_MOD128,
4150 .class2_alg_type = OP_ALG_ALGSEL_SHA1 |
4151 OP_ALG_AAI_HMAC_PRECOMP,
4152 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
4153 .rfc3686 = true,
4154 .geniv = true,
4155 },
4156 },
4157 {
4158 .aead = {
4159 .base = {
4160 .cra_name = "authenc(hmac(sha224),"
4161 "rfc3686(ctr(aes)))",
4162 .cra_driver_name = "authenc-hmac-sha224-"
4163 "rfc3686-ctr-aes-caam",
4164 .cra_blocksize = 1,
4165 },
4166 .setkey = aead_setkey,
4167 .setauthsize = aead_setauthsize,
4168 .encrypt = aead_encrypt,
4169 .decrypt = aead_decrypt,
4170 .ivsize = CTR_RFC3686_IV_SIZE,
4171 .maxauthsize = SHA224_DIGEST_SIZE,
4172 },
4173 .caam = {
4174 .class1_alg_type = OP_ALG_ALGSEL_AES |
4175 OP_ALG_AAI_CTR_MOD128,
4176 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
4177 OP_ALG_AAI_HMAC_PRECOMP,
4178 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
4179 .rfc3686 = true,
4180 },
4181 },
4182 {
4183 .aead = {
4184 .base = {
4185 .cra_name = "seqiv(authenc("
4186 "hmac(sha224),rfc3686(ctr(aes))))",
4187 .cra_driver_name = "seqiv-authenc-hmac-sha224-"
4188 "rfc3686-ctr-aes-caam",
4189 .cra_blocksize = 1,
4190 },
4191 .setkey = aead_setkey,
4192 .setauthsize = aead_setauthsize,
4193 .encrypt = aead_encrypt,
4194 .decrypt = aead_decrypt,
4195 .ivsize = CTR_RFC3686_IV_SIZE,
4196 .maxauthsize = SHA224_DIGEST_SIZE,
4197 },
4198 .caam = {
4199 .class1_alg_type = OP_ALG_ALGSEL_AES |
4200 OP_ALG_AAI_CTR_MOD128,
4201 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
4202 OP_ALG_AAI_HMAC_PRECOMP,
4203 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
4204 .rfc3686 = true,
4205 .geniv = true,
4206 },
4207 },
4208 {
4209 .aead = {
4210 .base = {
4211 .cra_name = "authenc(hmac(sha256),"
4212 "rfc3686(ctr(aes)))",
4213 .cra_driver_name = "authenc-hmac-sha256-"
4214 "rfc3686-ctr-aes-caam",
4215 .cra_blocksize = 1,
4216 },
4217 .setkey = aead_setkey,
4218 .setauthsize = aead_setauthsize,
4219 .encrypt = aead_encrypt,
4220 .decrypt = aead_decrypt,
4221 .ivsize = CTR_RFC3686_IV_SIZE,
4222 .maxauthsize = SHA256_DIGEST_SIZE,
4223 },
4224 .caam = {
4225 .class1_alg_type = OP_ALG_ALGSEL_AES |
4226 OP_ALG_AAI_CTR_MOD128,
4227 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
4228 OP_ALG_AAI_HMAC_PRECOMP,
4229 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
4230 .rfc3686 = true,
4231 },
4232 },
4233 {
4234 .aead = {
4235 .base = {
4236 .cra_name = "seqiv(authenc(hmac(sha256),"
4237 "rfc3686(ctr(aes))))",
4238 .cra_driver_name = "seqiv-authenc-hmac-sha256-"
4239 "rfc3686-ctr-aes-caam",
4240 .cra_blocksize = 1,
4241 },
4242 .setkey = aead_setkey,
4243 .setauthsize = aead_setauthsize,
4244 .encrypt = aead_encrypt,
4245 .decrypt = aead_decrypt,
4246 .ivsize = CTR_RFC3686_IV_SIZE,
4247 .maxauthsize = SHA256_DIGEST_SIZE,
4248 },
4249 .caam = {
4250 .class1_alg_type = OP_ALG_ALGSEL_AES |
4251 OP_ALG_AAI_CTR_MOD128,
4252 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
4253 OP_ALG_AAI_HMAC_PRECOMP,
4254 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
4255 .rfc3686 = true,
4256 .geniv = true,
4257 },
4258 },
4259 {
4260 .aead = {
4261 .base = {
4262 .cra_name = "authenc(hmac(sha384),"
4263 "rfc3686(ctr(aes)))",
4264 .cra_driver_name = "authenc-hmac-sha384-"
4265 "rfc3686-ctr-aes-caam",
4266 .cra_blocksize = 1,
4267 },
4268 .setkey = aead_setkey,
4269 .setauthsize = aead_setauthsize,
4270 .encrypt = aead_encrypt,
4271 .decrypt = aead_decrypt,
4272 .ivsize = CTR_RFC3686_IV_SIZE,
4273 .maxauthsize = SHA384_DIGEST_SIZE,
4274 },
4275 .caam = {
4276 .class1_alg_type = OP_ALG_ALGSEL_AES |
4277 OP_ALG_AAI_CTR_MOD128,
4278 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
4279 OP_ALG_AAI_HMAC_PRECOMP,
4280 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
4281 .rfc3686 = true,
4282 },
4283 },
4284 {
4285 .aead = {
4286 .base = {
4287 .cra_name = "seqiv(authenc(hmac(sha384),"
4288 "rfc3686(ctr(aes))))",
4289 .cra_driver_name = "seqiv-authenc-hmac-sha384-"
4290 "rfc3686-ctr-aes-caam",
4291 .cra_blocksize = 1,
4292 },
4293 .setkey = aead_setkey,
4294 .setauthsize = aead_setauthsize,
4295 .encrypt = aead_encrypt,
4296 .decrypt = aead_decrypt,
4297 .ivsize = CTR_RFC3686_IV_SIZE,
4298 .maxauthsize = SHA384_DIGEST_SIZE,
4299 },
4300 .caam = {
4301 .class1_alg_type = OP_ALG_ALGSEL_AES |
4302 OP_ALG_AAI_CTR_MOD128,
4303 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
4304 OP_ALG_AAI_HMAC_PRECOMP,
4305 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
4306 .rfc3686 = true,
4307 .geniv = true,
4308 },
4309 },
4310 {
4311 .aead = {
4312 .base = {
4313 .cra_name = "authenc(hmac(sha512),"
4314 "rfc3686(ctr(aes)))",
4315 .cra_driver_name = "authenc-hmac-sha512-"
4316 "rfc3686-ctr-aes-caam",
4317 .cra_blocksize = 1,
4318 },
4319 .setkey = aead_setkey,
4320 .setauthsize = aead_setauthsize,
4321 .encrypt = aead_encrypt,
4322 .decrypt = aead_decrypt,
4323 .ivsize = CTR_RFC3686_IV_SIZE,
4324 .maxauthsize = SHA512_DIGEST_SIZE,
4325 },
4326 .caam = {
4327 .class1_alg_type = OP_ALG_ALGSEL_AES |
4328 OP_ALG_AAI_CTR_MOD128,
4329 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4330 OP_ALG_AAI_HMAC_PRECOMP,
4331 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4332 .rfc3686 = true,
4333 },
4334 },
4335 {
4336 .aead = {
4337 .base = {
4338 .cra_name = "seqiv(authenc(hmac(sha512),"
4339 "rfc3686(ctr(aes))))",
4340 .cra_driver_name = "seqiv-authenc-hmac-sha512-"
4341 "rfc3686-ctr-aes-caam",
4342 .cra_blocksize = 1,
4343 },
4344 .setkey = aead_setkey,
4345 .setauthsize = aead_setauthsize,
4346 .encrypt = aead_encrypt,
4347 .decrypt = aead_decrypt,
4348 .ivsize = CTR_RFC3686_IV_SIZE,
4349 .maxauthsize = SHA512_DIGEST_SIZE,
4350 },
4351 .caam = {
4352 .class1_alg_type = OP_ALG_ALGSEL_AES |
4353 OP_ALG_AAI_CTR_MOD128,
4354 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
4355 OP_ALG_AAI_HMAC_PRECOMP,
4356 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
4357 .rfc3686 = true,
4358 .geniv = true,
4359 },
4360 },
4361 };
4362
4363 struct caam_crypto_alg {
4364 struct crypto_alg crypto_alg;
4365 struct list_head entry;
4366 struct caam_alg_entry caam;
4367 };
4368
4369 static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam)
4370 {
4371 ctx->jrdev = caam_jr_alloc();
4372 if (IS_ERR(ctx->jrdev)) {
4373 pr_err("Job Ring Device allocation for transform failed\n");
4374 return PTR_ERR(ctx->jrdev);
4375 }
4376
4377 /* copy descriptor header template value */
4378 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type;
4379 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type;
4380 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op;
4381
4382 return 0;
4383 }
4384
4385 static int caam_cra_init(struct crypto_tfm *tfm)
4386 {
4387 struct crypto_alg *alg = tfm->__crt_alg;
4388 struct caam_crypto_alg *caam_alg =
4389 container_of(alg, struct caam_crypto_alg, crypto_alg);
4390 struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
4391
4392 return caam_init_common(ctx, &caam_alg->caam);
4393 }
4394
4395 static int caam_aead_init(struct crypto_aead *tfm)
4396 {
4397 struct aead_alg *alg = crypto_aead_alg(tfm);
4398 struct caam_aead_alg *caam_alg =
4399 container_of(alg, struct caam_aead_alg, aead);
4400 struct caam_ctx *ctx = crypto_aead_ctx(tfm);
4401
4402 return caam_init_common(ctx, &caam_alg->caam);
4403 }
4404
4405 static void caam_exit_common(struct caam_ctx *ctx)
4406 {
4407 if (ctx->sh_desc_enc_dma &&
4408 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
4409 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
4410 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
4411 if (ctx->sh_desc_dec_dma &&
4412 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
4413 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
4414 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
4415 if (ctx->sh_desc_givenc_dma &&
4416 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
4417 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
4418 desc_bytes(ctx->sh_desc_givenc),
4419 DMA_TO_DEVICE);
4420 if (ctx->key_dma &&
4421 !dma_mapping_error(ctx->jrdev, ctx->key_dma))
4422 dma_unmap_single(ctx->jrdev, ctx->key_dma,
4423 ctx->enckeylen + ctx->split_key_pad_len,
4424 DMA_TO_DEVICE);
4425
4426 caam_jr_free(ctx->jrdev);
4427 }
4428
4429 static void caam_cra_exit(struct crypto_tfm *tfm)
4430 {
4431 caam_exit_common(crypto_tfm_ctx(tfm));
4432 }
4433
4434 static void caam_aead_exit(struct crypto_aead *tfm)
4435 {
4436 caam_exit_common(crypto_aead_ctx(tfm));
4437 }
4438
4439 static void __exit caam_algapi_exit(void)
4440 {
4441
4442 struct caam_crypto_alg *t_alg, *n;
4443 int i;
4444
4445 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
4446 struct caam_aead_alg *t_alg = driver_aeads + i;
4447
4448 if (t_alg->registered)
4449 crypto_unregister_aead(&t_alg->aead);
4450 }
4451
4452 if (!alg_list.next)
4453 return;
4454
4455 list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
4456 crypto_unregister_alg(&t_alg->crypto_alg);
4457 list_del(&t_alg->entry);
4458 kfree(t_alg);
4459 }
4460 }
4461
4462 static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
4463 *template)
4464 {
4465 struct caam_crypto_alg *t_alg;
4466 struct crypto_alg *alg;
4467
4468 t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL);
4469 if (!t_alg) {
4470 pr_err("failed to allocate t_alg\n");
4471 return ERR_PTR(-ENOMEM);
4472 }
4473
4474 alg = &t_alg->crypto_alg;
4475
4476 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
4477 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4478 template->driver_name);
4479 alg->cra_module = THIS_MODULE;
4480 alg->cra_init = caam_cra_init;
4481 alg->cra_exit = caam_cra_exit;
4482 alg->cra_priority = CAAM_CRA_PRIORITY;
4483 alg->cra_blocksize = template->blocksize;
4484 alg->cra_alignmask = 0;
4485 alg->cra_ctxsize = sizeof(struct caam_ctx);
4486 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
4487 template->type;
4488 switch (template->type) {
4489 case CRYPTO_ALG_TYPE_GIVCIPHER:
4490 alg->cra_type = &crypto_givcipher_type;
4491 alg->cra_ablkcipher = template->template_ablkcipher;
4492 break;
4493 case CRYPTO_ALG_TYPE_ABLKCIPHER:
4494 alg->cra_type = &crypto_ablkcipher_type;
4495 alg->cra_ablkcipher = template->template_ablkcipher;
4496 break;
4497 }
4498
4499 t_alg->caam.class1_alg_type = template->class1_alg_type;
4500 t_alg->caam.class2_alg_type = template->class2_alg_type;
4501 t_alg->caam.alg_op = template->alg_op;
4502
4503 return t_alg;
4504 }
4505
4506 static void caam_aead_alg_init(struct caam_aead_alg *t_alg)
4507 {
4508 struct aead_alg *alg = &t_alg->aead;
4509
4510 alg->base.cra_module = THIS_MODULE;
4511 alg->base.cra_priority = CAAM_CRA_PRIORITY;
4512 alg->base.cra_ctxsize = sizeof(struct caam_ctx);
4513 alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY;
4514
4515 alg->init = caam_aead_init;
4516 alg->exit = caam_aead_exit;
4517 }
4518
4519 static int __init caam_algapi_init(void)
4520 {
4521 struct device_node *dev_node;
4522 struct platform_device *pdev;
4523 struct device *ctrldev;
4524 struct caam_drv_private *priv;
4525 int i = 0, err = 0;
4526 u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst;
4527 unsigned int md_limit = SHA512_DIGEST_SIZE;
4528 bool registered = false;
4529
4530 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
4531 if (!dev_node) {
4532 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
4533 if (!dev_node)
4534 return -ENODEV;
4535 }
4536
4537 pdev = of_find_device_by_node(dev_node);
4538 if (!pdev) {
4539 of_node_put(dev_node);
4540 return -ENODEV;
4541 }
4542
4543 ctrldev = &pdev->dev;
4544 priv = dev_get_drvdata(ctrldev);
4545 of_node_put(dev_node);
4546
4547 /*
4548 * If priv is NULL, it's probably because the caam driver wasn't
4549 * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
4550 */
4551 if (!priv)
4552 return -ENODEV;
4553
4554
4555 INIT_LIST_HEAD(&alg_list);
4556
4557 /*
4558 * Register crypto algorithms the device supports.
4559 * First, detect presence and attributes of DES, AES, and MD blocks.
4560 */
4561 cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls);
4562 cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls);
4563 des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT;
4564 aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT;
4565 md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT;
4566
4567 /* If MD is present, limit digest size based on LP256 */
4568 if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256))
4569 md_limit = SHA256_DIGEST_SIZE;
4570
4571 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4572 struct caam_crypto_alg *t_alg;
4573 struct caam_alg_template *alg = driver_algs + i;
4574 u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK;
4575
4576 /* Skip DES algorithms if not supported by device */
4577 if (!des_inst &&
4578 ((alg_sel == OP_ALG_ALGSEL_3DES) ||
4579 (alg_sel == OP_ALG_ALGSEL_DES)))
4580 continue;
4581
4582 /* Skip AES algorithms if not supported by device */
4583 if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES))
4584 continue;
4585
4586 t_alg = caam_alg_alloc(alg);
4587 if (IS_ERR(t_alg)) {
4588 err = PTR_ERR(t_alg);
4589 pr_warn("%s alg allocation failed\n", alg->driver_name);
4590 continue;
4591 }
4592
4593 err = crypto_register_alg(&t_alg->crypto_alg);
4594 if (err) {
4595 pr_warn("%s alg registration failed\n",
4596 t_alg->crypto_alg.cra_driver_name);
4597 kfree(t_alg);
4598 continue;
4599 }
4600
4601 list_add_tail(&t_alg->entry, &alg_list);
4602 registered = true;
4603 }
4604
4605 for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) {
4606 struct caam_aead_alg *t_alg = driver_aeads + i;
4607 u32 c1_alg_sel = t_alg->caam.class1_alg_type &
4608 OP_ALG_ALGSEL_MASK;
4609 u32 c2_alg_sel = t_alg->caam.class2_alg_type &
4610 OP_ALG_ALGSEL_MASK;
4611 u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK;
4612
4613 /* Skip DES algorithms if not supported by device */
4614 if (!des_inst &&
4615 ((c1_alg_sel == OP_ALG_ALGSEL_3DES) ||
4616 (c1_alg_sel == OP_ALG_ALGSEL_DES)))
4617 continue;
4618
4619 /* Skip AES algorithms if not supported by device */
4620 if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES))
4621 continue;
4622
4623 /*
4624 * Check support for AES algorithms not available
4625 * on LP devices.
4626 */
4627 if ((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP)
4628 if (alg_aai == OP_ALG_AAI_GCM)
4629 continue;
4630
4631 /*
4632 * Skip algorithms requiring message digests
4633 * if MD or MD size is not supported by device.
4634 */
4635 if (c2_alg_sel &&
4636 (!md_inst || (t_alg->aead.maxauthsize > md_limit)))
4637 continue;
4638
4639 caam_aead_alg_init(t_alg);
4640
4641 err = crypto_register_aead(&t_alg->aead);
4642 if (err) {
4643 pr_warn("%s alg registration failed\n",
4644 t_alg->aead.base.cra_driver_name);
4645 continue;
4646 }
4647
4648 t_alg->registered = true;
4649 registered = true;
4650 }
4651
4652 if (registered)
4653 pr_info("caam algorithms registered in /proc/crypto\n");
4654
4655 return err;
4656 }
4657
4658 module_init(caam_algapi_init);
4659 module_exit(caam_algapi_exit);
4660
4661 MODULE_LICENSE("GPL");
4662 MODULE_DESCRIPTION("FSL CAAM support for crypto API");
4663 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
ubuntu-bionic-kernel mirror