]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - drivers/crypto/caam/caamalg.c
projects / mirror_ubuntu-artful-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
crypto: caam - don't emit ICV check failures to dmesg
[mirror_ubuntu-artful-kernel.git] / drivers / crypto / caam / caamalg.c
CommitLineData
8e8ec596
KP		 1/*
2 * caam - Freescale FSL CAAM support for crypto API
3 *
4 * Copyright 2008-2011 Freescale Semiconductor, Inc.
5 *
6 * Based on talitos crypto API driver.
7 *
8 * relationship of job descriptors to shared descriptors (SteveC Dec 10 2008):
9 *
10 * --------------- ---------------
11 * | JobDesc #1 |-------------------->| ShareDesc |
12 * | *(packet 1) | | (PDB) |
13 * --------------- |------------->| (hashKey) |
14 * . | | (cipherKey) |
15 * . | |-------->| (operation) |
16 * --------------- | | ---------------
17 * | JobDesc #2 |------| |
18 * | *(packet 2) | |
19 * --------------- |
20 * . |
21 * . |
22 * --------------- |
23 * | JobDesc #3 |------------
24 * | *(packet 3) |
25 * ---------------
26 *
27 * The SharedDesc never changes for a connection unless rekeyed, but
28 * each packet will likely be in a different place. So all we need
29 * to know to process the packet is where the input is, where the
30 * output goes, and what context we want to process with. Context is
31 * in the SharedDesc, packet references in the JobDesc.
32 *
33 * So, a job desc looks like:
34 *
35 * ---------------------
36 * | Header |
37 * | ShareDesc Pointer |
38 * | SEQ_OUT_PTR |
39 * | (output buffer) |
6ec47334 40 * | (output length) |
8e8ec596
KP		 41 * | SEQ_IN_PTR |
42 * | (input buffer) |
6ec47334 43 * | (input length) |
8e8ec596
KP		 44 * ---------------------
45 */
46
47#include "compat.h"
48
49#include "regs.h"
50#include "intern.h"
51#include "desc_constr.h"
52#include "jr.h"
53#include "error.h"
a299c837 54#include "sg_sw_sec4.h"
4c1ec1f9 55#include "key_gen.h"
8e8ec596
KP		 56
57/*
58 * crypto alg
59 */
60#define CAAM_CRA_PRIORITY 3000
61/* max key is sum of AES_MAX_KEY_SIZE, max split key size */
62#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \
daebc465 63 CTR_RFC3686_NONCE_SIZE + \
8e8ec596
KP		 64 SHA512_DIGEST_SIZE * 2)
65/* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
66#define CAAM_MAX_IV_LENGTH 16
67
4427b1b4 68/* length of descriptors text */
1acebad3 69#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ)
4464a7d4
HG		 70#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ)
71#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 18 * CAAM_CMD_SZ)
1acebad3
YK		 72#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ)
73
daebc465
CV		 74/* Note: Nonce is counted in enckeylen */
75#define DESC_AEAD_CTR_RFC3686_LEN (6 * CAAM_CMD_SZ)
76
ae4a825f
HG		 77#define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ)
78#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 14 * CAAM_CMD_SZ)
79#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 17 * CAAM_CMD_SZ)
80
3ef8d945
TA		 81#define DESC_GCM_BASE (3 * CAAM_CMD_SZ)
82#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 23 * CAAM_CMD_SZ)
83#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 19 * CAAM_CMD_SZ)
84
bac68f2c
TA		 85#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ)
86#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 15 * CAAM_CMD_SZ)
87#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 14 * CAAM_CMD_SZ)
88#define DESC_RFC4106_GIVENC_LEN (DESC_RFC4106_BASE + 21 * CAAM_CMD_SZ)
89
5d0429a3
TA		 90#define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ)
91#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 25 * CAAM_CMD_SZ)
92#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 27 * CAAM_CMD_SZ)
93#define DESC_RFC4543_GIVENC_LEN (DESC_RFC4543_BASE + 30 * CAAM_CMD_SZ)
94
acdca31d
YK		 95#define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ)
96#define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \
97 20 * CAAM_CMD_SZ)
98#define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \
99 15 * CAAM_CMD_SZ)
100
5d0429a3 101#define DESC_MAX_USED_BYTES (DESC_RFC4543_GIVENC_LEN + \
1acebad3
YK		 102 CAAM_MAX_KEY_SIZE)
103#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ)
4427b1b4 104
8e8ec596
KP		 105#ifdef DEBUG
106/* for print_hex_dumps with line references */
8e8ec596
KP		 107#define debug(format, arg...) printk(format, arg)
108#else
109#define debug(format, arg...)
110#endif
cfc6f11b 111static struct list_head alg_list;
8e8ec596 112
1acebad3
YK		 113/* Set DK bit in class 1 operation if shared */
114static inline void append_dec_op1(u32 *desc, u32 type)
115{
116 u32 *jump_cmd, *uncond_jump_cmd;
117
a60384df
HG		 118 /* DK bit is valid only for AES */
119 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
120 append_operation(desc, type | OP_ALG_AS_INITFINAL |
121 OP_ALG_DECRYPT);
122 return;
123 }
124
1acebad3
YK		 125 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
126 append_operation(desc, type | OP_ALG_AS_INITFINAL |
127 OP_ALG_DECRYPT);
128 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
129 set_jump_tgt_here(desc, jump_cmd);
130 append_operation(desc, type | OP_ALG_AS_INITFINAL |
131 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
132 set_jump_tgt_here(desc, uncond_jump_cmd);
133}
134
1acebad3
YK		 135/*
136 * For aead functions, read payload and write payload,
137 * both of which are specified in req->src and req->dst
138 */
139static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
140{
ae4a825f 141 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1acebad3
YK		 142 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
143 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
1acebad3
YK		 144}
145
146/*
147 * For aead encrypt and decrypt, read iv for both classes
148 */
daebc465 149static inline void aead_append_ld_iv(u32 *desc, int ivsize, int ivoffset)
1acebad3 150{
daebc465
CV		 151 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
152 LDST_SRCDST_BYTE_CONTEXT |
153 (ivoffset << LDST_OFFSET_SHIFT));
154 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
155 (ivoffset << MOVE_OFFSET_SHIFT) | ivsize);
1acebad3
YK		 156}
157
acdca31d
YK		 158/*
159 * For ablkcipher encrypt and decrypt, read from req->src and
160 * write to req->dst
161 */
162static inline void ablkcipher_append_src_dst(u32 *desc)
163{
70d793cc
KP		 164 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
165 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
166 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
167 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
168 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
acdca31d
YK		 169}
170
1acebad3
YK		 171/*
172 * If all data, including src (with assoc and iv) or dst (with iv only) are
173 * contiguous
174 */
175#define GIV_SRC_CONTIG 1
176#define GIV_DST_CONTIG (1 << 1)
177
8e8ec596
KP		 178/*
179 * per-session context
180 */
181struct caam_ctx {
182 struct device *jrdev;
1acebad3
YK		 183 u32 sh_desc_enc[DESC_MAX_USED_LEN];
184 u32 sh_desc_dec[DESC_MAX_USED_LEN];
185 u32 sh_desc_givenc[DESC_MAX_USED_LEN];
186 dma_addr_t sh_desc_enc_dma;
187 dma_addr_t sh_desc_dec_dma;
188 dma_addr_t sh_desc_givenc_dma;
8e8ec596
KP		 189 u32 class1_alg_type;
190 u32 class2_alg_type;
191 u32 alg_op;
1acebad3 192 u8 key[CAAM_MAX_KEY_SIZE];
885e9e2f 193 dma_addr_t key_dma;
8e8ec596 194 unsigned int enckeylen;
8e8ec596
KP		 195 unsigned int split_key_len;
196 unsigned int split_key_pad_len;
197 unsigned int authsize;
198};
199
1acebad3 200static void append_key_aead(u32 *desc, struct caam_ctx *ctx,
daebc465 201 int keys_fit_inline, bool is_rfc3686)
1acebad3 202{
daebc465
CV		 203 u32 *nonce;
204 unsigned int enckeylen = ctx->enckeylen;
205
206 /*
207 * RFC3686 specific:
208 * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE}
209 * | enckeylen = encryption key size + nonce size
210 */
211 if (is_rfc3686)
212 enckeylen -= CTR_RFC3686_NONCE_SIZE;
213
1acebad3
YK		 214 if (keys_fit_inline) {
215 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
216 ctx->split_key_len, CLASS_2 |
217 KEY_DEST_MDHA_SPLIT | KEY_ENC);
218 append_key_as_imm(desc, (void *)ctx->key +
daebc465
CV		 219 ctx->split_key_pad_len, enckeylen,
220 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1acebad3
YK		 221 } else {
222 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
223 KEY_DEST_MDHA_SPLIT | KEY_ENC);
224 append_key(desc, ctx->key_dma + ctx->split_key_pad_len,
daebc465
CV		 225 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
226 }
227
228 /* Load Counter into CONTEXT1 reg */
229 if (is_rfc3686) {
230 nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len +
231 enckeylen);
232 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
233 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
234 append_move(desc,
235 MOVE_SRC_OUTFIFO |
236 MOVE_DEST_CLASS1CTX |
237 (16 << MOVE_OFFSET_SHIFT) |
238 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1acebad3
YK		 239 }
240}
241
242static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx,
daebc465 243 int keys_fit_inline, bool is_rfc3686)
1acebad3
YK		 244{
245 u32 *key_jump_cmd;
246
daebc465
CV		 247 /* Note: Context registers are saved. */
248 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1acebad3
YK		 249
250 /* Skip if already shared */
251 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
252 JUMP_COND_SHRD);
253
daebc465 254 append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
1acebad3
YK		 255
256 set_jump_tgt_here(desc, key_jump_cmd);
1acebad3
YK		 257}
258
ae4a825f
HG		 259static int aead_null_set_sh_desc(struct crypto_aead *aead)
260{
261 struct aead_tfm *tfm = &aead->base.crt_aead;
262 struct caam_ctx *ctx = crypto_aead_ctx(aead);
263 struct device *jrdev = ctx->jrdev;
264 bool keys_fit_inline = false;
265 u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd;
266 u32 *desc;
267
268 /*
269 * Job Descriptor and Shared Descriptors
270 * must all fit into the 64-word Descriptor h/w Buffer
271 */
272 if (DESC_AEAD_NULL_ENC_LEN + DESC_JOB_IO_LEN +
273 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
274 keys_fit_inline = true;
275
276 /* aead_encrypt shared descriptor */
277 desc = ctx->sh_desc_enc;
278
279 init_sh_desc(desc, HDR_SHARE_SERIAL);
280
281 /* Skip if already shared */
282 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
283 JUMP_COND_SHRD);
284 if (keys_fit_inline)
285 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
286 ctx->split_key_len, CLASS_2 |
287 KEY_DEST_MDHA_SPLIT | KEY_ENC);
288 else
289 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
290 KEY_DEST_MDHA_SPLIT | KEY_ENC);
291 set_jump_tgt_here(desc, key_jump_cmd);
292
293 /* cryptlen = seqoutlen - authsize */
294 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
295
296 /*
297 * NULL encryption; IV is zero
298 * assoclen = (assoclen + cryptlen) - cryptlen
299 */
300 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
301
302 /* read assoc before reading payload */
303 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
304 KEY_VLF);
305
306 /* Prepare to read and write cryptlen bytes */
307 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
308 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
309
310 /*
311 * MOVE_LEN opcode is not available in all SEC HW revisions,
312 * thus need to do some magic, i.e. self-patch the descriptor
313 * buffer.
314 */
315 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
316 MOVE_DEST_MATH3 |
317 (0x6 << MOVE_LEN_SHIFT));
318 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
319 MOVE_DEST_DESCBUF |
320 MOVE_WAITCOMP |
321 (0x8 << MOVE_LEN_SHIFT));
322
323 /* Class 2 operation */
324 append_operation(desc, ctx->class2_alg_type |
325 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
326
327 /* Read and write cryptlen bytes */
328 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
329
330 set_move_tgt_here(desc, read_move_cmd);
331 set_move_tgt_here(desc, write_move_cmd);
332 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
333 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
334 MOVE_AUX_LS);
335
336 /* Write ICV */
337 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
338 LDST_SRCDST_BYTE_CONTEXT);
339
340 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
341 desc_bytes(desc),
342 DMA_TO_DEVICE);
343 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
344 dev_err(jrdev, "unable to map shared descriptor\n");
345 return -ENOMEM;
346 }
347#ifdef DEBUG
348 print_hex_dump(KERN_ERR,
349 "aead null enc shdesc@"__stringify(__LINE__)": ",
350 DUMP_PREFIX_ADDRESS, 16, 4, desc,
351 desc_bytes(desc), 1);
352#endif
353
354 /*
355 * Job Descriptor and Shared Descriptors
356 * must all fit into the 64-word Descriptor h/w Buffer
357 */
80cd88f2 358 keys_fit_inline = false;
ae4a825f
HG		 359 if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN +
360 ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX)
361 keys_fit_inline = true;
362
363 desc = ctx->sh_desc_dec;
364
365 /* aead_decrypt shared descriptor */
366 init_sh_desc(desc, HDR_SHARE_SERIAL);
367
368 /* Skip if already shared */
369 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
370 JUMP_COND_SHRD);
371 if (keys_fit_inline)
372 append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len,
373 ctx->split_key_len, CLASS_2 |
374 KEY_DEST_MDHA_SPLIT | KEY_ENC);
375 else
376 append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 |
377 KEY_DEST_MDHA_SPLIT | KEY_ENC);
378 set_jump_tgt_here(desc, key_jump_cmd);
379
380 /* Class 2 operation */
381 append_operation(desc, ctx->class2_alg_type |
382 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
383
384 /* assoclen + cryptlen = seqinlen - ivsize - authsize */
385 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
386 ctx->authsize + tfm->ivsize);
387 /* assoclen = (assoclen + cryptlen) - cryptlen */
388 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
389 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
390
391 /* read assoc before reading payload */
392 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
393 KEY_VLF);
394
395 /* Prepare to read and write cryptlen bytes */
396 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
397 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
398
399 /*
400 * MOVE_LEN opcode is not available in all SEC HW revisions,
401 * thus need to do some magic, i.e. self-patch the descriptor
402 * buffer.
403 */
404 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
405 MOVE_DEST_MATH2 |
406 (0x6 << MOVE_LEN_SHIFT));
407 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
408 MOVE_DEST_DESCBUF |
409 MOVE_WAITCOMP |
410 (0x8 << MOVE_LEN_SHIFT));
411
412 /* Read and write cryptlen bytes */
413 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
414
415 /*
416 * Insert a NOP here, since we need at least 4 instructions between
417 * code patching the descriptor buffer and the location being patched.
418 */
419 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
420 set_jump_tgt_here(desc, jump_cmd);
421
422 set_move_tgt_here(desc, read_move_cmd);
423 set_move_tgt_here(desc, write_move_cmd);
424 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
425 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
426 MOVE_AUX_LS);
427 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
428
429 /* Load ICV */
430 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
431 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
432
433 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
434 desc_bytes(desc),
435 DMA_TO_DEVICE);
436 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
437 dev_err(jrdev, "unable to map shared descriptor\n");
438 return -ENOMEM;
439 }
440#ifdef DEBUG
441 print_hex_dump(KERN_ERR,
442 "aead null dec shdesc@"__stringify(__LINE__)": ",
443 DUMP_PREFIX_ADDRESS, 16, 4, desc,
444 desc_bytes(desc), 1);
445#endif
446
447 return 0;
448}
449
1acebad3
YK		 450static int aead_set_sh_desc(struct crypto_aead *aead)
451{
452 struct aead_tfm *tfm = &aead->base.crt_aead;
453 struct caam_ctx *ctx = crypto_aead_ctx(aead);
daebc465
CV		 454 struct crypto_tfm *ctfm = crypto_aead_tfm(aead);
455 const char *alg_name = crypto_tfm_alg_name(ctfm);
1acebad3 456 struct device *jrdev = ctx->jrdev;
daebc465 457 bool keys_fit_inline;
1acebad3 458 u32 geniv, moveiv;
daebc465 459 u32 ctx1_iv_off = 0;
1acebad3 460 u32 *desc;
daebc465
CV		 461 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
462 OP_ALG_AAI_CTR_MOD128);
463 const bool is_rfc3686 = (ctr_mode &&
464 (strstr(alg_name, "rfc3686") != NULL));
1acebad3 465
ae4a825f 466 if (!ctx->authsize)
1acebad3
YK		 467 return 0;
468
ae4a825f
HG		 469 /* NULL encryption / decryption */
470 if (!ctx->enckeylen)
471 return aead_null_set_sh_desc(aead);
472
daebc465
CV		 473 /*
474 * AES-CTR needs to load IV in CONTEXT1 reg
475 * at an offset of 128bits (16bytes)
476 * CONTEXT1[255:128] = IV
477 */
478 if (ctr_mode)
479 ctx1_iv_off = 16;
480
481 /*
482 * RFC3686 specific:
483 * CONTEXT1[255:128] = {NONCE, IV, COUNTER}
484 */
485 if (is_rfc3686)
486 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
487
1acebad3
YK		 488 /*
489 * Job Descriptor and Shared Descriptors
490 * must all fit into the 64-word Descriptor h/w Buffer
491 */
daebc465 492 keys_fit_inline = false;
1acebad3 493 if (DESC_AEAD_ENC_LEN + DESC_JOB_IO_LEN +
daebc465
CV		 494 ctx->split_key_pad_len + ctx->enckeylen +
495 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
1acebad3 496 CAAM_DESC_BYTES_MAX)
2af8f4a2 497 keys_fit_inline = true;
1acebad3
YK		 498
499 /* aead_encrypt shared descriptor */
500 desc = ctx->sh_desc_enc;
501
daebc465
CV		 502 /* Note: Context registers are saved. */
503 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
1acebad3
YK		 504
505 /* Class 2 operation */
506 append_operation(desc, ctx->class2_alg_type |
507 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
508
509 /* cryptlen = seqoutlen - authsize */
510 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
511
512 /* assoclen + cryptlen = seqinlen - ivsize */
513 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
514
4464a7d4 515 /* assoclen = (assoclen + cryptlen) - cryptlen */
1acebad3
YK		 516 append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
517
518 /* read assoc before reading payload */
519 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
520 KEY_VLF);
daebc465
CV		 521 aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off);
522
523 /* Load Counter into CONTEXT1 reg */
524 if (is_rfc3686)
525 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
526 LDST_CLASS_1_CCB |
527 LDST_SRCDST_BYTE_CONTEXT |
528 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
529 LDST_OFFSET_SHIFT));
1acebad3
YK		 530
531 /* Class 1 operation */
532 append_operation(desc, ctx->class1_alg_type |
533 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
534
535 /* Read and write cryptlen bytes */
536 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
537 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
538 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
539
540 /* Write ICV */
541 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
542 LDST_SRCDST_BYTE_CONTEXT);
543
544 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
545 desc_bytes(desc),
546 DMA_TO_DEVICE);
547 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
548 dev_err(jrdev, "unable to map shared descriptor\n");
549 return -ENOMEM;
550 }
551#ifdef DEBUG
514df281 552 print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ",
1acebad3
YK		 553 DUMP_PREFIX_ADDRESS, 16, 4, desc,
554 desc_bytes(desc), 1);
555#endif
556
557 /*
558 * Job Descriptor and Shared Descriptors
559 * must all fit into the 64-word Descriptor h/w Buffer
560 */
80cd88f2 561 keys_fit_inline = false;
1acebad3 562 if (DESC_AEAD_DEC_LEN + DESC_JOB_IO_LEN +
daebc465
CV		 563 ctx->split_key_pad_len + ctx->enckeylen +
564 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
1acebad3 565 CAAM_DESC_BYTES_MAX)
2af8f4a2 566 keys_fit_inline = true;
1acebad3 567
1acebad3 568 /* aead_decrypt shared descriptor */
4464a7d4 569 desc = ctx->sh_desc_dec;
1acebad3 570
daebc465
CV		 571 /* Note: Context registers are saved. */
572 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
1acebad3
YK		 573
574 /* Class 2 operation */
575 append_operation(desc, ctx->class2_alg_type |
576 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
577
4464a7d4 578 /* assoclen + cryptlen = seqinlen - ivsize - authsize */
1acebad3 579 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
ae4a825f 580 ctx->authsize + tfm->ivsize);
1acebad3
YK		 581 /* assoclen = (assoclen + cryptlen) - cryptlen */
582 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
583 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
584
585 /* read assoc before reading payload */
586 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
587 KEY_VLF);
588
daebc465 589 aead_append_ld_iv(desc, tfm->ivsize, ctx1_iv_off);
1acebad3 590
daebc465
CV		 591 /* Load Counter into CONTEXT1 reg */
592 if (is_rfc3686)
593 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
594 LDST_CLASS_1_CCB |
595 LDST_SRCDST_BYTE_CONTEXT |
596 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
597 LDST_OFFSET_SHIFT));
598
599 /* Choose operation */
600 if (ctr_mode)
601 append_operation(desc, ctx->class1_alg_type |
602 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
603 else
604 append_dec_op1(desc, ctx->class1_alg_type);
1acebad3
YK		 605
606 /* Read and write cryptlen bytes */
607 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
608 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
609 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
610
611 /* Load ICV */
612 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 |
613 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1acebad3
YK		 614
615 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
616 desc_bytes(desc),
617 DMA_TO_DEVICE);
618 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
619 dev_err(jrdev, "unable to map shared descriptor\n");
620 return -ENOMEM;
621 }
622#ifdef DEBUG
514df281 623 print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ",
1acebad3
YK		 624 DUMP_PREFIX_ADDRESS, 16, 4, desc,
625 desc_bytes(desc), 1);
626#endif
627
628 /*
629 * Job Descriptor and Shared Descriptors
630 * must all fit into the 64-word Descriptor h/w Buffer
631 */
80cd88f2 632 keys_fit_inline = false;
1acebad3 633 if (DESC_AEAD_GIVENC_LEN + DESC_JOB_IO_LEN +
daebc465
CV		 634 ctx->split_key_pad_len + ctx->enckeylen +
635 (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <=
1acebad3 636 CAAM_DESC_BYTES_MAX)
2af8f4a2 637 keys_fit_inline = true;
1acebad3
YK		 638
639 /* aead_givencrypt shared descriptor */
640 desc = ctx->sh_desc_givenc;
641
daebc465
CV		 642 /* Note: Context registers are saved. */
643 init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686);
1acebad3
YK		 644
645 /* Generate IV */
646 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
647 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
648 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
649 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
650 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
651 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
daebc465
CV		 652 append_move(desc, MOVE_WAITCOMP |
653 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
654 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
655 (tfm->ivsize << MOVE_LEN_SHIFT));
1acebad3
YK		 656 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
657
658 /* Copy IV to class 1 context */
daebc465
CV		 659 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
660 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
661 (tfm->ivsize << MOVE_LEN_SHIFT));
1acebad3
YK		 662
663 /* Return to encryption */
664 append_operation(desc, ctx->class2_alg_type |
665 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
666
667 /* ivsize + cryptlen = seqoutlen - authsize */
668 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
669
670 /* assoclen = seqinlen - (ivsize + cryptlen) */
671 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
672
673 /* read assoc before reading payload */
674 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
675 KEY_VLF);
676
daebc465 677 /* Copy iv from outfifo to class 2 fifo */
1acebad3
YK		 678 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
679 NFIFOENTRY_DTYPE_MSG | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
680 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
681 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
682 append_load_imm_u32(desc, tfm->ivsize, LDST_CLASS_2_CCB |
683 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
684
daebc465
CV		 685 /* Load Counter into CONTEXT1 reg */
686 if (is_rfc3686)
687 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
688 LDST_CLASS_1_CCB |
689 LDST_SRCDST_BYTE_CONTEXT |
690 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
691 LDST_OFFSET_SHIFT));
692
1acebad3
YK		 693 /* Class 1 operation */
694 append_operation(desc, ctx->class1_alg_type |
695 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
696
697 /* Will write ivsize + cryptlen */
698 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
699
700 /* Not need to reload iv */
701 append_seq_fifo_load(desc, tfm->ivsize,
702 FIFOLD_CLASS_SKIP);
703
704 /* Will read cryptlen */
705 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
706 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
707
708 /* Write ICV */
709 append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB |
710 LDST_SRCDST_BYTE_CONTEXT);
711
712 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
713 desc_bytes(desc),
714 DMA_TO_DEVICE);
715 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
716 dev_err(jrdev, "unable to map shared descriptor\n");
717 return -ENOMEM;
718 }
719#ifdef DEBUG
514df281 720 print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ",
1acebad3
YK		 721 DUMP_PREFIX_ADDRESS, 16, 4, desc,
722 desc_bytes(desc), 1);
723#endif
724
725 return 0;
726}
727
0e479300 728static int aead_setauthsize(struct crypto_aead *authenc,
8e8ec596
KP		 729 unsigned int authsize)
730{
731 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
732
733 ctx->authsize = authsize;
1acebad3 734 aead_set_sh_desc(authenc);
8e8ec596
KP		 735
736 return 0;
737}
738
3ef8d945
TA		 739static int gcm_set_sh_desc(struct crypto_aead *aead)
740{
741 struct aead_tfm *tfm = &aead->base.crt_aead;
742 struct caam_ctx *ctx = crypto_aead_ctx(aead);
743 struct device *jrdev = ctx->jrdev;
744 bool keys_fit_inline = false;
745 u32 *key_jump_cmd, *zero_payload_jump_cmd,
746 *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2;
747 u32 *desc;
748
749 if (!ctx->enckeylen || !ctx->authsize)
750 return 0;
751
752 /*
753 * AES GCM encrypt shared descriptor
754 * Job Descriptor and Shared Descriptor
755 * must fit into the 64-word Descriptor h/w Buffer
756 */
757 if (DESC_GCM_ENC_LEN + DESC_JOB_IO_LEN +
758 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
759 keys_fit_inline = true;
760
761 desc = ctx->sh_desc_enc;
762
763 init_sh_desc(desc, HDR_SHARE_SERIAL);
764
765 /* skip key loading if they are loaded due to sharing */
766 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
767 JUMP_COND_SHRD | JUMP_COND_SELF);
768 if (keys_fit_inline)
769 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
770 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
771 else
772 append_key(desc, ctx->key_dma, ctx->enckeylen,
773 CLASS_1 | KEY_DEST_CLASS_REG);
774 set_jump_tgt_here(desc, key_jump_cmd);
775
776 /* class 1 operation */
777 append_operation(desc, ctx->class1_alg_type |
778 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
779
780 /* cryptlen = seqoutlen - authsize */
781 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
782
783 /* assoclen + cryptlen = seqinlen - ivsize */
784 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
785
786 /* assoclen = (assoclen + cryptlen) - cryptlen */
787 append_math_sub(desc, REG1, REG2, REG3, CAAM_CMD_SZ);
788
789 /* if cryptlen is ZERO jump to zero-payload commands */
790 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
791 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
792 JUMP_COND_MATH_Z);
793 /* read IV */
794 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
795 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
796
797 /* if assoclen is ZERO, skip reading the assoc data */
798 append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
799 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
800 JUMP_COND_MATH_Z);
801
802 /* read assoc data */
803 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
804 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
805 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
806
807 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
808
809 /* write encrypted data */
810 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
811
812 /* read payload data */
813 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
814 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
815
816 /* jump the zero-payload commands */
817 append_jump(desc, JUMP_TEST_ALL | 7);
818
819 /* zero-payload commands */
820 set_jump_tgt_here(desc, zero_payload_jump_cmd);
821
822 /* if assoclen is ZERO, jump to IV reading - is the only input data */
823 append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
824 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
825 JUMP_COND_MATH_Z);
826 /* read IV */
827 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
828 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
829
830 /* read assoc data */
831 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
832 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
833
834 /* jump to ICV writing */
835 append_jump(desc, JUMP_TEST_ALL | 2);
836
837 /* read IV - is the only input data */
838 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
839 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
840 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
841 FIFOLD_TYPE_LAST1);
842
843 /* write ICV */
844 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
845 LDST_SRCDST_BYTE_CONTEXT);
846
847 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
848 desc_bytes(desc),
849 DMA_TO_DEVICE);
850 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
851 dev_err(jrdev, "unable to map shared descriptor\n");
852 return -ENOMEM;
853 }
854#ifdef DEBUG
855 print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ",
856 DUMP_PREFIX_ADDRESS, 16, 4, desc,
857 desc_bytes(desc), 1);
858#endif
859
860 /*
861 * Job Descriptor and Shared Descriptors
862 * must all fit into the 64-word Descriptor h/w Buffer
863 */
864 keys_fit_inline = false;
865 if (DESC_GCM_DEC_LEN + DESC_JOB_IO_LEN +
866 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
867 keys_fit_inline = true;
868
869 desc = ctx->sh_desc_dec;
870
871 init_sh_desc(desc, HDR_SHARE_SERIAL);
872
873 /* skip key loading if they are loaded due to sharing */
874 key_jump_cmd = append_jump(desc, JUMP_JSL |
875 JUMP_TEST_ALL | JUMP_COND_SHRD |
876 JUMP_COND_SELF);
877 if (keys_fit_inline)
878 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
879 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
880 else
881 append_key(desc, ctx->key_dma, ctx->enckeylen,
882 CLASS_1 | KEY_DEST_CLASS_REG);
883 set_jump_tgt_here(desc, key_jump_cmd);
884
885 /* class 1 operation */
886 append_operation(desc, ctx->class1_alg_type |
887 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
888
889 /* assoclen + cryptlen = seqinlen - ivsize - icvsize */
890 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
891 ctx->authsize + tfm->ivsize);
892
893 /* assoclen = (assoclen + cryptlen) - cryptlen */
894 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
895 append_math_sub(desc, REG1, REG3, REG2, CAAM_CMD_SZ);
896
897 /* read IV */
898 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
899 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
900
901 /* jump to zero-payload command if cryptlen is zero */
902 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
903 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
904 JUMP_COND_MATH_Z);
905
906 append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
907 /* if asoclen is ZERO, skip reading assoc data */
908 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
909 JUMP_COND_MATH_Z);
910 /* read assoc data */
911 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
912 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
913 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
914
915 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
916
917 /* store encrypted data */
918 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
919
920 /* read payload data */
921 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
922 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
923
924 /* jump the zero-payload commands */
925 append_jump(desc, JUMP_TEST_ALL | 4);
926
927 /* zero-payload command */
928 set_jump_tgt_here(desc, zero_payload_jump_cmd);
929
930 /* if assoclen is ZERO, jump to ICV reading */
931 append_math_add(desc, VARSEQINLEN, ZERO, REG1, CAAM_CMD_SZ);
932 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
933 JUMP_COND_MATH_Z);
934 /* read assoc data */
935 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
936 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
937 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
938
939 /* read ICV */
940 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
941 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
942
943 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
944 desc_bytes(desc),
945 DMA_TO_DEVICE);
946 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
947 dev_err(jrdev, "unable to map shared descriptor\n");
948 return -ENOMEM;
949 }
950#ifdef DEBUG
951 print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ",
952 DUMP_PREFIX_ADDRESS, 16, 4, desc,
953 desc_bytes(desc), 1);
954#endif
955
956 return 0;
957}
958
959static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize)
960{
961 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
962
963 ctx->authsize = authsize;
964 gcm_set_sh_desc(authenc);
965
966 return 0;
967}
968
bac68f2c
TA		 969static int rfc4106_set_sh_desc(struct crypto_aead *aead)
970{
971 struct aead_tfm *tfm = &aead->base.crt_aead;
972 struct caam_ctx *ctx = crypto_aead_ctx(aead);
973 struct device *jrdev = ctx->jrdev;
974 bool keys_fit_inline = false;
975 u32 *key_jump_cmd, *move_cmd, *write_iv_cmd;
976 u32 *desc;
977 u32 geniv;
978
979 if (!ctx->enckeylen || !ctx->authsize)
980 return 0;
981
982 /*
983 * RFC4106 encrypt shared descriptor
984 * Job Descriptor and Shared Descriptor
985 * must fit into the 64-word Descriptor h/w Buffer
986 */
987 if (DESC_RFC4106_ENC_LEN + DESC_JOB_IO_LEN +
988 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
989 keys_fit_inline = true;
990
991 desc = ctx->sh_desc_enc;
992
993 init_sh_desc(desc, HDR_SHARE_SERIAL);
994
995 /* Skip key loading if it is loaded due to sharing */
996 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
997 JUMP_COND_SHRD);
998 if (keys_fit_inline)
999 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1000 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1001 else
1002 append_key(desc, ctx->key_dma, ctx->enckeylen,
1003 CLASS_1 | KEY_DEST_CLASS_REG);
1004 set_jump_tgt_here(desc, key_jump_cmd);
1005
1006 /* Class 1 operation */
1007 append_operation(desc, ctx->class1_alg_type |
1008 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1009
1010 /* cryptlen = seqoutlen - authsize */
1011 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1012 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1013
1014 /* assoclen + cryptlen = seqinlen - ivsize */
1015 append_math_sub_imm_u32(desc, REG2, SEQINLEN, IMM, tfm->ivsize);
1016
1017 /* assoclen = (assoclen + cryptlen) - cryptlen */
1018 append_math_sub(desc, VARSEQINLEN, REG2, REG3, CAAM_CMD_SZ);
1019
1020 /* Read Salt */
1021 append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen),
1022 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV);
1023 /* Read AES-GCM-ESP IV */
1024 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
1025 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1026
1027 /* Read assoc data */
1028 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1029 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1030
1031 /* Will read cryptlen bytes */
1032 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1033
1034 /* Write encrypted data */
1035 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1036
1037 /* Read payload data */
1038 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1039 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1040
1041 /* Write ICV */
1042 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1043 LDST_SRCDST_BYTE_CONTEXT);
1044
1045 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1046 desc_bytes(desc),
1047 DMA_TO_DEVICE);
1048 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1049 dev_err(jrdev, "unable to map shared descriptor\n");
1050 return -ENOMEM;
1051 }
1052#ifdef DEBUG
1053 print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ",
1054 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1055 desc_bytes(desc), 1);
1056#endif
1057
1058 /*
1059 * Job Descriptor and Shared Descriptors
1060 * must all fit into the 64-word Descriptor h/w Buffer
1061 */
1062 keys_fit_inline = false;
1063 if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN +
1064 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1065 keys_fit_inline = true;
1066
1067 desc = ctx->sh_desc_dec;
1068
1069 init_sh_desc(desc, HDR_SHARE_SERIAL);
1070
1071 /* Skip key loading if it is loaded due to sharing */
1072 key_jump_cmd = append_jump(desc, JUMP_JSL |
1073 JUMP_TEST_ALL | JUMP_COND_SHRD);
1074 if (keys_fit_inline)
1075 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1076 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1077 else
1078 append_key(desc, ctx->key_dma, ctx->enckeylen,
1079 CLASS_1 | KEY_DEST_CLASS_REG);
1080 set_jump_tgt_here(desc, key_jump_cmd);
1081
1082 /* Class 1 operation */
1083 append_operation(desc, ctx->class1_alg_type |
1084 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1085
1086 /* assoclen + cryptlen = seqinlen - ivsize - icvsize */
1087 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM,
1088 ctx->authsize + tfm->ivsize);
1089
1090 /* assoclen = (assoclen + cryptlen) - cryptlen */
1091 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1092 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
1093
1094 /* Will write cryptlen bytes */
1095 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1096
1097 /* Read Salt */
1098 append_fifo_load_as_imm(desc, (void *)(ctx->key + ctx->enckeylen),
1099 4, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV);
1100 /* Read AES-GCM-ESP IV */
1101 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_CLASS1 |
1102 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1103
1104 /* Read assoc data */
1105 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1106 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1107
1108 /* Will read cryptlen bytes */
1109 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
1110
1111 /* Store payload data */
1112 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1113
1114 /* Read encrypted data */
1115 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1116 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1117
1118 /* Read ICV */
1119 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1120 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1121
1122 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1123 desc_bytes(desc),
1124 DMA_TO_DEVICE);
1125 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1126 dev_err(jrdev, "unable to map shared descriptor\n");
1127 return -ENOMEM;
1128 }
1129#ifdef DEBUG
1130 print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ",
1131 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1132 desc_bytes(desc), 1);
1133#endif
1134
1135 /*
1136 * Job Descriptor and Shared Descriptors
1137 * must all fit into the 64-word Descriptor h/w Buffer
1138 */
1139 keys_fit_inline = false;
1140 if (DESC_RFC4106_GIVENC_LEN + DESC_JOB_IO_LEN +
1141 ctx->split_key_pad_len + ctx->enckeylen <=
1142 CAAM_DESC_BYTES_MAX)
1143 keys_fit_inline = true;
1144
1145 /* rfc4106_givencrypt shared descriptor */
1146 desc = ctx->sh_desc_givenc;
1147
1148 init_sh_desc(desc, HDR_SHARE_SERIAL);
1149
1150 /* Skip key loading if it is loaded due to sharing */
1151 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1152 JUMP_COND_SHRD);
1153 if (keys_fit_inline)
1154 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1155 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1156 else
1157 append_key(desc, ctx->key_dma, ctx->enckeylen,
1158 CLASS_1 | KEY_DEST_CLASS_REG);
1159 set_jump_tgt_here(desc, key_jump_cmd);
1160
1161 /* Generate IV */
1162 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1163 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
1164 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
1165 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1166 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1167 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1168 move_cmd = append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_DESCBUF |
1169 (tfm->ivsize << MOVE_LEN_SHIFT));
1170 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1171
1172 /* Copy generated IV to OFIFO */
1173 write_iv_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_OUTFIFO |
1174 (tfm->ivsize << MOVE_LEN_SHIFT));
1175
1176 /* Class 1 operation */
1177 append_operation(desc, ctx->class1_alg_type |
1178 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1179
1180 /* ivsize + cryptlen = seqoutlen - authsize */
1181 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1182
1183 /* assoclen = seqinlen - (ivsize + cryptlen) */
1184 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
1185
1186 /* Will write ivsize + cryptlen */
1187 append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ);
1188
1189 /* Read Salt and generated IV */
1190 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_IV |
1191 FIFOLD_TYPE_FLUSH1 | IMMEDIATE | 12);
1192 /* Append Salt */
1193 append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1194 set_move_tgt_here(desc, move_cmd);
1195 set_move_tgt_here(desc, write_iv_cmd);
1196 /* Blank commands. Will be overwritten by generated IV. */
1197 append_cmd(desc, 0x00000000);
1198 append_cmd(desc, 0x00000000);
1199 /* End of blank commands */
1200
1201 /* No need to reload iv */
1202 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP);
1203
1204 /* Read assoc data */
1205 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1206 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1207
1208 /* Will read cryptlen */
1209 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1210
1211 /* Store generated IV and encrypted data */
1212 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1213
1214 /* Read payload data */
1215 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1216 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1217
1218 /* Write ICV */
1219 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1220 LDST_SRCDST_BYTE_CONTEXT);
1221
1222 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
1223 desc_bytes(desc),
1224 DMA_TO_DEVICE);
1225 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
1226 dev_err(jrdev, "unable to map shared descriptor\n");
1227 return -ENOMEM;
1228 }
1229#ifdef DEBUG
1230 print_hex_dump(KERN_ERR,
1231 "rfc4106 givenc shdesc@"__stringify(__LINE__)": ",
1232 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1233 desc_bytes(desc), 1);
1234#endif
1235
1236 return 0;
1237}
1238
1239static int rfc4106_setauthsize(struct crypto_aead *authenc,
1240 unsigned int authsize)
1241{
1242 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1243
1244 ctx->authsize = authsize;
1245 rfc4106_set_sh_desc(authenc);
1246
1247 return 0;
1248}
1249
5d0429a3
TA		 1250static int rfc4543_set_sh_desc(struct crypto_aead *aead)
1251{
1252 struct aead_tfm *tfm = &aead->base.crt_aead;
1253 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1254 struct device *jrdev = ctx->jrdev;
1255 bool keys_fit_inline = false;
1256 u32 *key_jump_cmd, *write_iv_cmd, *write_aad_cmd;
1257 u32 *read_move_cmd, *write_move_cmd;
1258 u32 *desc;
1259 u32 geniv;
1260
1261 if (!ctx->enckeylen || !ctx->authsize)
1262 return 0;
1263
1264 /*
1265 * RFC4543 encrypt shared descriptor
1266 * Job Descriptor and Shared Descriptor
1267 * must fit into the 64-word Descriptor h/w Buffer
1268 */
1269 if (DESC_RFC4543_ENC_LEN + DESC_JOB_IO_LEN +
1270 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1271 keys_fit_inline = true;
1272
1273 desc = ctx->sh_desc_enc;
1274
1275 init_sh_desc(desc, HDR_SHARE_SERIAL);
1276
1277 /* Skip key loading if it is loaded due to sharing */
1278 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1279 JUMP_COND_SHRD);
1280 if (keys_fit_inline)
1281 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1282 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1283 else
1284 append_key(desc, ctx->key_dma, ctx->enckeylen,
1285 CLASS_1 | KEY_DEST_CLASS_REG);
1286 set_jump_tgt_here(desc, key_jump_cmd);
1287
1288 /* Class 1 operation */
1289 append_operation(desc, ctx->class1_alg_type |
1290 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1291
1292 /* Load AES-GMAC ESP IV into Math1 register */
1293 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 |
1294 LDST_CLASS_DECO | tfm->ivsize);
1295
1296 /* Wait the DMA transaction to finish */
1297 append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM |
1298 (1 << JUMP_OFFSET_SHIFT));
1299
1300 /* Overwrite blank immediate AES-GMAC ESP IV data */
1301 write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1302 (tfm->ivsize << MOVE_LEN_SHIFT));
1303
1304 /* Overwrite blank immediate AAD data */
1305 write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1306 (tfm->ivsize << MOVE_LEN_SHIFT));
1307
1308 /* cryptlen = seqoutlen - authsize */
1309 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1310
1311 /* assoclen = (seqinlen - ivsize) - cryptlen */
1312 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
1313
1314 /* Read Salt and AES-GMAC ESP IV */
1315 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1316 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize));
1317 /* Append Salt */
1318 append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1319 set_move_tgt_here(desc, write_iv_cmd);
1320 /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1321 append_cmd(desc, 0x00000000);
1322 append_cmd(desc, 0x00000000);
1323 /* End of blank commands */
1324
1325 /* Read assoc data */
1326 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1327 FIFOLD_TYPE_AAD);
1328
1329 /* Will read cryptlen bytes */
1330 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1331
1332 /* Will write cryptlen bytes */
1333 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1334
1335 /*
1336 * MOVE_LEN opcode is not available in all SEC HW revisions,
1337 * thus need to do some magic, i.e. self-patch the descriptor
1338 * buffer.
1339 */
1340 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1341 (0x6 << MOVE_LEN_SHIFT));
1342 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1343 (0x8 << MOVE_LEN_SHIFT));
1344
1345 /* Authenticate AES-GMAC ESP IV */
1346 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1347 FIFOLD_TYPE_AAD | tfm->ivsize);
1348 set_move_tgt_here(desc, write_aad_cmd);
1349 /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1350 append_cmd(desc, 0x00000000);
1351 append_cmd(desc, 0x00000000);
1352 /* End of blank commands */
1353
1354 /* Read and write cryptlen bytes */
1355 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1356
1357 set_move_tgt_here(desc, read_move_cmd);
1358 set_move_tgt_here(desc, write_move_cmd);
1359 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1360 /* Move payload data to OFIFO */
1361 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1362
1363 /* Write ICV */
1364 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1365 LDST_SRCDST_BYTE_CONTEXT);
1366
1367 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1368 desc_bytes(desc),
1369 DMA_TO_DEVICE);
1370 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1371 dev_err(jrdev, "unable to map shared descriptor\n");
1372 return -ENOMEM;
1373 }
1374#ifdef DEBUG
1375 print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ",
1376 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1377 desc_bytes(desc), 1);
1378#endif
1379
1380 /*
1381 * Job Descriptor and Shared Descriptors
1382 * must all fit into the 64-word Descriptor h/w Buffer
1383 */
1384 keys_fit_inline = false;
1385 if (DESC_RFC4543_DEC_LEN + DESC_JOB_IO_LEN +
1386 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1387 keys_fit_inline = true;
1388
1389 desc = ctx->sh_desc_dec;
1390
1391 init_sh_desc(desc, HDR_SHARE_SERIAL);
1392
1393 /* Skip key loading if it is loaded due to sharing */
1394 key_jump_cmd = append_jump(desc, JUMP_JSL |
1395 JUMP_TEST_ALL | JUMP_COND_SHRD);
1396 if (keys_fit_inline)
1397 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1398 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1399 else
1400 append_key(desc, ctx->key_dma, ctx->enckeylen,
1401 CLASS_1 | KEY_DEST_CLASS_REG);
1402 set_jump_tgt_here(desc, key_jump_cmd);
1403
1404 /* Class 1 operation */
1405 append_operation(desc, ctx->class1_alg_type |
1406 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1407
1408 /* Load AES-GMAC ESP IV into Math1 register */
1409 append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_DECO_MATH1 |
1410 LDST_CLASS_DECO | tfm->ivsize);
1411
1412 /* Wait the DMA transaction to finish */
1413 append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM |
1414 (1 << JUMP_OFFSET_SHIFT));
1415
1416 /* assoclen + cryptlen = (seqinlen - ivsize) - icvsize */
1417 append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, ctx->authsize);
1418
1419 /* Overwrite blank immediate AES-GMAC ESP IV data */
1420 write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1421 (tfm->ivsize << MOVE_LEN_SHIFT));
1422
1423 /* Overwrite blank immediate AAD data */
1424 write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1425 (tfm->ivsize << MOVE_LEN_SHIFT));
1426
1427 /* assoclen = (assoclen + cryptlen) - cryptlen */
1428 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1429 append_math_sub(desc, VARSEQINLEN, REG3, REG2, CAAM_CMD_SZ);
1430
1431 /*
1432 * MOVE_LEN opcode is not available in all SEC HW revisions,
1433 * thus need to do some magic, i.e. self-patch the descriptor
1434 * buffer.
1435 */
1436 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1437 (0x6 << MOVE_LEN_SHIFT));
1438 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1439 (0x8 << MOVE_LEN_SHIFT));
1440
1441 /* Read Salt and AES-GMAC ESP IV */
1442 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1443 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize));
1444 /* Append Salt */
1445 append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1446 set_move_tgt_here(desc, write_iv_cmd);
1447 /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1448 append_cmd(desc, 0x00000000);
1449 append_cmd(desc, 0x00000000);
1450 /* End of blank commands */
1451
1452 /* Read assoc data */
1453 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1454 FIFOLD_TYPE_AAD);
1455
1456 /* Will read cryptlen bytes */
1457 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
1458
1459 /* Will write cryptlen bytes */
1460 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
1461
1462 /* Authenticate AES-GMAC ESP IV */
1463 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1464 FIFOLD_TYPE_AAD | tfm->ivsize);
1465 set_move_tgt_here(desc, write_aad_cmd);
1466 /* Blank commands. Will be overwritten by AES-GMAC ESP IV. */
1467 append_cmd(desc, 0x00000000);
1468 append_cmd(desc, 0x00000000);
1469 /* End of blank commands */
1470
1471 /* Store payload data */
1472 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1473
1474 /* In-snoop cryptlen data */
1475 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1476 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1477
1478 set_move_tgt_here(desc, read_move_cmd);
1479 set_move_tgt_here(desc, write_move_cmd);
1480 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1481 /* Move payload data to OFIFO */
1482 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1483 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1484
1485 /* Read ICV */
1486 append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 |
1487 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1488
1489 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1490 desc_bytes(desc),
1491 DMA_TO_DEVICE);
1492 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
1493 dev_err(jrdev, "unable to map shared descriptor\n");
1494 return -ENOMEM;
1495 }
1496#ifdef DEBUG
1497 print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ",
1498 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1499 desc_bytes(desc), 1);
1500#endif
1501
1502 /*
1503 * Job Descriptor and Shared Descriptors
1504 * must all fit into the 64-word Descriptor h/w Buffer
1505 */
1506 keys_fit_inline = false;
1507 if (DESC_RFC4543_GIVENC_LEN + DESC_JOB_IO_LEN +
1508 ctx->enckeylen <= CAAM_DESC_BYTES_MAX)
1509 keys_fit_inline = true;
1510
1511 /* rfc4543_givencrypt shared descriptor */
1512 desc = ctx->sh_desc_givenc;
1513
1514 init_sh_desc(desc, HDR_SHARE_SERIAL);
1515
1516 /* Skip key loading if it is loaded due to sharing */
1517 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1518 JUMP_COND_SHRD);
1519 if (keys_fit_inline)
1520 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1521 ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
1522 else
1523 append_key(desc, ctx->key_dma, ctx->enckeylen,
1524 CLASS_1 | KEY_DEST_CLASS_REG);
1525 set_jump_tgt_here(desc, key_jump_cmd);
1526
1527 /* Generate IV */
1528 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1529 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
1530 NFIFOENTRY_PTYPE_RND | (tfm->ivsize << NFIFOENTRY_DLEN_SHIFT);
1531 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1532 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1533 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1534 /* Move generated IV to Math1 register */
1535 append_move(desc, MOVE_SRC_INFIFO | MOVE_DEST_MATH1 |
1536 (tfm->ivsize << MOVE_LEN_SHIFT));
1537 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1538
1539 /* Overwrite blank immediate AES-GMAC IV data */
1540 write_iv_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1541 (tfm->ivsize << MOVE_LEN_SHIFT));
1542
1543 /* Overwrite blank immediate AAD data */
1544 write_aad_cmd = append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_DESCBUF |
1545 (tfm->ivsize << MOVE_LEN_SHIFT));
1546
1547 /* Copy generated IV to OFIFO */
1548 append_move(desc, MOVE_SRC_MATH1 | MOVE_DEST_OUTFIFO |
1549 (tfm->ivsize << MOVE_LEN_SHIFT));
1550
1551 /* Class 1 operation */
1552 append_operation(desc, ctx->class1_alg_type |
1553 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1554
1555 /* ivsize + cryptlen = seqoutlen - authsize */
1556 append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize);
1557
1558 /* assoclen = seqinlen - (ivsize + cryptlen) */
1559 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
1560
1561 /* Will write ivsize + cryptlen */
1562 append_math_add(desc, VARSEQOUTLEN, REG3, REG0, CAAM_CMD_SZ);
1563
1564 /*
1565 * MOVE_LEN opcode is not available in all SEC HW revisions,
1566 * thus need to do some magic, i.e. self-patch the descriptor
1567 * buffer.
1568 */
1569 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1570 (0x6 << MOVE_LEN_SHIFT));
1571 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1572 (0x8 << MOVE_LEN_SHIFT));
1573
1574 /* Read Salt and AES-GMAC generated IV */
1575 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1576 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | (4 + tfm->ivsize));
1577 /* Append Salt */
1578 append_data(desc, (void *)(ctx->key + ctx->enckeylen), 4);
1579 set_move_tgt_here(desc, write_iv_cmd);
1580 /* Blank commands. Will be overwritten by AES-GMAC generated IV. */
1581 append_cmd(desc, 0x00000000);
1582 append_cmd(desc, 0x00000000);
1583 /* End of blank commands */
1584
1585 /* No need to reload iv */
1586 append_seq_fifo_load(desc, tfm->ivsize, FIFOLD_CLASS_SKIP);
1587
1588 /* Read assoc data */
1589 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1590 FIFOLD_TYPE_AAD);
1591
1592 /* Will read cryptlen */
1593 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1594
1595 /* Authenticate AES-GMAC IV */
1596 append_cmd(desc, CMD_FIFO_LOAD | FIFOLD_CLASS_CLASS1 | IMMEDIATE |
1597 FIFOLD_TYPE_AAD | tfm->ivsize);
1598 set_move_tgt_here(desc, write_aad_cmd);
1599 /* Blank commands. Will be overwritten by AES-GMAC IV. */
1600 append_cmd(desc, 0x00000000);
1601 append_cmd(desc, 0x00000000);
1602 /* End of blank commands */
1603
1604 /* Read and write cryptlen bytes */
1605 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1606
1607 set_move_tgt_here(desc, read_move_cmd);
1608 set_move_tgt_here(desc, write_move_cmd);
1609 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1610 /* Move payload data to OFIFO */
1611 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1612
1613 /* Write ICV */
1614 append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB |
1615 LDST_SRCDST_BYTE_CONTEXT);
1616
1617 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
1618 desc_bytes(desc),
1619 DMA_TO_DEVICE);
1620 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
1621 dev_err(jrdev, "unable to map shared descriptor\n");
1622 return -ENOMEM;
1623 }
1624#ifdef DEBUG
1625 print_hex_dump(KERN_ERR,
1626 "rfc4543 givenc shdesc@"__stringify(__LINE__)": ",
1627 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1628 desc_bytes(desc), 1);
1629#endif
1630
1631 return 0;
1632}
1633
1634static int rfc4543_setauthsize(struct crypto_aead *authenc,
1635 unsigned int authsize)
1636{
1637 struct caam_ctx *ctx = crypto_aead_ctx(authenc);
1638
1639 ctx->authsize = authsize;
1640 rfc4543_set_sh_desc(authenc);
1641
1642 return 0;
1643}
1644
4c1ec1f9
YK		 1645static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in,
1646 u32 authkeylen)
8e8ec596 1647{
4c1ec1f9
YK		 1648 return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len,
1649 ctx->split_key_pad_len, key_in, authkeylen,
1650 ctx->alg_op);
8e8ec596
KP		 1651}
1652
0e479300 1653static int aead_setkey(struct crypto_aead *aead,
8e8ec596
KP		 1654 const u8 *key, unsigned int keylen)
1655{
1656 /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */
1657 static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 };
1658 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1659 struct device *jrdev = ctx->jrdev;
4e6e0b27 1660 struct crypto_authenc_keys keys;
8e8ec596
KP		 1661 int ret = 0;
1662
4e6e0b27 1663 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
8e8ec596
KP		 1664 goto badkey;
1665
1666 /* Pick class 2 key length from algorithm submask */
1667 ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >>
1668 OP_ALG_ALGSEL_SHIFT] * 2;
1669 ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16);
1670
4e6e0b27
HG		 1671 if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE)
1672 goto badkey;
1673
8e8ec596
KP		 1674#ifdef DEBUG
1675 printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n",
4e6e0b27
HG		 1676 keys.authkeylen + keys.enckeylen, keys.enckeylen,
1677 keys.authkeylen);
8e8ec596
KP		 1678 printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n",
1679 ctx->split_key_len, ctx->split_key_pad_len);
514df281 1680 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
8e8ec596
KP		 1681 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1682#endif
8e8ec596 1683
4e6e0b27 1684 ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen);
8e8ec596 1685 if (ret) {
8e8ec596
KP		 1686 goto badkey;
1687 }
1688
1689 /* postpend encryption key to auth split key */
4e6e0b27 1690 memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen);
8e8ec596 1691
885e9e2f 1692 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len +
4e6e0b27 1693 keys.enckeylen, DMA_TO_DEVICE);
885e9e2f 1694 if (dma_mapping_error(jrdev, ctx->key_dma)) {
8e8ec596 1695 dev_err(jrdev, "unable to map key i/o memory\n");
8e8ec596
KP		 1696 return -ENOMEM;
1697 }
1698#ifdef DEBUG
514df281 1699 print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ",
8e8ec596 1700 DUMP_PREFIX_ADDRESS, 16, 4, ctx->key,
4e6e0b27 1701 ctx->split_key_pad_len + keys.enckeylen, 1);
8e8ec596
KP		 1702#endif
1703
4e6e0b27 1704 ctx->enckeylen = keys.enckeylen;
8e8ec596 1705
1acebad3 1706 ret = aead_set_sh_desc(aead);
8e8ec596 1707 if (ret) {
885e9e2f 1708 dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len +
4e6e0b27 1709 keys.enckeylen, DMA_TO_DEVICE);
8e8ec596
KP		 1710 }
1711
1712 return ret;
1713badkey:
1714 crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN);
1715 return -EINVAL;
1716}
1717
3ef8d945
TA		 1718static int gcm_setkey(struct crypto_aead *aead,
1719 const u8 *key, unsigned int keylen)
1720{
1721 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1722 struct device *jrdev = ctx->jrdev;
1723 int ret = 0;
1724
1725#ifdef DEBUG
1726 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1727 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1728#endif
1729
1730 memcpy(ctx->key, key, keylen);
1731 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1732 DMA_TO_DEVICE);
1733 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1734 dev_err(jrdev, "unable to map key i/o memory\n");
1735 return -ENOMEM;
1736 }
1737 ctx->enckeylen = keylen;
1738
1739 ret = gcm_set_sh_desc(aead);
1740 if (ret) {
1741 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1742 DMA_TO_DEVICE);
1743 }
1744
1745 return ret;
1746}
1747
bac68f2c
TA		 1748static int rfc4106_setkey(struct crypto_aead *aead,
1749 const u8 *key, unsigned int keylen)
1750{
1751 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1752 struct device *jrdev = ctx->jrdev;
1753 int ret = 0;
1754
1755 if (keylen < 4)
1756 return -EINVAL;
1757
1758#ifdef DEBUG
1759 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1760 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1761#endif
1762
1763 memcpy(ctx->key, key, keylen);
1764
1765 /*
1766 * The last four bytes of the key material are used as the salt value
1767 * in the nonce. Update the AES key length.
1768 */
1769 ctx->enckeylen = keylen - 4;
1770
1771 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1772 DMA_TO_DEVICE);
1773 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1774 dev_err(jrdev, "unable to map key i/o memory\n");
1775 return -ENOMEM;
1776 }
1777
1778 ret = rfc4106_set_sh_desc(aead);
1779 if (ret) {
1780 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1781 DMA_TO_DEVICE);
1782 }
1783
1784 return ret;
1785}
1786
5d0429a3
TA		 1787static int rfc4543_setkey(struct crypto_aead *aead,
1788 const u8 *key, unsigned int keylen)
1789{
1790 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1791 struct device *jrdev = ctx->jrdev;
1792 int ret = 0;
1793
1794 if (keylen < 4)
1795 return -EINVAL;
1796
1797#ifdef DEBUG
1798 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
1799 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1800#endif
1801
1802 memcpy(ctx->key, key, keylen);
1803
1804 /*
1805 * The last four bytes of the key material are used as the salt value
1806 * in the nonce. Update the AES key length.
1807 */
1808 ctx->enckeylen = keylen - 4;
1809
1810 ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen,
1811 DMA_TO_DEVICE);
1812 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1813 dev_err(jrdev, "unable to map key i/o memory\n");
1814 return -ENOMEM;
1815 }
1816
1817 ret = rfc4543_set_sh_desc(aead);
1818 if (ret) {
1819 dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen,
1820 DMA_TO_DEVICE);
1821 }
1822
1823 return ret;
1824}
1825
acdca31d
YK		 1826static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher,
1827 const u8 *key, unsigned int keylen)
1828{
1829 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
a5f57cff
CV		 1830 struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher;
1831 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher);
1832 const char *alg_name = crypto_tfm_alg_name(tfm);
acdca31d
YK		 1833 struct device *jrdev = ctx->jrdev;
1834 int ret = 0;
4464a7d4 1835 u32 *key_jump_cmd;
acdca31d 1836 u32 *desc;
a5f57cff 1837 u32 *nonce;
7222d1a3 1838 u32 geniv;
2b22f6c5
CV		 1839 u32 ctx1_iv_off = 0;
1840 const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) ==
1841 OP_ALG_AAI_CTR_MOD128);
a5f57cff
CV		 1842 const bool is_rfc3686 = (ctr_mode &&
1843 (strstr(alg_name, "rfc3686") != NULL));
acdca31d
YK		 1844
1845#ifdef DEBUG
514df281 1846 print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ",
acdca31d
YK		 1847 DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1);
1848#endif
2b22f6c5
CV		 1849 /*
1850 * AES-CTR needs to load IV in CONTEXT1 reg
1851 * at an offset of 128bits (16bytes)
1852 * CONTEXT1[255:128] = IV
1853 */
1854 if (ctr_mode)
1855 ctx1_iv_off = 16;
acdca31d 1856
a5f57cff
CV		 1857 /*
1858 * RFC3686 specific:
1859 * | CONTEXT1[255:128] = {NONCE, IV, COUNTER}
1860 * | *key = {KEY, NONCE}
1861 */
1862 if (is_rfc3686) {
1863 ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE;
1864 keylen -= CTR_RFC3686_NONCE_SIZE;
1865 }
1866
acdca31d
YK		 1867 memcpy(ctx->key, key, keylen);
1868 ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen,
1869 DMA_TO_DEVICE);
1870 if (dma_mapping_error(jrdev, ctx->key_dma)) {
1871 dev_err(jrdev, "unable to map key i/o memory\n");
1872 return -ENOMEM;
1873 }
1874 ctx->enckeylen = keylen;
1875
1876 /* ablkcipher_encrypt shared descriptor */
1877 desc = ctx->sh_desc_enc;
a5f57cff 1878 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
acdca31d
YK		 1879 /* Skip if already shared */
1880 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1881 JUMP_COND_SHRD);
1882
1883 /* Load class1 key only */
1884 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1885 ctx->enckeylen, CLASS_1 |
1886 KEY_DEST_CLASS_REG);
1887
a5f57cff
CV		 1888 /* Load nonce into CONTEXT1 reg */
1889 if (is_rfc3686) {
1890 nonce = (u32 *)(key + keylen);
1891 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1892 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1893 append_move(desc, MOVE_WAITCOMP |
1894 MOVE_SRC_OUTFIFO |
1895 MOVE_DEST_CLASS1CTX |
1896 (16 << MOVE_OFFSET_SHIFT) |
1897 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1898 }
1899
acdca31d
YK		 1900 set_jump_tgt_here(desc, key_jump_cmd);
1901
acdca31d 1902 /* Load iv */
a5f57cff 1903 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
2b22f6c5 1904 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
acdca31d 1905
a5f57cff
CV		 1906 /* Load counter into CONTEXT1 reg */
1907 if (is_rfc3686)
1908 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1909 LDST_CLASS_1_CCB |
1910 LDST_SRCDST_BYTE_CONTEXT |
1911 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1912 LDST_OFFSET_SHIFT));
1913
acdca31d
YK		 1914 /* Load operation */
1915 append_operation(desc, ctx->class1_alg_type |
1916 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
1917
1918 /* Perform operation */
1919 ablkcipher_append_src_dst(desc);
1920
1921 ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc,
1922 desc_bytes(desc),
1923 DMA_TO_DEVICE);
1924 if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) {
1925 dev_err(jrdev, "unable to map shared descriptor\n");
1926 return -ENOMEM;
1927 }
1928#ifdef DEBUG
514df281
AP		 1929 print_hex_dump(KERN_ERR,
1930 "ablkcipher enc shdesc@"__stringify(__LINE__)": ",
acdca31d
YK		 1931 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1932 desc_bytes(desc), 1);
1933#endif
1934 /* ablkcipher_decrypt shared descriptor */
1935 desc = ctx->sh_desc_dec;
1936
a5f57cff 1937 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
acdca31d
YK		 1938 /* Skip if already shared */
1939 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1940 JUMP_COND_SHRD);
1941
1942 /* Load class1 key only */
1943 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
1944 ctx->enckeylen, CLASS_1 |
1945 KEY_DEST_CLASS_REG);
1946
a5f57cff
CV		 1947 /* Load nonce into CONTEXT1 reg */
1948 if (is_rfc3686) {
1949 nonce = (u32 *)(key + keylen);
1950 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
1951 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1952 append_move(desc, MOVE_WAITCOMP |
1953 MOVE_SRC_OUTFIFO |
1954 MOVE_DEST_CLASS1CTX |
1955 (16 << MOVE_OFFSET_SHIFT) |
1956 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1957 }
1958
acdca31d 1959 set_jump_tgt_here(desc, key_jump_cmd);
acdca31d
YK		 1960
1961 /* load IV */
a5f57cff 1962 append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT |
2b22f6c5 1963 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
acdca31d 1964
a5f57cff
CV		 1965 /* Load counter into CONTEXT1 reg */
1966 if (is_rfc3686)
1967 append_load_imm_u32(desc, be32_to_cpu(1), LDST_IMM |
1968 LDST_CLASS_1_CCB |
1969 LDST_SRCDST_BYTE_CONTEXT |
1970 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1971 LDST_OFFSET_SHIFT));
1972
acdca31d 1973 /* Choose operation */
2b22f6c5
CV		 1974 if (ctr_mode)
1975 append_operation(desc, ctx->class1_alg_type |
1976 OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT);
1977 else
1978 append_dec_op1(desc, ctx->class1_alg_type);
acdca31d
YK		 1979
1980 /* Perform operation */
1981 ablkcipher_append_src_dst(desc);
1982
acdca31d
YK		 1983 ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc,
1984 desc_bytes(desc),
1985 DMA_TO_DEVICE);
71c65f7c 1986 if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) {
acdca31d
YK		 1987 dev_err(jrdev, "unable to map shared descriptor\n");
1988 return -ENOMEM;
1989 }
1990
1991#ifdef DEBUG
514df281
AP		 1992 print_hex_dump(KERN_ERR,
1993 "ablkcipher dec shdesc@"__stringify(__LINE__)": ",
acdca31d
YK		 1994 DUMP_PREFIX_ADDRESS, 16, 4, desc,
1995 desc_bytes(desc), 1);
1996#endif
7222d1a3
CV		 1997 /* ablkcipher_givencrypt shared descriptor */
1998 desc = ctx->sh_desc_givenc;
1999
2000 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
2001 /* Skip if already shared */
2002 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
2003 JUMP_COND_SHRD);
2004
2005 /* Load class1 key only */
2006 append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen,
2007 ctx->enckeylen, CLASS_1 |
2008 KEY_DEST_CLASS_REG);
2009
2010 /* Load Nonce into CONTEXT1 reg */
2011 if (is_rfc3686) {
2012 nonce = (u32 *)(key + keylen);
2013 append_load_imm_u32(desc, *nonce, LDST_CLASS_IND_CCB |
2014 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
2015 append_move(desc, MOVE_WAITCOMP |
2016 MOVE_SRC_OUTFIFO |
2017 MOVE_DEST_CLASS1CTX |
2018 (16 << MOVE_OFFSET_SHIFT) |
2019 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
2020 }
2021 set_jump_tgt_here(desc, key_jump_cmd);
2022
2023 /* Generate IV */
2024 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
2025 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
2026 NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT);
2027 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
2028 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
2029 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
2030 append_move(desc, MOVE_WAITCOMP |
2031 MOVE_SRC_INFIFO |
2032 MOVE_DEST_CLASS1CTX |
2033 (crt->ivsize << MOVE_LEN_SHIFT) |
2034 (ctx1_iv_off << MOVE_OFFSET_SHIFT));
2035 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
2036
2037 /* Copy generated IV to memory */
2038 append_seq_store(desc, crt->ivsize,
2039 LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
2040 (ctx1_iv_off << LDST_OFFSET_SHIFT));
2041
2042 /* Load Counter into CONTEXT1 reg */
2043 if (is_rfc3686)
2044 append_load_imm_u32(desc, (u32)1, LDST_IMM |
2045 LDST_CLASS_1_CCB |
2046 LDST_SRCDST_BYTE_CONTEXT |
2047 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
2048 LDST_OFFSET_SHIFT));
2049
2050 if (ctx1_iv_off)
2051 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
2052 (1 << JUMP_OFFSET_SHIFT));
2053
2054 /* Load operation */
2055 append_operation(desc, ctx->class1_alg_type |
2056 OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT);
2057
2058 /* Perform operation */
2059 ablkcipher_append_src_dst(desc);
2060
2061 ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc,
2062 desc_bytes(desc),
2063 DMA_TO_DEVICE);
2064 if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) {
2065 dev_err(jrdev, "unable to map shared descriptor\n");
2066 return -ENOMEM;
2067 }
2068#ifdef DEBUG
2069 print_hex_dump(KERN_ERR,
2070 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
2071 DUMP_PREFIX_ADDRESS, 16, 4, desc,
2072 desc_bytes(desc), 1);
2073#endif
acdca31d
YK		 2074
2075 return ret;
2076}
2077
8e8ec596 2078/*
1acebad3
YK		 2079 * aead_edesc - s/w-extended aead descriptor
2080 * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist
643b39b0 2081 * @assoc_chained: if source is chained
8e8ec596 2082 * @src_nents: number of segments in input scatterlist
643b39b0 2083 * @src_chained: if source is chained
8e8ec596 2084 * @dst_nents: number of segments in output scatterlist
643b39b0 2085 * @dst_chained: if destination is chained
1acebad3 2086 * @iv_dma: dma address of iv for checking continuity and link table
8e8ec596 2087 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
a299c837
YK		 2088 * @sec4_sg_bytes: length of dma mapped sec4_sg space
2089 * @sec4_sg_dma: bus physical mapped address of h/w link table
8e8ec596
KP		 2090 * @hw_desc: the h/w job descriptor followed by any referenced link tables
2091 */
0e479300 2092struct aead_edesc {
8e8ec596 2093 int assoc_nents;
643b39b0 2094 bool assoc_chained;
8e8ec596 2095 int src_nents;
643b39b0 2096 bool src_chained;
8e8ec596 2097 int dst_nents;
643b39b0 2098 bool dst_chained;
1acebad3 2099 dma_addr_t iv_dma;
a299c837
YK		 2100 int sec4_sg_bytes;
2101 dma_addr_t sec4_sg_dma;
2102 struct sec4_sg_entry *sec4_sg;
8e8ec596
KP		 2103 u32 hw_desc[0];
2104};
2105
acdca31d
YK		 2106/*
2107 * ablkcipher_edesc - s/w-extended ablkcipher descriptor
2108 * @src_nents: number of segments in input scatterlist
643b39b0 2109 * @src_chained: if source is chained
acdca31d 2110 * @dst_nents: number of segments in output scatterlist
643b39b0 2111 * @dst_chained: if destination is chained
acdca31d
YK		 2112 * @iv_dma: dma address of iv for checking continuity and link table
2113 * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE)
a299c837
YK		 2114 * @sec4_sg_bytes: length of dma mapped sec4_sg space
2115 * @sec4_sg_dma: bus physical mapped address of h/w link table
acdca31d
YK		 2116 * @hw_desc: the h/w job descriptor followed by any referenced link tables
2117 */
2118struct ablkcipher_edesc {
2119 int src_nents;
643b39b0 2120 bool src_chained;
acdca31d 2121 int dst_nents;
643b39b0 2122 bool dst_chained;
acdca31d 2123 dma_addr_t iv_dma;
a299c837
YK		 2124 int sec4_sg_bytes;
2125 dma_addr_t sec4_sg_dma;
2126 struct sec4_sg_entry *sec4_sg;
acdca31d
YK		 2127 u32 hw_desc[0];
2128};
2129
1acebad3 2130static void caam_unmap(struct device *dev, struct scatterlist *src,
643b39b0
YK		 2131 struct scatterlist *dst, int src_nents,
2132 bool src_chained, int dst_nents, bool dst_chained,
a299c837
YK		 2133 dma_addr_t iv_dma, int ivsize, dma_addr_t sec4_sg_dma,
2134 int sec4_sg_bytes)
8e8ec596 2135{
643b39b0
YK		 2136 if (dst != src) {
2137 dma_unmap_sg_chained(dev, src, src_nents ? : 1, DMA_TO_DEVICE,
2138 src_chained);
2139 dma_unmap_sg_chained(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE,
2140 dst_chained);
8e8ec596 2141 } else {
643b39b0
YK		 2142 dma_unmap_sg_chained(dev, src, src_nents ? : 1,
2143 DMA_BIDIRECTIONAL, src_chained);
8e8ec596
KP		 2144 }
2145
1acebad3
YK		 2146 if (iv_dma)
2147 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
a299c837
YK		 2148 if (sec4_sg_bytes)
2149 dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes,
8e8ec596
KP		 2150 DMA_TO_DEVICE);
2151}
2152
1acebad3
YK		 2153static void aead_unmap(struct device *dev,
2154 struct aead_edesc *edesc,
2155 struct aead_request *req)
2156{
2157 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2158 int ivsize = crypto_aead_ivsize(aead);
2159
643b39b0
YK		 2160 dma_unmap_sg_chained(dev, req->assoc, edesc->assoc_nents,
2161 DMA_TO_DEVICE, edesc->assoc_chained);
1acebad3
YK		 2162
2163 caam_unmap(dev, req->src, req->dst,
643b39b0
YK		 2164 edesc->src_nents, edesc->src_chained, edesc->dst_nents,
2165 edesc->dst_chained, edesc->iv_dma, ivsize,
2166 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
1acebad3
YK		 2167}
2168
acdca31d
YK		 2169static void ablkcipher_unmap(struct device *dev,
2170 struct ablkcipher_edesc *edesc,
2171 struct ablkcipher_request *req)
2172{
2173 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2174 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2175
2176 caam_unmap(dev, req->src, req->dst,
643b39b0
YK		 2177 edesc->src_nents, edesc->src_chained, edesc->dst_nents,
2178 edesc->dst_chained, edesc->iv_dma, ivsize,
2179 edesc->sec4_sg_dma, edesc->sec4_sg_bytes);
acdca31d
YK		 2180}
2181
0e479300 2182static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
8e8ec596
KP		 2183 void *context)
2184{
0e479300
YK		 2185 struct aead_request *req = context;
2186 struct aead_edesc *edesc;
8e8ec596 2187#ifdef DEBUG
0e479300 2188 struct crypto_aead *aead = crypto_aead_reqtfm(req);
8e8ec596 2189 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1acebad3 2190 int ivsize = crypto_aead_ivsize(aead);
8e8ec596
KP		 2191
2192 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2193#endif
1acebad3 2194
0e479300
YK		 2195 edesc = (struct aead_edesc *)((char *)desc -
2196 offsetof(struct aead_edesc, hw_desc));
8e8ec596 2197
fa9659cd
MV		 2198 if (err)
2199 caam_jr_strstatus(jrdev, err);
8e8ec596 2200
0e479300 2201 aead_unmap(jrdev, edesc, req);
8e8ec596
KP		 2202
2203#ifdef DEBUG
514df281 2204 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
0e479300
YK		 2205 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
2206 req->assoclen , 1);
514df281 2207 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
0e479300 2208 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src) - ivsize,
8e8ec596 2209 edesc->src_nents ? 100 : ivsize, 1);
514df281 2210 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
0e479300
YK		 2211 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2212 edesc->src_nents ? 100 : req->cryptlen +
8e8ec596
KP		 2213 ctx->authsize + 4, 1);
2214#endif
2215
2216 kfree(edesc);
2217
0e479300 2218 aead_request_complete(req, err);
8e8ec596
KP		 2219}
2220
0e479300 2221static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
8e8ec596
KP		 2222 void *context)
2223{
0e479300
YK		 2224 struct aead_request *req = context;
2225 struct aead_edesc *edesc;
8e8ec596 2226#ifdef DEBUG
0e479300 2227 struct crypto_aead *aead = crypto_aead_reqtfm(req);
8e8ec596 2228 struct caam_ctx *ctx = crypto_aead_ctx(aead);
1acebad3 2229 int ivsize = crypto_aead_ivsize(aead);
8e8ec596
KP		 2230
2231 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2232#endif
1acebad3 2233
0e479300
YK		 2234 edesc = (struct aead_edesc *)((char *)desc -
2235 offsetof(struct aead_edesc, hw_desc));
8e8ec596 2236
1acebad3 2237#ifdef DEBUG
514df281 2238 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
1acebad3
YK		 2239 DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
2240 ivsize, 1);
514df281 2241 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
1acebad3 2242 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->dst),
bbf9c893 2243 req->cryptlen - ctx->authsize, 1);
1acebad3
YK		 2244#endif
2245
fa9659cd
MV		 2246 if (err)
2247 caam_jr_strstatus(jrdev, err);
8e8ec596 2248
0e479300 2249 aead_unmap(jrdev, edesc, req);
8e8ec596
KP		 2250
2251 /*
2252 * verify hw auth check passed else return -EBADMSG
2253 */
2254 if ((err & JRSTA_CCBERR_ERRID_MASK) == JRSTA_CCBERR_ERRID_ICVCHK)
2255 err = -EBADMSG;
2256
2257#ifdef DEBUG
514df281 2258 print_hex_dump(KERN_ERR, "iphdrout@"__stringify(__LINE__)": ",
8e8ec596 2259 DUMP_PREFIX_ADDRESS, 16, 4,
0e479300
YK		 2260 ((char *)sg_virt(req->assoc) - sizeof(struct iphdr)),
2261 sizeof(struct iphdr) + req->assoclen +
2262 ((req->cryptlen > 1500) ? 1500 : req->cryptlen) +
8e8ec596 2263 ctx->authsize + 36, 1);
a299c837 2264 if (!err && edesc->sec4_sg_bytes) {
0e479300 2265 struct scatterlist *sg = sg_last(req->src, edesc->src_nents);
514df281 2266 print_hex_dump(KERN_ERR, "sglastout@"__stringify(__LINE__)": ",
8e8ec596
KP		 2267 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(sg),
2268 sg->length + ctx->authsize + 16, 1);
2269 }
2270#endif
1acebad3 2271
8e8ec596
KP		 2272 kfree(edesc);
2273
0e479300 2274 aead_request_complete(req, err);
8e8ec596
KP		 2275}
2276
acdca31d
YK		 2277static void ablkcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err,
2278 void *context)
2279{
2280 struct ablkcipher_request *req = context;
2281 struct ablkcipher_edesc *edesc;
2282#ifdef DEBUG
2283 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2284 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2285
2286 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2287#endif
2288
2289 edesc = (struct ablkcipher_edesc *)((char *)desc -
2290 offsetof(struct ablkcipher_edesc, hw_desc));
2291
fa9659cd
MV		 2292 if (err)
2293 caam_jr_strstatus(jrdev, err);
acdca31d
YK		 2294
2295#ifdef DEBUG
514df281 2296 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
acdca31d
YK		 2297 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2298 edesc->src_nents > 1 ? 100 : ivsize, 1);
514df281 2299 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
acdca31d
YK		 2300 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2301 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
2302#endif
2303
2304 ablkcipher_unmap(jrdev, edesc, req);
2305 kfree(edesc);
2306
2307 ablkcipher_request_complete(req, err);
2308}
2309
2310static void ablkcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err,
2311 void *context)
2312{
2313 struct ablkcipher_request *req = context;
2314 struct ablkcipher_edesc *edesc;
2315#ifdef DEBUG
2316 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2317 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2318
2319 dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err);
2320#endif
2321
2322 edesc = (struct ablkcipher_edesc *)((char *)desc -
2323 offsetof(struct ablkcipher_edesc, hw_desc));
fa9659cd
MV		 2324 if (err)
2325 caam_jr_strstatus(jrdev, err);
acdca31d
YK		 2326
2327#ifdef DEBUG
514df281 2328 print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ",
acdca31d
YK		 2329 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2330 ivsize, 1);
514df281 2331 print_hex_dump(KERN_ERR, "dst @"__stringify(__LINE__)": ",
acdca31d
YK		 2332 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2333 edesc->dst_nents > 1 ? 100 : req->nbytes, 1);
2334#endif
2335
2336 ablkcipher_unmap(jrdev, edesc, req);
2337 kfree(edesc);
2338
2339 ablkcipher_request_complete(req, err);
2340}
2341
8e8ec596 2342/*
1acebad3 2343 * Fill in aead job descriptor
8e8ec596 2344 */
1acebad3
YK		 2345static void init_aead_job(u32 *sh_desc, dma_addr_t ptr,
2346 struct aead_edesc *edesc,
2347 struct aead_request *req,
2348 bool all_contig, bool encrypt)
8e8ec596 2349{
0e479300 2350 struct crypto_aead *aead = crypto_aead_reqtfm(req);
8e8ec596 2351 struct caam_ctx *ctx = crypto_aead_ctx(aead);
8e8ec596
KP		 2352 int ivsize = crypto_aead_ivsize(aead);
2353 int authsize = ctx->authsize;
1acebad3
YK		 2354 u32 *desc = edesc->hw_desc;
2355 u32 out_options = 0, in_options;
2356 dma_addr_t dst_dma, src_dma;
a299c837 2357 int len, sec4_sg_index = 0;
3ef8d945 2358 bool is_gcm = false;
8e8ec596 2359
1acebad3 2360#ifdef DEBUG
8e8ec596 2361 debug("assoclen %d cryptlen %d authsize %d\n",
0e479300 2362 req->assoclen, req->cryptlen, authsize);
514df281 2363 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
0e479300
YK		 2364 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
2365 req->assoclen , 1);
514df281 2366 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1acebad3 2367 DUMP_PREFIX_ADDRESS, 16, 4, req->iv,
8e8ec596 2368 edesc->src_nents ? 100 : ivsize, 1);
514df281 2369 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
0e479300 2370 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
1acebad3 2371 edesc->src_nents ? 100 : req->cryptlen, 1);
514df281 2372 print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
8e8ec596
KP		 2373 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
2374 desc_bytes(sh_desc), 1);
2375#endif
8e8ec596 2376
3ef8d945
TA		 2377 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2378 OP_ALG_ALGSEL_AES) &&
2379 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2380 is_gcm = true;
2381
1acebad3
YK		 2382 len = desc_len(sh_desc);
2383 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
8e8ec596 2384
1acebad3 2385 if (all_contig) {
3ef8d945
TA		 2386 if (is_gcm)
2387 src_dma = edesc->iv_dma;
2388 else
2389 src_dma = sg_dma_address(req->assoc);
1acebad3 2390 in_options = 0;
8e8ec596 2391 } else {
a299c837
YK		 2392 src_dma = edesc->sec4_sg_dma;
2393 sec4_sg_index += (edesc->assoc_nents ? : 1) + 1 +
2394 (edesc->src_nents ? : 1);
1acebad3 2395 in_options = LDST_SGF;
8e8ec596 2396 }
bbf9c893
HG		 2397
2398 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen,
2399 in_options);
8e8ec596 2400
1acebad3
YK		 2401 if (likely(req->src == req->dst)) {
2402 if (all_contig) {
2403 dst_dma = sg_dma_address(req->src);
2404 } else {
a299c837 2405 dst_dma = src_dma + sizeof(struct sec4_sg_entry) *
1acebad3
YK		 2406 ((edesc->assoc_nents ? : 1) + 1);
2407 out_options = LDST_SGF;
2408 }
8e8ec596 2409 } else {
8e8ec596 2410 if (!edesc->dst_nents) {
0e479300 2411 dst_dma = sg_dma_address(req->dst);
8e8ec596 2412 } else {
a299c837
YK		 2413 dst_dma = edesc->sec4_sg_dma +
2414 sec4_sg_index *
2415 sizeof(struct sec4_sg_entry);
1acebad3 2416 out_options = LDST_SGF;
8e8ec596
KP		 2417 }
2418 }
8e8ec596 2419 if (encrypt)
bbf9c893
HG		 2420 append_seq_out_ptr(desc, dst_dma, req->cryptlen + authsize,
2421 out_options);
8e8ec596 2422 else
1acebad3
YK		 2423 append_seq_out_ptr(desc, dst_dma, req->cryptlen - authsize,
2424 out_options);
2425}
2426
2427/*
2428 * Fill in aead givencrypt job descriptor
2429 */
2430static void init_aead_giv_job(u32 *sh_desc, dma_addr_t ptr,
2431 struct aead_edesc *edesc,
2432 struct aead_request *req,
2433 int contig)
2434{
2435 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2436 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2437 int ivsize = crypto_aead_ivsize(aead);
2438 int authsize = ctx->authsize;
2439 u32 *desc = edesc->hw_desc;
2440 u32 out_options = 0, in_options;
2441 dma_addr_t dst_dma, src_dma;
a299c837 2442 int len, sec4_sg_index = 0;
bac68f2c 2443 bool is_gcm = false;
8e8ec596
KP		 2444
2445#ifdef DEBUG
1acebad3
YK		 2446 debug("assoclen %d cryptlen %d authsize %d\n",
2447 req->assoclen, req->cryptlen, authsize);
514df281 2448 print_hex_dump(KERN_ERR, "assoc @"__stringify(__LINE__)": ",
1acebad3
YK		 2449 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->assoc),
2450 req->assoclen , 1);
514df281 2451 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
1acebad3 2452 DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1);
514df281 2453 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
1acebad3
YK		 2454 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2455 edesc->src_nents > 1 ? 100 : req->cryptlen, 1);
514df281 2456 print_hex_dump(KERN_ERR, "shrdesc@"__stringify(__LINE__)": ",
1acebad3
YK		 2457 DUMP_PREFIX_ADDRESS, 16, 4, sh_desc,
2458 desc_bytes(sh_desc), 1);
8e8ec596
KP		 2459#endif
2460
bac68f2c
TA		 2461 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2462 OP_ALG_ALGSEL_AES) &&
2463 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2464 is_gcm = true;
2465
1acebad3
YK		 2466 len = desc_len(sh_desc);
2467 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2468
2469 if (contig & GIV_SRC_CONTIG) {
bac68f2c
TA		 2470 if (is_gcm)
2471 src_dma = edesc->iv_dma;
2472 else
2473 src_dma = sg_dma_address(req->assoc);
1acebad3
YK		 2474 in_options = 0;
2475 } else {
a299c837
YK		 2476 src_dma = edesc->sec4_sg_dma;
2477 sec4_sg_index += edesc->assoc_nents + 1 + edesc->src_nents;
1acebad3 2478 in_options = LDST_SGF;
8e8ec596 2479 }
bbf9c893
HG		 2480 append_seq_in_ptr(desc, src_dma, req->assoclen + ivsize + req->cryptlen,
2481 in_options);
8e8ec596 2482
1acebad3
YK		 2483 if (contig & GIV_DST_CONTIG) {
2484 dst_dma = edesc->iv_dma;
2485 } else {
2486 if (likely(req->src == req->dst)) {
a299c837 2487 dst_dma = src_dma + sizeof(struct sec4_sg_entry) *
bac68f2c
TA		 2488 (edesc->assoc_nents +
2489 (is_gcm ? 1 + edesc->src_nents : 0));
1acebad3
YK		 2490 out_options = LDST_SGF;
2491 } else {
a299c837
YK		 2492 dst_dma = edesc->sec4_sg_dma +
2493 sec4_sg_index *
2494 sizeof(struct sec4_sg_entry);
1acebad3
YK		 2495 out_options = LDST_SGF;
2496 }
2497 }
2498
bbf9c893
HG		 2499 append_seq_out_ptr(desc, dst_dma, ivsize + req->cryptlen + authsize,
2500 out_options);
8e8ec596
KP		 2501}
2502
acdca31d
YK		 2503/*
2504 * Fill in ablkcipher job descriptor
2505 */
2506static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr,
2507 struct ablkcipher_edesc *edesc,
2508 struct ablkcipher_request *req,
2509 bool iv_contig)
2510{
2511 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2512 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2513 u32 *desc = edesc->hw_desc;
2514 u32 out_options = 0, in_options;
2515 dma_addr_t dst_dma, src_dma;
a299c837 2516 int len, sec4_sg_index = 0;
acdca31d
YK		 2517
2518#ifdef DEBUG
514df281 2519 print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ",
acdca31d
YK		 2520 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2521 ivsize, 1);
514df281 2522 print_hex_dump(KERN_ERR, "src @"__stringify(__LINE__)": ",
acdca31d
YK		 2523 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2524 edesc->src_nents ? 100 : req->nbytes, 1);
2525#endif
2526
2527 len = desc_len(sh_desc);
2528 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2529
2530 if (iv_contig) {
2531 src_dma = edesc->iv_dma;
2532 in_options = 0;
2533 } else {
a299c837
YK		 2534 src_dma = edesc->sec4_sg_dma;
2535 sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents;
acdca31d
YK		 2536 in_options = LDST_SGF;
2537 }
2538 append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options);
2539
2540 if (likely(req->src == req->dst)) {
2541 if (!edesc->src_nents && iv_contig) {
2542 dst_dma = sg_dma_address(req->src);
2543 } else {
a299c837
YK		 2544 dst_dma = edesc->sec4_sg_dma +
2545 sizeof(struct sec4_sg_entry);
acdca31d
YK		 2546 out_options = LDST_SGF;
2547 }
2548 } else {
2549 if (!edesc->dst_nents) {
2550 dst_dma = sg_dma_address(req->dst);
2551 } else {
a299c837
YK		 2552 dst_dma = edesc->sec4_sg_dma +
2553 sec4_sg_index * sizeof(struct sec4_sg_entry);
acdca31d
YK		 2554 out_options = LDST_SGF;
2555 }
2556 }
2557 append_seq_out_ptr(desc, dst_dma, req->nbytes, out_options);
2558}
2559
7222d1a3
CV		 2560/*
2561 * Fill in ablkcipher givencrypt job descriptor
2562 */
2563static void init_ablkcipher_giv_job(u32 *sh_desc, dma_addr_t ptr,
2564 struct ablkcipher_edesc *edesc,
2565 struct ablkcipher_request *req,
2566 bool iv_contig)
2567{
2568 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
2569 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
2570 u32 *desc = edesc->hw_desc;
2571 u32 out_options, in_options;
2572 dma_addr_t dst_dma, src_dma;
2573 int len, sec4_sg_index = 0;
2574
2575#ifdef DEBUG
2576 print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ",
2577 DUMP_PREFIX_ADDRESS, 16, 4, req->info,
2578 ivsize, 1);
2579 print_hex_dump(KERN_ERR, "src @" __stringify(__LINE__) ": ",
2580 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2581 edesc->src_nents ? 100 : req->nbytes, 1);
2582#endif
2583
2584 len = desc_len(sh_desc);
2585 init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE);
2586
2587 if (!edesc->src_nents) {
2588 src_dma = sg_dma_address(req->src);
2589 in_options = 0;
2590 } else {
2591 src_dma = edesc->sec4_sg_dma;
2592 sec4_sg_index += edesc->src_nents;
2593 in_options = LDST_SGF;
2594 }
2595 append_seq_in_ptr(desc, src_dma, req->nbytes, in_options);
2596
2597 if (iv_contig) {
2598 dst_dma = edesc->iv_dma;
2599 out_options = 0;
2600 } else {
2601 dst_dma = edesc->sec4_sg_dma +
2602 sec4_sg_index * sizeof(struct sec4_sg_entry);
2603 out_options = LDST_SGF;
2604 }
2605 append_seq_out_ptr(desc, dst_dma, req->nbytes + ivsize, out_options);
2606}
2607
8e8ec596 2608/*
1acebad3 2609 * allocate and map the aead extended descriptor
8e8ec596 2610 */
0e479300 2611static struct aead_edesc *aead_edesc_alloc(struct aead_request *req,
bbf9c893
HG		 2612 int desc_bytes, bool *all_contig_ptr,
2613 bool encrypt)
8e8ec596 2614{
0e479300 2615 struct crypto_aead *aead = crypto_aead_reqtfm(req);
8e8ec596
KP		 2616 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2617 struct device *jrdev = ctx->jrdev;
1acebad3
YK		 2618 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2619 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2620 int assoc_nents, src_nents, dst_nents = 0;
0e479300 2621 struct aead_edesc *edesc;
1acebad3
YK		 2622 dma_addr_t iv_dma = 0;
2623 int sgc;
2624 bool all_contig = true;
643b39b0 2625 bool assoc_chained = false, src_chained = false, dst_chained = false;
1acebad3 2626 int ivsize = crypto_aead_ivsize(aead);
a299c837 2627 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
bbf9c893 2628 unsigned int authsize = ctx->authsize;
3ef8d945 2629 bool is_gcm = false;
8e8ec596 2630
643b39b0 2631 assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained);
1acebad3 2632
bbf9c893
HG		 2633 if (unlikely(req->dst != req->src)) {
2634 src_nents = sg_count(req->src, req->cryptlen, &src_chained);
2635 dst_nents = sg_count(req->dst,
2636 req->cryptlen +
2637 (encrypt ? authsize : (-authsize)),
2638 &dst_chained);
2639 } else {
2640 src_nents = sg_count(req->src,
2641 req->cryptlen +
2642 (encrypt ? authsize : 0),
2643 &src_chained);
2644 }
1acebad3 2645
643b39b0 2646 sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1,
286233e6 2647 DMA_TO_DEVICE, assoc_chained);
1acebad3 2648 if (likely(req->src == req->dst)) {
643b39b0
YK		 2649 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2650 DMA_BIDIRECTIONAL, src_chained);
1acebad3 2651 } else {
643b39b0
YK		 2652 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2653 DMA_TO_DEVICE, src_chained);
2654 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
2655 DMA_FROM_DEVICE, dst_chained);
1acebad3
YK		 2656 }
2657
1acebad3 2658 iv_dma = dma_map_single(jrdev, req->iv, ivsize, DMA_TO_DEVICE);
ce572085
HG		 2659 if (dma_mapping_error(jrdev, iv_dma)) {
2660 dev_err(jrdev, "unable to map IV\n");
2661 return ERR_PTR(-ENOMEM);
2662 }
2663
3ef8d945
TA		 2664 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2665 OP_ALG_ALGSEL_AES) &&
2666 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2667 is_gcm = true;
2668
2669 /*
2670 * Check if data are contiguous.
2671 * GCM expected input sequence: IV, AAD, text
2672 * All other - expected input sequence: AAD, IV, text
2673 */
2674 if (is_gcm)
2675 all_contig = (!assoc_nents &&
2676 iv_dma + ivsize == sg_dma_address(req->assoc) &&
2677 !src_nents && sg_dma_address(req->assoc) +
2678 req->assoclen == sg_dma_address(req->src));
2679 else
2680 all_contig = (!assoc_nents && sg_dma_address(req->assoc) +
2681 req->assoclen == iv_dma && !src_nents &&
2682 iv_dma + ivsize == sg_dma_address(req->src));
2683 if (!all_contig) {
1acebad3
YK		 2684 assoc_nents = assoc_nents ? : 1;
2685 src_nents = src_nents ? : 1;
a299c837 2686 sec4_sg_len = assoc_nents + 1 + src_nents;
8e8ec596 2687 }
3ef8d945 2688
a299c837 2689 sec4_sg_len += dst_nents;
8e8ec596 2690
a299c837 2691 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
8e8ec596
KP		 2692
2693 /* allocate space for base edesc and hw desc commands, link tables */
0e479300 2694 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
a299c837 2695 sec4_sg_bytes, GFP_DMA | flags);
8e8ec596
KP		 2696 if (!edesc) {
2697 dev_err(jrdev, "could not allocate extended descriptor\n");
2698 return ERR_PTR(-ENOMEM);
2699 }
2700
2701 edesc->assoc_nents = assoc_nents;
643b39b0 2702 edesc->assoc_chained = assoc_chained;
8e8ec596 2703 edesc->src_nents = src_nents;
643b39b0 2704 edesc->src_chained = src_chained;
8e8ec596 2705 edesc->dst_nents = dst_nents;
643b39b0 2706 edesc->dst_chained = dst_chained;
1acebad3 2707 edesc->iv_dma = iv_dma;
a299c837
YK		 2708 edesc->sec4_sg_bytes = sec4_sg_bytes;
2709 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
2710 desc_bytes;
1acebad3
YK		 2711 *all_contig_ptr = all_contig;
2712
a299c837 2713 sec4_sg_index = 0;
1acebad3 2714 if (!all_contig) {
3ef8d945
TA		 2715 if (!is_gcm) {
2716 sg_to_sec4_sg(req->assoc,
2717 (assoc_nents ? : 1),
2718 edesc->sec4_sg +
2719 sec4_sg_index, 0);
2720 sec4_sg_index += assoc_nents ? : 1;
2721 }
2722
a299c837 2723 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
1acebad3 2724 iv_dma, ivsize, 0);
a299c837 2725 sec4_sg_index += 1;
3ef8d945
TA		 2726
2727 if (is_gcm) {
2728 sg_to_sec4_sg(req->assoc,
2729 (assoc_nents ? : 1),
2730 edesc->sec4_sg +
2731 sec4_sg_index, 0);
2732 sec4_sg_index += assoc_nents ? : 1;
2733 }
2734
a299c837
YK		 2735 sg_to_sec4_sg_last(req->src,
2736 (src_nents ? : 1),
2737 edesc->sec4_sg +
2738 sec4_sg_index, 0);
2739 sec4_sg_index += src_nents ? : 1;
1acebad3
YK		 2740 }
2741 if (dst_nents) {
a299c837
YK		 2742 sg_to_sec4_sg_last(req->dst, dst_nents,
2743 edesc->sec4_sg + sec4_sg_index, 0);
1acebad3 2744 }
1da2be33
RG		 2745 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2746 sec4_sg_bytes, DMA_TO_DEVICE);
ce572085
HG		 2747 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2748 dev_err(jrdev, "unable to map S/G table\n");
2749 return ERR_PTR(-ENOMEM);
2750 }
8e8ec596
KP		 2751
2752 return edesc;
2753}
2754
0e479300 2755static int aead_encrypt(struct aead_request *req)
8e8ec596 2756{
0e479300
YK		 2757 struct aead_edesc *edesc;
2758 struct crypto_aead *aead = crypto_aead_reqtfm(req);
8e8ec596
KP		 2759 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2760 struct device *jrdev = ctx->jrdev;
1acebad3 2761 bool all_contig;
8e8ec596 2762 u32 *desc;
1acebad3
YK		 2763 int ret = 0;
2764
8e8ec596 2765 /* allocate extended descriptor */
1acebad3 2766 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
bbf9c893 2767 CAAM_CMD_SZ, &all_contig, true);
8e8ec596
KP		 2768 if (IS_ERR(edesc))
2769 return PTR_ERR(edesc);
2770
1acebad3
YK		 2771 /* Create and submit job descriptor */
2772 init_aead_job(ctx->sh_desc_enc, ctx->sh_desc_enc_dma, edesc, req,
2773 all_contig, true);
2774#ifdef DEBUG
514df281 2775 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1acebad3
YK		 2776 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2777 desc_bytes(edesc->hw_desc), 1);
2778#endif
8e8ec596 2779
1acebad3
YK		 2780 desc = edesc->hw_desc;
2781 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
2782 if (!ret) {
2783 ret = -EINPROGRESS;
2784 } else {
2785 aead_unmap(jrdev, edesc, req);
2786 kfree(edesc);
2787 }
8e8ec596 2788
1acebad3 2789 return ret;
8e8ec596
KP		 2790}
2791
0e479300 2792static int aead_decrypt(struct aead_request *req)
8e8ec596 2793{
1acebad3 2794 struct aead_edesc *edesc;
8e8ec596 2795 struct crypto_aead *aead = crypto_aead_reqtfm(req);
8e8ec596
KP		 2796 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2797 struct device *jrdev = ctx->jrdev;
1acebad3 2798 bool all_contig;
8e8ec596 2799 u32 *desc;
1acebad3 2800 int ret = 0;
8e8ec596
KP		 2801
2802 /* allocate extended descriptor */
1acebad3 2803 edesc = aead_edesc_alloc(req, DESC_JOB_IO_LEN *
bbf9c893 2804 CAAM_CMD_SZ, &all_contig, false);
8e8ec596
KP		 2805 if (IS_ERR(edesc))
2806 return PTR_ERR(edesc);
2807
1acebad3 2808#ifdef DEBUG
514df281 2809 print_hex_dump(KERN_ERR, "dec src@"__stringify(__LINE__)": ",
1acebad3
YK		 2810 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
2811 req->cryptlen, 1);
2812#endif
2813
2814 /* Create and submit job descriptor*/
2815 init_aead_job(ctx->sh_desc_dec,
2816 ctx->sh_desc_dec_dma, edesc, req, all_contig, false);
2817#ifdef DEBUG
514df281 2818 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1acebad3
YK		 2819 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
2820 desc_bytes(edesc->hw_desc), 1);
2821#endif
2822
8e8ec596 2823 desc = edesc->hw_desc;
1acebad3
YK		 2824 ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req);
2825 if (!ret) {
2826 ret = -EINPROGRESS;
2827 } else {
2828 aead_unmap(jrdev, edesc, req);
2829 kfree(edesc);
2830 }
8e8ec596 2831
1acebad3
YK		 2832 return ret;
2833}
8e8ec596 2834
1acebad3
YK		 2835/*
2836 * allocate and map the aead extended descriptor for aead givencrypt
2837 */
2838static struct aead_edesc *aead_giv_edesc_alloc(struct aead_givcrypt_request
2839 *greq, int desc_bytes,
2840 u32 *contig_ptr)
2841{
2842 struct aead_request *req = &greq->areq;
2843 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2844 struct caam_ctx *ctx = crypto_aead_ctx(aead);
2845 struct device *jrdev = ctx->jrdev;
2846 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
2847 CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC;
2848 int assoc_nents, src_nents, dst_nents = 0;
2849 struct aead_edesc *edesc;
2850 dma_addr_t iv_dma = 0;
2851 int sgc;
2852 u32 contig = GIV_SRC_CONTIG | GIV_DST_CONTIG;
2853 int ivsize = crypto_aead_ivsize(aead);
643b39b0 2854 bool assoc_chained = false, src_chained = false, dst_chained = false;
a299c837 2855 int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes;
bac68f2c 2856 bool is_gcm = false;
8e8ec596 2857
643b39b0
YK		 2858 assoc_nents = sg_count(req->assoc, req->assoclen, &assoc_chained);
2859 src_nents = sg_count(req->src, req->cryptlen, &src_chained);
8e8ec596 2860
1acebad3 2861 if (unlikely(req->dst != req->src))
bbf9c893
HG		 2862 dst_nents = sg_count(req->dst, req->cryptlen + ctx->authsize,
2863 &dst_chained);
1acebad3 2864
643b39b0 2865 sgc = dma_map_sg_chained(jrdev, req->assoc, assoc_nents ? : 1,
286233e6 2866 DMA_TO_DEVICE, assoc_chained);
1acebad3 2867 if (likely(req->src == req->dst)) {
643b39b0
YK		 2868 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2869 DMA_BIDIRECTIONAL, src_chained);
1acebad3 2870 } else {
643b39b0
YK		 2871 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
2872 DMA_TO_DEVICE, src_chained);
2873 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
2874 DMA_FROM_DEVICE, dst_chained);
1acebad3
YK		 2875 }
2876
1acebad3 2877 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
ce572085
HG		 2878 if (dma_mapping_error(jrdev, iv_dma)) {
2879 dev_err(jrdev, "unable to map IV\n");
2880 return ERR_PTR(-ENOMEM);
2881 }
2882
bac68f2c
TA		 2883 if (((ctx->class1_alg_type & OP_ALG_ALGSEL_MASK) ==
2884 OP_ALG_ALGSEL_AES) &&
2885 ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == OP_ALG_AAI_GCM))
2886 is_gcm = true;
2887
2888 /*
2889 * Check if data are contiguous.
2890 * GCM expected input sequence: IV, AAD, text
2891 * All other - expected input sequence: AAD, IV, text
2892 */
2893
2894 if (is_gcm) {
2895 if (assoc_nents || iv_dma + ivsize !=
2896 sg_dma_address(req->assoc) || src_nents ||
2897 sg_dma_address(req->assoc) + req->assoclen !=
2898 sg_dma_address(req->src))
2899 contig &= ~GIV_SRC_CONTIG;
2900 } else {
2901 if (assoc_nents ||
2902 sg_dma_address(req->assoc) + req->assoclen != iv_dma ||
2903 src_nents || iv_dma + ivsize != sg_dma_address(req->src))
2904 contig &= ~GIV_SRC_CONTIG;
2905 }
2906
1acebad3
YK		 2907 if (dst_nents || iv_dma + ivsize != sg_dma_address(req->dst))
2908 contig &= ~GIV_DST_CONTIG;
bac68f2c 2909
1acebad3
YK		 2910 if (!(contig & GIV_SRC_CONTIG)) {
2911 assoc_nents = assoc_nents ? : 1;
2912 src_nents = src_nents ? : 1;
a299c837 2913 sec4_sg_len += assoc_nents + 1 + src_nents;
19167bfd
TA		 2914 if (req->src == req->dst &&
2915 (src_nents || iv_dma + ivsize != sg_dma_address(req->src)))
1acebad3
YK		 2916 contig &= ~GIV_DST_CONTIG;
2917 }
bac68f2c
TA		 2918
2919 /*
2920 * Add new sg entries for GCM output sequence.
2921 * Expected output sequence: IV, encrypted text.
2922 */
2923 if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG))
2924 sec4_sg_len += 1 + src_nents;
2925
2926 if (unlikely(req->src != req->dst)) {
2927 dst_nents = dst_nents ? : 1;
2928 sec4_sg_len += 1 + dst_nents;
2929 }
1acebad3 2930
a299c837 2931 sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry);
1acebad3
YK		 2932
2933 /* allocate space for base edesc and hw desc commands, link tables */
2934 edesc = kmalloc(sizeof(struct aead_edesc) + desc_bytes +
a299c837 2935 sec4_sg_bytes, GFP_DMA | flags);
1acebad3
YK		 2936 if (!edesc) {
2937 dev_err(jrdev, "could not allocate extended descriptor\n");
2938 return ERR_PTR(-ENOMEM);
2939 }
2940
2941 edesc->assoc_nents = assoc_nents;
643b39b0 2942 edesc->assoc_chained = assoc_chained;
1acebad3 2943 edesc->src_nents = src_nents;
643b39b0 2944 edesc->src_chained = src_chained;
1acebad3 2945 edesc->dst_nents = dst_nents;
643b39b0 2946 edesc->dst_chained = dst_chained;
1acebad3 2947 edesc->iv_dma = iv_dma;
a299c837
YK		 2948 edesc->sec4_sg_bytes = sec4_sg_bytes;
2949 edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) +
2950 desc_bytes;
1acebad3
YK		 2951 *contig_ptr = contig;
2952
a299c837 2953 sec4_sg_index = 0;
1acebad3 2954 if (!(contig & GIV_SRC_CONTIG)) {
bac68f2c
TA		 2955 if (!is_gcm) {
2956 sg_to_sec4_sg(req->assoc, assoc_nents,
2957 edesc->sec4_sg + sec4_sg_index, 0);
2958 sec4_sg_index += assoc_nents;
2959 }
2960
a299c837 2961 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
1acebad3 2962 iv_dma, ivsize, 0);
a299c837 2963 sec4_sg_index += 1;
bac68f2c
TA		 2964
2965 if (is_gcm) {
2966 sg_to_sec4_sg(req->assoc, assoc_nents,
2967 edesc->sec4_sg + sec4_sg_index, 0);
2968 sec4_sg_index += assoc_nents;
2969 }
2970
a299c837
YK		 2971 sg_to_sec4_sg_last(req->src, src_nents,
2972 edesc->sec4_sg +
2973 sec4_sg_index, 0);
2974 sec4_sg_index += src_nents;
1acebad3 2975 }
bac68f2c
TA		 2976
2977 if (is_gcm && req->src == req->dst && !(contig & GIV_DST_CONTIG)) {
2978 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
2979 iv_dma, ivsize, 0);
2980 sec4_sg_index += 1;
2981 sg_to_sec4_sg_last(req->src, src_nents,
2982 edesc->sec4_sg + sec4_sg_index, 0);
2983 }
2984
1acebad3 2985 if (unlikely(req->src != req->dst && !(contig & GIV_DST_CONTIG))) {
a299c837 2986 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
1acebad3 2987 iv_dma, ivsize, 0);
a299c837
YK		 2988 sec4_sg_index += 1;
2989 sg_to_sec4_sg_last(req->dst, dst_nents,
2990 edesc->sec4_sg + sec4_sg_index, 0);
1acebad3 2991 }
1da2be33
RG		 2992 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
2993 sec4_sg_bytes, DMA_TO_DEVICE);
ce572085
HG		 2994 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
2995 dev_err(jrdev, "unable to map S/G table\n");
2996 return ERR_PTR(-ENOMEM);
2997 }
1acebad3
YK		 2998
2999 return edesc;
8e8ec596
KP		 3000}
3001
0e479300 3002static int aead_givencrypt(struct aead_givcrypt_request *areq)
8e8ec596 3003{
0e479300
YK		 3004 struct aead_request *req = &areq->areq;
3005 struct aead_edesc *edesc;
3006 struct crypto_aead *aead = crypto_aead_reqtfm(req);
8e8ec596
KP		 3007 struct caam_ctx *ctx = crypto_aead_ctx(aead);
3008 struct device *jrdev = ctx->jrdev;
1acebad3 3009 u32 contig;
8e8ec596 3010 u32 *desc;
1acebad3 3011 int ret = 0;
8e8ec596 3012
8e8ec596 3013 /* allocate extended descriptor */
1acebad3
YK		 3014 edesc = aead_giv_edesc_alloc(areq, DESC_JOB_IO_LEN *
3015 CAAM_CMD_SZ, &contig);
3016
8e8ec596
KP		 3017 if (IS_ERR(edesc))
3018 return PTR_ERR(edesc);
3019
1acebad3 3020#ifdef DEBUG
514df281 3021 print_hex_dump(KERN_ERR, "giv src@"__stringify(__LINE__)": ",
1acebad3
YK		 3022 DUMP_PREFIX_ADDRESS, 16, 4, sg_virt(req->src),
3023 req->cryptlen, 1);
3024#endif
8e8ec596 3025
1acebad3
YK		 3026 /* Create and submit job descriptor*/
3027 init_aead_giv_job(ctx->sh_desc_givenc,
3028 ctx->sh_desc_givenc_dma, edesc, req, contig);
3029#ifdef DEBUG
514df281 3030 print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ",
1acebad3
YK		 3031 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3032 desc_bytes(edesc->hw_desc), 1);
3033#endif
8e8ec596 3034
1acebad3
YK		 3035 desc = edesc->hw_desc;
3036 ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req);
3037 if (!ret) {
3038 ret = -EINPROGRESS;
3039 } else {
3040 aead_unmap(jrdev, edesc, req);
3041 kfree(edesc);
3042 }
8e8ec596 3043
1acebad3 3044 return ret;
8e8ec596
KP		 3045}
3046
ae4a825f
HG		 3047static int aead_null_givencrypt(struct aead_givcrypt_request *areq)
3048{
3049 return aead_encrypt(&areq->areq);
3050}
3051
acdca31d
YK		 3052/*
3053 * allocate and map the ablkcipher extended descriptor for ablkcipher
3054 */
3055static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request
3056 *req, int desc_bytes,
3057 bool *iv_contig_out)
3058{
3059 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3060 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3061 struct device *jrdev = ctx->jrdev;
3062 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
3063 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
3064 GFP_KERNEL : GFP_ATOMIC;
a299c837 3065 int src_nents, dst_nents = 0, sec4_sg_bytes;
acdca31d
YK		 3066 struct ablkcipher_edesc *edesc;
3067 dma_addr_t iv_dma = 0;
3068 bool iv_contig = false;
3069 int sgc;
3070 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
643b39b0 3071 bool src_chained = false, dst_chained = false;
a299c837 3072 int sec4_sg_index;
acdca31d 3073
643b39b0 3074 src_nents = sg_count(req->src, req->nbytes, &src_chained);
acdca31d 3075
643b39b0
YK		 3076 if (req->dst != req->src)
3077 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained);
acdca31d
YK		 3078
3079 if (likely(req->src == req->dst)) {
643b39b0
YK		 3080 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3081 DMA_BIDIRECTIONAL, src_chained);
acdca31d 3082 } else {
643b39b0
YK		 3083 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3084 DMA_TO_DEVICE, src_chained);
3085 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
3086 DMA_FROM_DEVICE, dst_chained);
acdca31d
YK		 3087 }
3088
ce572085
HG		 3089 iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE);
3090 if (dma_mapping_error(jrdev, iv_dma)) {
3091 dev_err(jrdev, "unable to map IV\n");
3092 return ERR_PTR(-ENOMEM);
3093 }
3094
acdca31d
YK		 3095 /*
3096 * Check if iv can be contiguous with source and destination.
3097 * If so, include it. If not, create scatterlist.
3098 */
acdca31d
YK		 3099 if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src))
3100 iv_contig = true;
3101 else
3102 src_nents = src_nents ? : 1;
a299c837
YK		 3103 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
3104 sizeof(struct sec4_sg_entry);
acdca31d
YK		 3105
3106 /* allocate space for base edesc and hw desc commands, link tables */
3107 edesc = kmalloc(sizeof(struct ablkcipher_edesc) + desc_bytes +
a299c837 3108 sec4_sg_bytes, GFP_DMA | flags);
acdca31d
YK		 3109 if (!edesc) {
3110 dev_err(jrdev, "could not allocate extended descriptor\n");
3111 return ERR_PTR(-ENOMEM);
3112 }
3113
3114 edesc->src_nents = src_nents;
643b39b0 3115 edesc->src_chained = src_chained;
acdca31d 3116 edesc->dst_nents = dst_nents;
643b39b0 3117 edesc->dst_chained = dst_chained;
a299c837
YK		 3118 edesc->sec4_sg_bytes = sec4_sg_bytes;
3119 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
3120 desc_bytes;
acdca31d 3121
a299c837 3122 sec4_sg_index = 0;
acdca31d 3123 if (!iv_contig) {
a299c837
YK		 3124 dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0);
3125 sg_to_sec4_sg_last(req->src, src_nents,
3126 edesc->sec4_sg + 1, 0);
3127 sec4_sg_index += 1 + src_nents;
acdca31d
YK		 3128 }
3129
643b39b0 3130 if (dst_nents) {
a299c837
YK		 3131 sg_to_sec4_sg_last(req->dst, dst_nents,
3132 edesc->sec4_sg + sec4_sg_index, 0);
acdca31d
YK		 3133 }
3134
a299c837
YK		 3135 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
3136 sec4_sg_bytes, DMA_TO_DEVICE);
ce572085
HG		 3137 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
3138 dev_err(jrdev, "unable to map S/G table\n");
3139 return ERR_PTR(-ENOMEM);
3140 }
3141
acdca31d
YK		 3142 edesc->iv_dma = iv_dma;
3143
3144#ifdef DEBUG
514df281 3145 print_hex_dump(KERN_ERR, "ablkcipher sec4_sg@"__stringify(__LINE__)": ",
a299c837
YK		 3146 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
3147 sec4_sg_bytes, 1);
acdca31d
YK		 3148#endif
3149
3150 *iv_contig_out = iv_contig;
3151 return edesc;
3152}
3153
3154static int ablkcipher_encrypt(struct ablkcipher_request *req)
3155{
3156 struct ablkcipher_edesc *edesc;
3157 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3158 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3159 struct device *jrdev = ctx->jrdev;
3160 bool iv_contig;
3161 u32 *desc;
3162 int ret = 0;
3163
3164 /* allocate extended descriptor */
3165 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
3166 CAAM_CMD_SZ, &iv_contig);
3167 if (IS_ERR(edesc))
3168 return PTR_ERR(edesc);
3169
3170 /* Create and submit job descriptor*/
3171 init_ablkcipher_job(ctx->sh_desc_enc,
3172 ctx->sh_desc_enc_dma, edesc, req, iv_contig);
3173#ifdef DEBUG
514df281 3174 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
acdca31d
YK		 3175 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3176 desc_bytes(edesc->hw_desc), 1);
3177#endif
3178 desc = edesc->hw_desc;
3179 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
3180
3181 if (!ret) {
3182 ret = -EINPROGRESS;
3183 } else {
3184 ablkcipher_unmap(jrdev, edesc, req);
3185 kfree(edesc);
3186 }
3187
3188 return ret;
3189}
3190
3191static int ablkcipher_decrypt(struct ablkcipher_request *req)
3192{
3193 struct ablkcipher_edesc *edesc;
3194 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3195 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3196 struct device *jrdev = ctx->jrdev;
3197 bool iv_contig;
3198 u32 *desc;
3199 int ret = 0;
3200
3201 /* allocate extended descriptor */
3202 edesc = ablkcipher_edesc_alloc(req, DESC_JOB_IO_LEN *
3203 CAAM_CMD_SZ, &iv_contig);
3204 if (IS_ERR(edesc))
3205 return PTR_ERR(edesc);
3206
3207 /* Create and submit job descriptor*/
3208 init_ablkcipher_job(ctx->sh_desc_dec,
3209 ctx->sh_desc_dec_dma, edesc, req, iv_contig);
3210 desc = edesc->hw_desc;
3211#ifdef DEBUG
514df281 3212 print_hex_dump(KERN_ERR, "ablkcipher jobdesc@"__stringify(__LINE__)": ",
acdca31d
YK		 3213 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3214 desc_bytes(edesc->hw_desc), 1);
3215#endif
3216
3217 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_decrypt_done, req);
3218 if (!ret) {
3219 ret = -EINPROGRESS;
3220 } else {
3221 ablkcipher_unmap(jrdev, edesc, req);
3222 kfree(edesc);
3223 }
3224
3225 return ret;
3226}
3227
7222d1a3
CV		 3228/*
3229 * allocate and map the ablkcipher extended descriptor
3230 * for ablkcipher givencrypt
3231 */
3232static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc(
3233 struct skcipher_givcrypt_request *greq,
3234 int desc_bytes,
3235 bool *iv_contig_out)
3236{
3237 struct ablkcipher_request *req = &greq->creq;
3238 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3239 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3240 struct device *jrdev = ctx->jrdev;
3241 gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG |
3242 CRYPTO_TFM_REQ_MAY_SLEEP)) ?
3243 GFP_KERNEL : GFP_ATOMIC;
3244 int src_nents, dst_nents = 0, sec4_sg_bytes;
3245 struct ablkcipher_edesc *edesc;
3246 dma_addr_t iv_dma = 0;
3247 bool iv_contig = false;
3248 int sgc;
3249 int ivsize = crypto_ablkcipher_ivsize(ablkcipher);
3250 bool src_chained = false, dst_chained = false;
3251 int sec4_sg_index;
3252
3253 src_nents = sg_count(req->src, req->nbytes, &src_chained);
3254
3255 if (unlikely(req->dst != req->src))
3256 dst_nents = sg_count(req->dst, req->nbytes, &dst_chained);
3257
3258 if (likely(req->src == req->dst)) {
3259 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3260 DMA_BIDIRECTIONAL, src_chained);
3261 } else {
3262 sgc = dma_map_sg_chained(jrdev, req->src, src_nents ? : 1,
3263 DMA_TO_DEVICE, src_chained);
3264 sgc = dma_map_sg_chained(jrdev, req->dst, dst_nents ? : 1,
3265 DMA_FROM_DEVICE, dst_chained);
3266 }
3267
3268 /*
3269 * Check if iv can be contiguous with source and destination.
3270 * If so, include it. If not, create scatterlist.
3271 */
3272 iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE);
3273 if (dma_mapping_error(jrdev, iv_dma)) {
3274 dev_err(jrdev, "unable to map IV\n");
3275 return ERR_PTR(-ENOMEM);
3276 }
3277
3278 if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst))
3279 iv_contig = true;
3280 else
3281 dst_nents = dst_nents ? : 1;
3282 sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) *
3283 sizeof(struct sec4_sg_entry);
3284
3285 /* allocate space for base edesc and hw desc commands, link tables */
3286 edesc = kmalloc(sizeof(*edesc) + desc_bytes +
3287 sec4_sg_bytes, GFP_DMA | flags);
3288 if (!edesc) {
3289 dev_err(jrdev, "could not allocate extended descriptor\n");
3290 return ERR_PTR(-ENOMEM);
3291 }
3292
3293 edesc->src_nents = src_nents;
3294 edesc->src_chained = src_chained;
3295 edesc->dst_nents = dst_nents;
3296 edesc->dst_chained = dst_chained;
3297 edesc->sec4_sg_bytes = sec4_sg_bytes;
3298 edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) +
3299 desc_bytes;
3300
3301 sec4_sg_index = 0;
3302 if (src_nents) {
3303 sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0);
3304 sec4_sg_index += src_nents;
3305 }
3306
3307 if (!iv_contig) {
3308 dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index,
3309 iv_dma, ivsize, 0);
3310 sec4_sg_index += 1;
3311 sg_to_sec4_sg_last(req->dst, dst_nents,
3312 edesc->sec4_sg + sec4_sg_index, 0);
3313 }
3314
3315 edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg,
3316 sec4_sg_bytes, DMA_TO_DEVICE);
3317 if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) {
3318 dev_err(jrdev, "unable to map S/G table\n");
3319 return ERR_PTR(-ENOMEM);
3320 }
3321 edesc->iv_dma = iv_dma;
3322
3323#ifdef DEBUG
3324 print_hex_dump(KERN_ERR,
3325 "ablkcipher sec4_sg@" __stringify(__LINE__) ": ",
3326 DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg,
3327 sec4_sg_bytes, 1);
3328#endif
3329
3330 *iv_contig_out = iv_contig;
3331 return edesc;
3332}
3333
3334static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq)
3335{
3336 struct ablkcipher_request *req = &creq->creq;
3337 struct ablkcipher_edesc *edesc;
3338 struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req);
3339 struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher);
3340 struct device *jrdev = ctx->jrdev;
3341 bool iv_contig;
3342 u32 *desc;
3343 int ret = 0;
3344
3345 /* allocate extended descriptor */
3346 edesc = ablkcipher_giv_edesc_alloc(creq, DESC_JOB_IO_LEN *
3347 CAAM_CMD_SZ, &iv_contig);
3348 if (IS_ERR(edesc))
3349 return PTR_ERR(edesc);
3350
3351 /* Create and submit job descriptor*/
3352 init_ablkcipher_giv_job(ctx->sh_desc_givenc, ctx->sh_desc_givenc_dma,
3353 edesc, req, iv_contig);
3354#ifdef DEBUG
3355 print_hex_dump(KERN_ERR,
3356 "ablkcipher jobdesc@" __stringify(__LINE__) ": ",
3357 DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc,
3358 desc_bytes(edesc->hw_desc), 1);
3359#endif
3360 desc = edesc->hw_desc;
3361 ret = caam_jr_enqueue(jrdev, desc, ablkcipher_encrypt_done, req);
3362
3363 if (!ret) {
3364 ret = -EINPROGRESS;
3365 } else {
3366 ablkcipher_unmap(jrdev, edesc, req);
3367 kfree(edesc);
3368 }
3369
3370 return ret;
3371}
3372
885e9e2f 3373#define template_aead template_u.aead
acdca31d 3374#define template_ablkcipher template_u.ablkcipher
8e8ec596
KP		 3375struct caam_alg_template {
3376 char name[CRYPTO_MAX_ALG_NAME];
3377 char driver_name[CRYPTO_MAX_ALG_NAME];
3378 unsigned int blocksize;
885e9e2f
YK		 3379 u32 type;
3380 union {
3381 struct ablkcipher_alg ablkcipher;
3382 struct aead_alg aead;
3383 struct blkcipher_alg blkcipher;
3384 struct cipher_alg cipher;
3385 struct compress_alg compress;
3386 struct rng_alg rng;
3387 } template_u;
8e8ec596
KP		 3388 u32 class1_alg_type;
3389 u32 class2_alg_type;
3390 u32 alg_op;
3391};
3392
3393static struct caam_alg_template driver_algs[] = {
246bbedb 3394 /* single-pass ipsec_esp descriptor */
ae4a825f
HG		 3395 {
3396 .name = "authenc(hmac(md5),ecb(cipher_null))",
3397 .driver_name = "authenc-hmac-md5-ecb-cipher_null-caam",
3398 .blocksize = NULL_BLOCK_SIZE,
3399 .type = CRYPTO_ALG_TYPE_AEAD,
3400 .template_aead = {
3401 .setkey = aead_setkey,
3402 .setauthsize = aead_setauthsize,
3403 .encrypt = aead_encrypt,
3404 .decrypt = aead_decrypt,
3405 .givencrypt = aead_null_givencrypt,
3406 .geniv = "<built-in>",
3407 .ivsize = NULL_IV_SIZE,
3408 .maxauthsize = MD5_DIGEST_SIZE,
3409 },
3410 .class1_alg_type = 0,
3411 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3412 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3413 },
3414 {
3415 .name = "authenc(hmac(sha1),ecb(cipher_null))",
3416 .driver_name = "authenc-hmac-sha1-ecb-cipher_null-caam",
3417 .blocksize = NULL_BLOCK_SIZE,
3418 .type = CRYPTO_ALG_TYPE_AEAD,
3419 .template_aead = {
3420 .setkey = aead_setkey,
3421 .setauthsize = aead_setauthsize,
3422 .encrypt = aead_encrypt,
3423 .decrypt = aead_decrypt,
3424 .givencrypt = aead_null_givencrypt,
3425 .geniv = "<built-in>",
3426 .ivsize = NULL_IV_SIZE,
3427 .maxauthsize = SHA1_DIGEST_SIZE,
3428 },
3429 .class1_alg_type = 0,
3430 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3431 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3432 },
3433 {
3434 .name = "authenc(hmac(sha224),ecb(cipher_null))",
3435 .driver_name = "authenc-hmac-sha224-ecb-cipher_null-caam",
3436 .blocksize = NULL_BLOCK_SIZE,
3437 .type = CRYPTO_ALG_TYPE_AEAD,
3438 .template_aead = {
3439 .setkey = aead_setkey,
3440 .setauthsize = aead_setauthsize,
3441 .encrypt = aead_encrypt,
3442 .decrypt = aead_decrypt,
3443 .givencrypt = aead_null_givencrypt,
3444 .geniv = "<built-in>",
3445 .ivsize = NULL_IV_SIZE,
3446 .maxauthsize = SHA224_DIGEST_SIZE,
3447 },
3448 .class1_alg_type = 0,
3449 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3450 OP_ALG_AAI_HMAC_PRECOMP,
3451 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3452 },
3453 {
3454 .name = "authenc(hmac(sha256),ecb(cipher_null))",
3455 .driver_name = "authenc-hmac-sha256-ecb-cipher_null-caam",
3456 .blocksize = NULL_BLOCK_SIZE,
3457 .type = CRYPTO_ALG_TYPE_AEAD,
3458 .template_aead = {
3459 .setkey = aead_setkey,
3460 .setauthsize = aead_setauthsize,
3461 .encrypt = aead_encrypt,
3462 .decrypt = aead_decrypt,
3463 .givencrypt = aead_null_givencrypt,
3464 .geniv = "<built-in>",
3465 .ivsize = NULL_IV_SIZE,
3466 .maxauthsize = SHA256_DIGEST_SIZE,
3467 },
3468 .class1_alg_type = 0,
3469 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3470 OP_ALG_AAI_HMAC_PRECOMP,
3471 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3472 },
3473 {
3474 .name = "authenc(hmac(sha384),ecb(cipher_null))",
3475 .driver_name = "authenc-hmac-sha384-ecb-cipher_null-caam",
3476 .blocksize = NULL_BLOCK_SIZE,
3477 .type = CRYPTO_ALG_TYPE_AEAD,
3478 .template_aead = {
3479 .setkey = aead_setkey,
3480 .setauthsize = aead_setauthsize,
3481 .encrypt = aead_encrypt,
3482 .decrypt = aead_decrypt,
3483 .givencrypt = aead_null_givencrypt,
3484 .geniv = "<built-in>",
3485 .ivsize = NULL_IV_SIZE,
3486 .maxauthsize = SHA384_DIGEST_SIZE,
3487 },
3488 .class1_alg_type = 0,
3489 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3490 OP_ALG_AAI_HMAC_PRECOMP,
3491 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3492 },
3493 {
3494 .name = "authenc(hmac(sha512),ecb(cipher_null))",
3495 .driver_name = "authenc-hmac-sha512-ecb-cipher_null-caam",
3496 .blocksize = NULL_BLOCK_SIZE,
3497 .type = CRYPTO_ALG_TYPE_AEAD,
3498 .template_aead = {
3499 .setkey = aead_setkey,
3500 .setauthsize = aead_setauthsize,
3501 .encrypt = aead_encrypt,
3502 .decrypt = aead_decrypt,
3503 .givencrypt = aead_null_givencrypt,
3504 .geniv = "<built-in>",
3505 .ivsize = NULL_IV_SIZE,
3506 .maxauthsize = SHA512_DIGEST_SIZE,
3507 },
3508 .class1_alg_type = 0,
3509 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3510 OP_ALG_AAI_HMAC_PRECOMP,
3511 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3512 },
8b4d43a4
KP		 3513 {
3514 .name = "authenc(hmac(md5),cbc(aes))",
3515 .driver_name = "authenc-hmac-md5-cbc-aes-caam",
3516 .blocksize = AES_BLOCK_SIZE,
3517 .type = CRYPTO_ALG_TYPE_AEAD,
3518 .template_aead = {
3519 .setkey = aead_setkey,
3520 .setauthsize = aead_setauthsize,
3521 .encrypt = aead_encrypt,
3522 .decrypt = aead_decrypt,
3523 .givencrypt = aead_givencrypt,
3524 .geniv = "<built-in>",
3525 .ivsize = AES_BLOCK_SIZE,
3526 .maxauthsize = MD5_DIGEST_SIZE,
3527 },
3528 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3529 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3530 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3531 },
8e8ec596
KP		 3532 {
3533 .name = "authenc(hmac(sha1),cbc(aes))",
3534 .driver_name = "authenc-hmac-sha1-cbc-aes-caam",
3535 .blocksize = AES_BLOCK_SIZE,
885e9e2f
YK		 3536 .type = CRYPTO_ALG_TYPE_AEAD,
3537 .template_aead = {
0e479300
YK		 3538 .setkey = aead_setkey,
3539 .setauthsize = aead_setauthsize,
3540 .encrypt = aead_encrypt,
3541 .decrypt = aead_decrypt,
3542 .givencrypt = aead_givencrypt,
8e8ec596
KP		 3543 .geniv = "<built-in>",
3544 .ivsize = AES_BLOCK_SIZE,
3545 .maxauthsize = SHA1_DIGEST_SIZE,
3546 },
3547 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3548 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3549 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3550 },
e863f9cc
HA		 3551 {
3552 .name = "authenc(hmac(sha224),cbc(aes))",
3553 .driver_name = "authenc-hmac-sha224-cbc-aes-caam",
3554 .blocksize = AES_BLOCK_SIZE,
cb7d5662 3555 .type = CRYPTO_ALG_TYPE_AEAD,
e863f9cc
HA		 3556 .template_aead = {
3557 .setkey = aead_setkey,
3558 .setauthsize = aead_setauthsize,
3559 .encrypt = aead_encrypt,
3560 .decrypt = aead_decrypt,
3561 .givencrypt = aead_givencrypt,
3562 .geniv = "<built-in>",
3563 .ivsize = AES_BLOCK_SIZE,
3564 .maxauthsize = SHA224_DIGEST_SIZE,
3565 },
3566 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3567 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3568 OP_ALG_AAI_HMAC_PRECOMP,
3569 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3570 },
8e8ec596
KP		 3571 {
3572 .name = "authenc(hmac(sha256),cbc(aes))",
3573 .driver_name = "authenc-hmac-sha256-cbc-aes-caam",
3574 .blocksize = AES_BLOCK_SIZE,
885e9e2f
YK		 3575 .type = CRYPTO_ALG_TYPE_AEAD,
3576 .template_aead = {
0e479300
YK		 3577 .setkey = aead_setkey,
3578 .setauthsize = aead_setauthsize,
3579 .encrypt = aead_encrypt,
3580 .decrypt = aead_decrypt,
3581 .givencrypt = aead_givencrypt,
8e8ec596
KP		 3582 .geniv = "<built-in>",
3583 .ivsize = AES_BLOCK_SIZE,
3584 .maxauthsize = SHA256_DIGEST_SIZE,
3585 },
3586 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3587 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3588 OP_ALG_AAI_HMAC_PRECOMP,
3589 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3590 },
e863f9cc
HA		 3591 {
3592 .name = "authenc(hmac(sha384),cbc(aes))",
3593 .driver_name = "authenc-hmac-sha384-cbc-aes-caam",
3594 .blocksize = AES_BLOCK_SIZE,
cb7d5662 3595 .type = CRYPTO_ALG_TYPE_AEAD,
e863f9cc
HA		 3596 .template_aead = {
3597 .setkey = aead_setkey,
3598 .setauthsize = aead_setauthsize,
3599 .encrypt = aead_encrypt,
3600 .decrypt = aead_decrypt,
3601 .givencrypt = aead_givencrypt,
3602 .geniv = "<built-in>",
3603 .ivsize = AES_BLOCK_SIZE,
3604 .maxauthsize = SHA384_DIGEST_SIZE,
3605 },
3606 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3607 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3608 OP_ALG_AAI_HMAC_PRECOMP,
3609 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3610 },
3611
4427b1b4
KP		 3612 {
3613 .name = "authenc(hmac(sha512),cbc(aes))",
3614 .driver_name = "authenc-hmac-sha512-cbc-aes-caam",
3615 .blocksize = AES_BLOCK_SIZE,
885e9e2f
YK		 3616 .type = CRYPTO_ALG_TYPE_AEAD,
3617 .template_aead = {
0e479300
YK		 3618 .setkey = aead_setkey,
3619 .setauthsize = aead_setauthsize,
3620 .encrypt = aead_encrypt,
3621 .decrypt = aead_decrypt,
3622 .givencrypt = aead_givencrypt,
4427b1b4
KP		 3623 .geniv = "<built-in>",
3624 .ivsize = AES_BLOCK_SIZE,
3625 .maxauthsize = SHA512_DIGEST_SIZE,
3626 },
3627 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
3628 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3629 OP_ALG_AAI_HMAC_PRECOMP,
3630 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3631 },
8b4d43a4
KP		 3632 {
3633 .name = "authenc(hmac(md5),cbc(des3_ede))",
3634 .driver_name = "authenc-hmac-md5-cbc-des3_ede-caam",
3635 .blocksize = DES3_EDE_BLOCK_SIZE,
3636 .type = CRYPTO_ALG_TYPE_AEAD,
3637 .template_aead = {
3638 .setkey = aead_setkey,
3639 .setauthsize = aead_setauthsize,
3640 .encrypt = aead_encrypt,
3641 .decrypt = aead_decrypt,
3642 .givencrypt = aead_givencrypt,
3643 .geniv = "<built-in>",
3644 .ivsize = DES3_EDE_BLOCK_SIZE,
3645 .maxauthsize = MD5_DIGEST_SIZE,
3646 },
3647 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3648 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3649 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3650 },
8e8ec596
KP		 3651 {
3652 .name = "authenc(hmac(sha1),cbc(des3_ede))",
3653 .driver_name = "authenc-hmac-sha1-cbc-des3_ede-caam",
3654 .blocksize = DES3_EDE_BLOCK_SIZE,
885e9e2f
YK		 3655 .type = CRYPTO_ALG_TYPE_AEAD,
3656 .template_aead = {
0e479300
YK		 3657 .setkey = aead_setkey,
3658 .setauthsize = aead_setauthsize,
3659 .encrypt = aead_encrypt,
3660 .decrypt = aead_decrypt,
3661 .givencrypt = aead_givencrypt,
8e8ec596
KP		 3662 .geniv = "<built-in>",
3663 .ivsize = DES3_EDE_BLOCK_SIZE,
3664 .maxauthsize = SHA1_DIGEST_SIZE,
3665 },
3666 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3667 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3668 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3669 },
e863f9cc
HA		 3670 {
3671 .name = "authenc(hmac(sha224),cbc(des3_ede))",
3672 .driver_name = "authenc-hmac-sha224-cbc-des3_ede-caam",
3673 .blocksize = DES3_EDE_BLOCK_SIZE,
cb7d5662 3674 .type = CRYPTO_ALG_TYPE_AEAD,
e863f9cc
HA		 3675 .template_aead = {
3676 .setkey = aead_setkey,
3677 .setauthsize = aead_setauthsize,
3678 .encrypt = aead_encrypt,
3679 .decrypt = aead_decrypt,
3680 .givencrypt = aead_givencrypt,
3681 .geniv = "<built-in>",
3682 .ivsize = DES3_EDE_BLOCK_SIZE,
3683 .maxauthsize = SHA224_DIGEST_SIZE,
3684 },
3685 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3686 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3687 OP_ALG_AAI_HMAC_PRECOMP,
3688 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3689 },
8e8ec596
KP		 3690 {
3691 .name = "authenc(hmac(sha256),cbc(des3_ede))",
3692 .driver_name = "authenc-hmac-sha256-cbc-des3_ede-caam",
3693 .blocksize = DES3_EDE_BLOCK_SIZE,
885e9e2f
YK		 3694 .type = CRYPTO_ALG_TYPE_AEAD,
3695 .template_aead = {
0e479300
YK		 3696 .setkey = aead_setkey,
3697 .setauthsize = aead_setauthsize,
3698 .encrypt = aead_encrypt,
3699 .decrypt = aead_decrypt,
3700 .givencrypt = aead_givencrypt,
8e8ec596
KP		 3701 .geniv = "<built-in>",
3702 .ivsize = DES3_EDE_BLOCK_SIZE,
3703 .maxauthsize = SHA256_DIGEST_SIZE,
3704 },
3705 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3706 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3707 OP_ALG_AAI_HMAC_PRECOMP,
3708 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3709 },
e863f9cc
HA		 3710 {
3711 .name = "authenc(hmac(sha384),cbc(des3_ede))",
3712 .driver_name = "authenc-hmac-sha384-cbc-des3_ede-caam",
3713 .blocksize = DES3_EDE_BLOCK_SIZE,
cb7d5662 3714 .type = CRYPTO_ALG_TYPE_AEAD,
e863f9cc
HA		 3715 .template_aead = {
3716 .setkey = aead_setkey,
3717 .setauthsize = aead_setauthsize,
3718 .encrypt = aead_encrypt,
3719 .decrypt = aead_decrypt,
3720 .givencrypt = aead_givencrypt,
3721 .geniv = "<built-in>",
3722 .ivsize = DES3_EDE_BLOCK_SIZE,
3723 .maxauthsize = SHA384_DIGEST_SIZE,
3724 },
3725 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3726 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3727 OP_ALG_AAI_HMAC_PRECOMP,
3728 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3729 },
4427b1b4
KP		 3730 {
3731 .name = "authenc(hmac(sha512),cbc(des3_ede))",
3732 .driver_name = "authenc-hmac-sha512-cbc-des3_ede-caam",
3733 .blocksize = DES3_EDE_BLOCK_SIZE,
885e9e2f
YK		 3734 .type = CRYPTO_ALG_TYPE_AEAD,
3735 .template_aead = {
0e479300
YK		 3736 .setkey = aead_setkey,
3737 .setauthsize = aead_setauthsize,
3738 .encrypt = aead_encrypt,
3739 .decrypt = aead_decrypt,
3740 .givencrypt = aead_givencrypt,
4427b1b4
KP		 3741 .geniv = "<built-in>",
3742 .ivsize = DES3_EDE_BLOCK_SIZE,
3743 .maxauthsize = SHA512_DIGEST_SIZE,
3744 },
3745 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
3746 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3747 OP_ALG_AAI_HMAC_PRECOMP,
3748 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3749 },
8b4d43a4
KP		 3750 {
3751 .name = "authenc(hmac(md5),cbc(des))",
3752 .driver_name = "authenc-hmac-md5-cbc-des-caam",
3753 .blocksize = DES_BLOCK_SIZE,
3754 .type = CRYPTO_ALG_TYPE_AEAD,
3755 .template_aead = {
3756 .setkey = aead_setkey,
3757 .setauthsize = aead_setauthsize,
3758 .encrypt = aead_encrypt,
3759 .decrypt = aead_decrypt,
3760 .givencrypt = aead_givencrypt,
3761 .geniv = "<built-in>",
3762 .ivsize = DES_BLOCK_SIZE,
3763 .maxauthsize = MD5_DIGEST_SIZE,
3764 },
3765 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3766 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3767 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3768 },
8e8ec596
KP		 3769 {
3770 .name = "authenc(hmac(sha1),cbc(des))",
3771 .driver_name = "authenc-hmac-sha1-cbc-des-caam",
3772 .blocksize = DES_BLOCK_SIZE,
885e9e2f
YK		 3773 .type = CRYPTO_ALG_TYPE_AEAD,
3774 .template_aead = {
0e479300
YK		 3775 .setkey = aead_setkey,
3776 .setauthsize = aead_setauthsize,
3777 .encrypt = aead_encrypt,
3778 .decrypt = aead_decrypt,
3779 .givencrypt = aead_givencrypt,
8e8ec596
KP		 3780 .geniv = "<built-in>",
3781 .ivsize = DES_BLOCK_SIZE,
3782 .maxauthsize = SHA1_DIGEST_SIZE,
3783 },
3784 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3785 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3786 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3787 },
e863f9cc
HA		 3788 {
3789 .name = "authenc(hmac(sha224),cbc(des))",
3790 .driver_name = "authenc-hmac-sha224-cbc-des-caam",
3791 .blocksize = DES_BLOCK_SIZE,
cb7d5662 3792 .type = CRYPTO_ALG_TYPE_AEAD,
e863f9cc
HA		 3793 .template_aead = {
3794 .setkey = aead_setkey,
3795 .setauthsize = aead_setauthsize,
3796 .encrypt = aead_encrypt,
3797 .decrypt = aead_decrypt,
3798 .givencrypt = aead_givencrypt,
3799 .geniv = "<built-in>",
3800 .ivsize = DES_BLOCK_SIZE,
3801 .maxauthsize = SHA224_DIGEST_SIZE,
3802 },
3803 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3804 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3805 OP_ALG_AAI_HMAC_PRECOMP,
3806 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3807 },
8e8ec596
KP		 3808 {
3809 .name = "authenc(hmac(sha256),cbc(des))",
3810 .driver_name = "authenc-hmac-sha256-cbc-des-caam",
3811 .blocksize = DES_BLOCK_SIZE,
885e9e2f
YK		 3812 .type = CRYPTO_ALG_TYPE_AEAD,
3813 .template_aead = {
0e479300
YK		 3814 .setkey = aead_setkey,
3815 .setauthsize = aead_setauthsize,
3816 .encrypt = aead_encrypt,
3817 .decrypt = aead_decrypt,
3818 .givencrypt = aead_givencrypt,
8e8ec596
KP		 3819 .geniv = "<built-in>",
3820 .ivsize = DES_BLOCK_SIZE,
3821 .maxauthsize = SHA256_DIGEST_SIZE,
3822 },
3823 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3824 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3825 OP_ALG_AAI_HMAC_PRECOMP,
3826 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3827 },
e863f9cc
HA		 3828 {
3829 .name = "authenc(hmac(sha384),cbc(des))",
3830 .driver_name = "authenc-hmac-sha384-cbc-des-caam",
3831 .blocksize = DES_BLOCK_SIZE,
cb7d5662 3832 .type = CRYPTO_ALG_TYPE_AEAD,
e863f9cc
HA		 3833 .template_aead = {
3834 .setkey = aead_setkey,
3835 .setauthsize = aead_setauthsize,
3836 .encrypt = aead_encrypt,
3837 .decrypt = aead_decrypt,
3838 .givencrypt = aead_givencrypt,
3839 .geniv = "<built-in>",
3840 .ivsize = DES_BLOCK_SIZE,
3841 .maxauthsize = SHA384_DIGEST_SIZE,
3842 },
3843 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3844 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3845 OP_ALG_AAI_HMAC_PRECOMP,
3846 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3847 },
4427b1b4
KP		 3848 {
3849 .name = "authenc(hmac(sha512),cbc(des))",
3850 .driver_name = "authenc-hmac-sha512-cbc-des-caam",
3851 .blocksize = DES_BLOCK_SIZE,
885e9e2f
YK		 3852 .type = CRYPTO_ALG_TYPE_AEAD,
3853 .template_aead = {
0e479300
YK		 3854 .setkey = aead_setkey,
3855 .setauthsize = aead_setauthsize,
3856 .encrypt = aead_encrypt,
3857 .decrypt = aead_decrypt,
3858 .givencrypt = aead_givencrypt,
4427b1b4
KP		 3859 .geniv = "<built-in>",
3860 .ivsize = DES_BLOCK_SIZE,
3861 .maxauthsize = SHA512_DIGEST_SIZE,
3862 },
3863 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
3864 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3865 OP_ALG_AAI_HMAC_PRECOMP,
3866 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3867 },
daebc465
CV		 3868 {
3869 .name = "authenc(hmac(md5),rfc3686(ctr(aes)))",
3870 .driver_name = "authenc-hmac-md5-rfc3686-ctr-aes-caam",
3871 .blocksize = 1,
3872 .type = CRYPTO_ALG_TYPE_AEAD,
3873 .template_aead = {
3874 .setkey = aead_setkey,
3875 .setauthsize = aead_setauthsize,
3876 .encrypt = aead_encrypt,
3877 .decrypt = aead_decrypt,
3878 .givencrypt = aead_givencrypt,
3879 .geniv = "<built-in>",
3880 .ivsize = CTR_RFC3686_IV_SIZE,
3881 .maxauthsize = MD5_DIGEST_SIZE,
3882 },
3883 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3884 .class2_alg_type = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC_PRECOMP,
3885 .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC,
3886 },
3887 {
3888 .name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
3889 .driver_name = "authenc-hmac-sha1-rfc3686-ctr-aes-caam",
3890 .blocksize = 1,
3891 .type = CRYPTO_ALG_TYPE_AEAD,
3892 .template_aead = {
3893 .setkey = aead_setkey,
3894 .setauthsize = aead_setauthsize,
3895 .encrypt = aead_encrypt,
3896 .decrypt = aead_decrypt,
3897 .givencrypt = aead_givencrypt,
3898 .geniv = "<built-in>",
3899 .ivsize = CTR_RFC3686_IV_SIZE,
3900 .maxauthsize = SHA1_DIGEST_SIZE,
3901 },
3902 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3903 .class2_alg_type = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC_PRECOMP,
3904 .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC,
3905 },
3906 {
3907 .name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
3908 .driver_name = "authenc-hmac-sha224-rfc3686-ctr-aes-caam",
3909 .blocksize = 1,
3910 .type = CRYPTO_ALG_TYPE_AEAD,
3911 .template_aead = {
3912 .setkey = aead_setkey,
3913 .setauthsize = aead_setauthsize,
3914 .encrypt = aead_encrypt,
3915 .decrypt = aead_decrypt,
3916 .givencrypt = aead_givencrypt,
3917 .geniv = "<built-in>",
3918 .ivsize = CTR_RFC3686_IV_SIZE,
3919 .maxauthsize = SHA224_DIGEST_SIZE,
3920 },
3921 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3922 .class2_alg_type = OP_ALG_ALGSEL_SHA224 |
3923 OP_ALG_AAI_HMAC_PRECOMP,
3924 .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC,
3925 },
3926 {
3927 .name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
3928 .driver_name = "authenc-hmac-sha256-rfc3686-ctr-aes-caam",
3929 .blocksize = 1,
3930 .type = CRYPTO_ALG_TYPE_AEAD,
3931 .template_aead = {
3932 .setkey = aead_setkey,
3933 .setauthsize = aead_setauthsize,
3934 .encrypt = aead_encrypt,
3935 .decrypt = aead_decrypt,
3936 .givencrypt = aead_givencrypt,
3937 .geniv = "<built-in>",
3938 .ivsize = CTR_RFC3686_IV_SIZE,
3939 .maxauthsize = SHA256_DIGEST_SIZE,
3940 },
3941 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3942 .class2_alg_type = OP_ALG_ALGSEL_SHA256 |
3943 OP_ALG_AAI_HMAC_PRECOMP,
3944 .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC,
3945 },
3946 {
3947 .name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
3948 .driver_name = "authenc-hmac-sha384-rfc3686-ctr-aes-caam",
3949 .blocksize = 1,
3950 .type = CRYPTO_ALG_TYPE_AEAD,
3951 .template_aead = {
3952 .setkey = aead_setkey,
3953 .setauthsize = aead_setauthsize,
3954 .encrypt = aead_encrypt,
3955 .decrypt = aead_decrypt,
3956 .givencrypt = aead_givencrypt,
3957 .geniv = "<built-in>",
3958 .ivsize = CTR_RFC3686_IV_SIZE,
3959 .maxauthsize = SHA384_DIGEST_SIZE,
3960 },
3961 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3962 .class2_alg_type = OP_ALG_ALGSEL_SHA384 |
3963 OP_ALG_AAI_HMAC_PRECOMP,
3964 .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC,
3965 },
3966 {
3967 .name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
3968 .driver_name = "authenc-hmac-sha512-rfc3686-ctr-aes-caam",
3969 .blocksize = 1,
3970 .type = CRYPTO_ALG_TYPE_AEAD,
3971 .template_aead = {
3972 .setkey = aead_setkey,
3973 .setauthsize = aead_setauthsize,
3974 .encrypt = aead_encrypt,
3975 .decrypt = aead_decrypt,
3976 .givencrypt = aead_givencrypt,
3977 .geniv = "<built-in>",
3978 .ivsize = CTR_RFC3686_IV_SIZE,
3979 .maxauthsize = SHA512_DIGEST_SIZE,
3980 },
3981 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
3982 .class2_alg_type = OP_ALG_ALGSEL_SHA512 |
3983 OP_ALG_AAI_HMAC_PRECOMP,
3984 .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC,
3985 },
bac68f2c
TA		 3986 {
3987 .name = "rfc4106(gcm(aes))",
3988 .driver_name = "rfc4106-gcm-aes-caam",
3989 .blocksize = 1,
3990 .type = CRYPTO_ALG_TYPE_AEAD,
3991 .template_aead = {
3992 .setkey = rfc4106_setkey,
3993 .setauthsize = rfc4106_setauthsize,
3994 .encrypt = aead_encrypt,
3995 .decrypt = aead_decrypt,
3996 .givencrypt = aead_givencrypt,
3997 .geniv = "<built-in>",
3998 .ivsize = 8,
3999 .maxauthsize = AES_BLOCK_SIZE,
4000 },
4001 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4002 },
5d0429a3
TA		 4003 {
4004 .name = "rfc4543(gcm(aes))",
4005 .driver_name = "rfc4543-gcm-aes-caam",
4006 .blocksize = 1,
4007 .type = CRYPTO_ALG_TYPE_AEAD,
4008 .template_aead = {
4009 .setkey = rfc4543_setkey,
4010 .setauthsize = rfc4543_setauthsize,
4011 .encrypt = aead_encrypt,
4012 .decrypt = aead_decrypt,
4013 .givencrypt = aead_givencrypt,
4014 .geniv = "<built-in>",
4015 .ivsize = 8,
4016 .maxauthsize = AES_BLOCK_SIZE,
4017 },
4018 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4019 },
3ef8d945
TA		 4020 /* Galois Counter Mode */
4021 {
4022 .name = "gcm(aes)",
4023 .driver_name = "gcm-aes-caam",
4024 .blocksize = 1,
4025 .type = CRYPTO_ALG_TYPE_AEAD,
4026 .template_aead = {
4027 .setkey = gcm_setkey,
4028 .setauthsize = gcm_setauthsize,
4029 .encrypt = aead_encrypt,
4030 .decrypt = aead_decrypt,
4031 .givencrypt = NULL,
4032 .geniv = "<built-in>",
4033 .ivsize = 12,
4034 .maxauthsize = AES_BLOCK_SIZE,
4035 },
4036 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM,
4037 },
acdca31d
YK		 4038 /* ablkcipher descriptor */
4039 {
4040 .name = "cbc(aes)",
4041 .driver_name = "cbc-aes-caam",
4042 .blocksize = AES_BLOCK_SIZE,
7222d1a3 4043 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
acdca31d
YK		 4044 .template_ablkcipher = {
4045 .setkey = ablkcipher_setkey,
4046 .encrypt = ablkcipher_encrypt,
4047 .decrypt = ablkcipher_decrypt,
7222d1a3
CV		 4048 .givencrypt = ablkcipher_givencrypt,
4049 .geniv = "<built-in>",
acdca31d
YK		 4050 .min_keysize = AES_MIN_KEY_SIZE,
4051 .max_keysize = AES_MAX_KEY_SIZE,
4052 .ivsize = AES_BLOCK_SIZE,
4053 },
4054 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC,
4055 },
4056 {
4057 .name = "cbc(des3_ede)",
4058 .driver_name = "cbc-3des-caam",
4059 .blocksize = DES3_EDE_BLOCK_SIZE,
ff2c3a3b 4060 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
acdca31d
YK		 4061 .template_ablkcipher = {
4062 .setkey = ablkcipher_setkey,
4063 .encrypt = ablkcipher_encrypt,
4064 .decrypt = ablkcipher_decrypt,
ff2c3a3b
CV		 4065 .givencrypt = ablkcipher_givencrypt,
4066 .geniv = "<built-in>",
acdca31d
YK		 4067 .min_keysize = DES3_EDE_KEY_SIZE,
4068 .max_keysize = DES3_EDE_KEY_SIZE,
4069 .ivsize = DES3_EDE_BLOCK_SIZE,
4070 },
4071 .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC,
4072 },
4073 {
4074 .name = "cbc(des)",
4075 .driver_name = "cbc-des-caam",
4076 .blocksize = DES_BLOCK_SIZE,
ff2c3a3b 4077 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
acdca31d
YK		 4078 .template_ablkcipher = {
4079 .setkey = ablkcipher_setkey,
4080 .encrypt = ablkcipher_encrypt,
4081 .decrypt = ablkcipher_decrypt,
ff2c3a3b
CV		 4082 .givencrypt = ablkcipher_givencrypt,
4083 .geniv = "<built-in>",
acdca31d
YK		 4084 .min_keysize = DES_KEY_SIZE,
4085 .max_keysize = DES_KEY_SIZE,
4086 .ivsize = DES_BLOCK_SIZE,
4087 },
4088 .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC,
2b22f6c5
CV		 4089 },
4090 {
4091 .name = "ctr(aes)",
4092 .driver_name = "ctr-aes-caam",
4093 .blocksize = 1,
4094 .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
4095 .template_ablkcipher = {
4096 .setkey = ablkcipher_setkey,
4097 .encrypt = ablkcipher_encrypt,
4098 .decrypt = ablkcipher_decrypt,
4099 .geniv = "chainiv",
4100 .min_keysize = AES_MIN_KEY_SIZE,
4101 .max_keysize = AES_MAX_KEY_SIZE,
4102 .ivsize = AES_BLOCK_SIZE,
4103 },
4104 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
a5f57cff
CV		 4105 },
4106 {
4107 .name = "rfc3686(ctr(aes))",
4108 .driver_name = "rfc3686-ctr-aes-caam",
4109 .blocksize = 1,
7222d1a3 4110 .type = CRYPTO_ALG_TYPE_GIVCIPHER,
a5f57cff
CV		 4111 .template_ablkcipher = {
4112 .setkey = ablkcipher_setkey,
4113 .encrypt = ablkcipher_encrypt,
4114 .decrypt = ablkcipher_decrypt,
7222d1a3
CV		 4115 .givencrypt = ablkcipher_givencrypt,
4116 .geniv = "<built-in>",
a5f57cff
CV		 4117 .min_keysize = AES_MIN_KEY_SIZE +
4118 CTR_RFC3686_NONCE_SIZE,
4119 .max_keysize = AES_MAX_KEY_SIZE +
4120 CTR_RFC3686_NONCE_SIZE,
4121 .ivsize = CTR_RFC3686_IV_SIZE,
4122 },
4123 .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128,
acdca31d 4124 }
8e8ec596
KP		 4125};
4126
4127struct caam_crypto_alg {
4128 struct list_head entry;
8e8ec596
KP		 4129 int class1_alg_type;
4130 int class2_alg_type;
4131 int alg_op;
4132 struct crypto_alg crypto_alg;
4133};
4134
4135static int caam_cra_init(struct crypto_tfm *tfm)
4136{
4137 struct crypto_alg *alg = tfm->__crt_alg;
4138 struct caam_crypto_alg *caam_alg =
4139 container_of(alg, struct caam_crypto_alg, crypto_alg);
4140 struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
8e8ec596 4141
cfc6f11b
RG		 4142 ctx->jrdev = caam_jr_alloc();
4143 if (IS_ERR(ctx->jrdev)) {
4144 pr_err("Job Ring Device allocation for transform failed\n");
4145 return PTR_ERR(ctx->jrdev);
4146 }
8e8ec596
KP		 4147
4148 /* copy descriptor header template value */
4149 ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam_alg->class1_alg_type;
4150 ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam_alg->class2_alg_type;
4151 ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_alg->alg_op;
4152
4153 return 0;
4154}
4155
4156static void caam_cra_exit(struct crypto_tfm *tfm)
4157{
4158 struct caam_ctx *ctx = crypto_tfm_ctx(tfm);
4159
1acebad3
YK		 4160 if (ctx->sh_desc_enc_dma &&
4161 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma))
4162 dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma,
4163 desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE);
4164 if (ctx->sh_desc_dec_dma &&
4165 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma))
4166 dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma,
4167 desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE);
4168 if (ctx->sh_desc_givenc_dma &&
4169 !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma))
4170 dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma,
4171 desc_bytes(ctx->sh_desc_givenc),
4427b1b4 4172 DMA_TO_DEVICE);
ec31eed7
HG		 4173 if (ctx->key_dma &&
4174 !dma_mapping_error(ctx->jrdev, ctx->key_dma))
4175 dma_unmap_single(ctx->jrdev, ctx->key_dma,
4176 ctx->enckeylen + ctx->split_key_pad_len,
4177 DMA_TO_DEVICE);
cfc6f11b
RG		 4178
4179 caam_jr_free(ctx->jrdev);
8e8ec596
KP		 4180}
4181
4182static void __exit caam_algapi_exit(void)
4183{
4184
8e8ec596 4185 struct caam_crypto_alg *t_alg, *n;
8e8ec596 4186
cfc6f11b 4187 if (!alg_list.next)
8e8ec596
KP		 4188 return;
4189
cfc6f11b 4190 list_for_each_entry_safe(t_alg, n, &alg_list, entry) {
8e8ec596
KP		 4191 crypto_unregister_alg(&t_alg->crypto_alg);
4192 list_del(&t_alg->entry);
4193 kfree(t_alg);
4194 }
8e8ec596
KP		 4195}
4196
cfc6f11b 4197static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template
8e8ec596
KP		 4198 *template)
4199{
4200 struct caam_crypto_alg *t_alg;
4201 struct crypto_alg *alg;
4202
4203 t_alg = kzalloc(sizeof(struct caam_crypto_alg), GFP_KERNEL);
4204 if (!t_alg) {
cfc6f11b 4205 pr_err("failed to allocate t_alg\n");
8e8ec596
KP		 4206 return ERR_PTR(-ENOMEM);
4207 }
4208
4209 alg = &t_alg->crypto_alg;
4210
4211 snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name);
4212 snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s",
4213 template->driver_name);
4214 alg->cra_module = THIS_MODULE;
4215 alg->cra_init = caam_cra_init;
4216 alg->cra_exit = caam_cra_exit;
4217 alg->cra_priority = CAAM_CRA_PRIORITY;
8e8ec596
KP		 4218 alg->cra_blocksize = template->blocksize;
4219 alg->cra_alignmask = 0;
8e8ec596 4220 alg->cra_ctxsize = sizeof(struct caam_ctx);
d912bb76
NM		 4221 alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY |
4222 template->type;
885e9e2f 4223 switch (template->type) {
7222d1a3
CV		 4224 case CRYPTO_ALG_TYPE_GIVCIPHER:
4225 alg->cra_type = &crypto_givcipher_type;
4226 alg->cra_ablkcipher = template->template_ablkcipher;
4227 break;
acdca31d
YK		 4228 case CRYPTO_ALG_TYPE_ABLKCIPHER:
4229 alg->cra_type = &crypto_ablkcipher_type;
4230 alg->cra_ablkcipher = template->template_ablkcipher;
4231 break;
885e9e2f
YK		 4232 case CRYPTO_ALG_TYPE_AEAD:
4233 alg->cra_type = &crypto_aead_type;
4234 alg->cra_aead = template->template_aead;
4235 break;
4236 }
8e8ec596
KP		 4237
4238 t_alg->class1_alg_type = template->class1_alg_type;
4239 t_alg->class2_alg_type = template->class2_alg_type;
4240 t_alg->alg_op = template->alg_op;
8e8ec596
KP		 4241
4242 return t_alg;
4243}
4244
4245static int __init caam_algapi_init(void)
4246{
35af6403
RG		 4247 struct device_node *dev_node;
4248 struct platform_device *pdev;
4249 struct device *ctrldev;
4250 void *priv;
8e8ec596
KP		 4251 int i = 0, err = 0;
4252
35af6403
RG		 4253 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0");
4254 if (!dev_node) {
4255 dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0");
4256 if (!dev_node)
4257 return -ENODEV;
4258 }
4259
4260 pdev = of_find_device_by_node(dev_node);
4261 if (!pdev) {
4262 of_node_put(dev_node);
4263 return -ENODEV;
4264 }
4265
4266 ctrldev = &pdev->dev;
4267 priv = dev_get_drvdata(ctrldev);
4268 of_node_put(dev_node);
4269
4270 /*
4271 * If priv is NULL, it's probably because the caam driver wasn't
4272 * properly initialized (e.g. RNG4 init failed). Thus, bail out here.
4273 */
4274 if (!priv)
4275 return -ENODEV;
4276
4277
cfc6f11b 4278 INIT_LIST_HEAD(&alg_list);
8e8ec596
KP		 4279
4280 /* register crypto algorithms the device supports */
4281 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4282 /* TODO: check if h/w supports alg */
4283 struct caam_crypto_alg *t_alg;
4284
cfc6f11b 4285 t_alg = caam_alg_alloc(&driver_algs[i]);
8e8ec596
KP		 4286 if (IS_ERR(t_alg)) {
4287 err = PTR_ERR(t_alg);
cfc6f11b
RG		 4288 pr_warn("%s alg allocation failed\n",
4289 driver_algs[i].driver_name);
8e8ec596
KP		 4290 continue;
4291 }
4292
4293 err = crypto_register_alg(&t_alg->crypto_alg);
4294 if (err) {
cfc6f11b 4295 pr_warn("%s alg registration failed\n",
8e8ec596
KP		 4296 t_alg->crypto_alg.cra_driver_name);
4297 kfree(t_alg);
246bbedb 4298 } else
cfc6f11b 4299 list_add_tail(&t_alg->entry, &alg_list);
8e8ec596 4300 }
cfc6f11b
RG		 4301 if (!list_empty(&alg_list))
4302 pr_info("caam algorithms registered in /proc/crypto\n");
8e8ec596
KP		 4303
4304 return err;
4305}
4306
4307module_init(caam_algapi_init);
4308module_exit(caam_algapi_exit);
4309
4310MODULE_LICENSE("GPL");
4311MODULE_DESCRIPTION("FSL CAAM support for crypto API");
4312MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
Ubuntu Artful kernel mirror