]> git.proxmox.com Git - mirror_ubuntu-bionic-kernel.git/blob - drivers/crypto/caam/caamalg_desc.c
projects / mirror_ubuntu-bionic-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mszeredi...
[mirror_ubuntu-bionic-kernel.git] / drivers / crypto / caam / caamalg_desc.c
1 /*
2 * Shared descriptors for aead, ablkcipher algorithms
3 *
4 * Copyright 2016 NXP
5 */
6
7 #include "compat.h"
8 #include "desc_constr.h"
9 #include "caamalg_desc.h"
10
11 /*
12 * For aead functions, read payload and write payload,
13 * both of which are specified in req->src and req->dst
14 */
15 static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
16 {
17 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
18 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
19 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
20 }
21
22 /* Set DK bit in class 1 operation if shared */
23 static inline void append_dec_op1(u32 *desc, u32 type)
24 {
25 u32 *jump_cmd, *uncond_jump_cmd;
26
27 /* DK bit is valid only for AES */
28 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
29 append_operation(desc, type | OP_ALG_AS_INITFINAL |
30 OP_ALG_DECRYPT);
31 return;
32 }
33
34 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
35 append_operation(desc, type | OP_ALG_AS_INITFINAL |
36 OP_ALG_DECRYPT);
37 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
38 set_jump_tgt_here(desc, jump_cmd);
39 append_operation(desc, type | OP_ALG_AS_INITFINAL |
40 OP_ALG_DECRYPT | OP_ALG_AAI_DK);
41 set_jump_tgt_here(desc, uncond_jump_cmd);
42 }
43
44 /**
45 * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor
46 * (non-protocol) with no (null) encryption.
47 * @desc: pointer to buffer used for descriptor construction
48 * @adata: pointer to authentication transform definitions. Note that since a
49 * split key is to be used, the size of the split key itself is
50 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
51 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
52 * @icvsize: integrity check value (ICV) size (truncated or full)
53 *
54 * Note: Requires an MDHA split key.
55 */
56 void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
57 unsigned int icvsize)
58 {
59 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
60
61 init_sh_desc(desc, HDR_SHARE_SERIAL);
62
63 /* Skip if already shared */
64 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
65 JUMP_COND_SHRD);
66 if (adata->key_inline)
67 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
68 adata->keylen, CLASS_2 | KEY_DEST_MDHA_SPLIT |
69 KEY_ENC);
70 else
71 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
72 KEY_DEST_MDHA_SPLIT | KEY_ENC);
73 set_jump_tgt_here(desc, key_jump_cmd);
74
75 /* assoclen + cryptlen = seqinlen */
76 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
77
78 /* Prepare to read and write cryptlen + assoclen bytes */
79 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
80 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
81
82 /*
83 * MOVE_LEN opcode is not available in all SEC HW revisions,
84 * thus need to do some magic, i.e. self-patch the descriptor
85 * buffer.
86 */
87 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
88 MOVE_DEST_MATH3 |
89 (0x6 << MOVE_LEN_SHIFT));
90 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
91 MOVE_DEST_DESCBUF |
92 MOVE_WAITCOMP |
93 (0x8 << MOVE_LEN_SHIFT));
94
95 /* Class 2 operation */
96 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
97 OP_ALG_ENCRYPT);
98
99 /* Read and write cryptlen bytes */
100 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
101
102 set_move_tgt_here(desc, read_move_cmd);
103 set_move_tgt_here(desc, write_move_cmd);
104 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
105 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
106 MOVE_AUX_LS);
107
108 /* Write ICV */
109 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
110 LDST_SRCDST_BYTE_CONTEXT);
111
112 #ifdef DEBUG
113 print_hex_dump(KERN_ERR,
114 "aead null enc shdesc@" __stringify(__LINE__)": ",
115 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
116 #endif
117 }
118 EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
119
120 /**
121 * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor
122 * (non-protocol) with no (null) decryption.
123 * @desc: pointer to buffer used for descriptor construction
124 * @adata: pointer to authentication transform definitions. Note that since a
125 * split key is to be used, the size of the split key itself is
126 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
127 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
128 * @icvsize: integrity check value (ICV) size (truncated or full)
129 *
130 * Note: Requires an MDHA split key.
131 */
132 void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
133 unsigned int icvsize)
134 {
135 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
136
137 init_sh_desc(desc, HDR_SHARE_SERIAL);
138
139 /* Skip if already shared */
140 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
141 JUMP_COND_SHRD);
142 if (adata->key_inline)
143 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
144 adata->keylen, CLASS_2 |
145 KEY_DEST_MDHA_SPLIT | KEY_ENC);
146 else
147 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
148 KEY_DEST_MDHA_SPLIT | KEY_ENC);
149 set_jump_tgt_here(desc, key_jump_cmd);
150
151 /* Class 2 operation */
152 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
153 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
154
155 /* assoclen + cryptlen = seqoutlen */
156 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
157
158 /* Prepare to read and write cryptlen + assoclen bytes */
159 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
160 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
161
162 /*
163 * MOVE_LEN opcode is not available in all SEC HW revisions,
164 * thus need to do some magic, i.e. self-patch the descriptor
165 * buffer.
166 */
167 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
168 MOVE_DEST_MATH2 |
169 (0x6 << MOVE_LEN_SHIFT));
170 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
171 MOVE_DEST_DESCBUF |
172 MOVE_WAITCOMP |
173 (0x8 << MOVE_LEN_SHIFT));
174
175 /* Read and write cryptlen bytes */
176 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
177
178 /*
179 * Insert a NOP here, since we need at least 4 instructions between
180 * code patching the descriptor buffer and the location being patched.
181 */
182 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
183 set_jump_tgt_here(desc, jump_cmd);
184
185 set_move_tgt_here(desc, read_move_cmd);
186 set_move_tgt_here(desc, write_move_cmd);
187 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
188 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
189 MOVE_AUX_LS);
190 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
191
192 /* Load ICV */
193 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
194 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
195
196 #ifdef DEBUG
197 print_hex_dump(KERN_ERR,
198 "aead null dec shdesc@" __stringify(__LINE__)": ",
199 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
200 #endif
201 }
202 EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
203
204 static void init_sh_desc_key_aead(u32 * const desc,
205 struct alginfo * const cdata,
206 struct alginfo * const adata,
207 const bool is_rfc3686, u32 *nonce)
208 {
209 u32 *key_jump_cmd;
210 unsigned int enckeylen = cdata->keylen;
211
212 /* Note: Context registers are saved. */
213 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
214
215 /* Skip if already shared */
216 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
217 JUMP_COND_SHRD);
218
219 /*
220 * RFC3686 specific:
221 * | key = {AUTH_KEY, ENC_KEY, NONCE}
222 * | enckeylen = encryption key size + nonce size
223 */
224 if (is_rfc3686)
225 enckeylen -= CTR_RFC3686_NONCE_SIZE;
226
227 if (adata->key_inline)
228 append_key_as_imm(desc, adata->key_virt, adata->keylen_pad,
229 adata->keylen, CLASS_2 |
230 KEY_DEST_MDHA_SPLIT | KEY_ENC);
231 else
232 append_key(desc, adata->key_dma, adata->keylen, CLASS_2 |
233 KEY_DEST_MDHA_SPLIT | KEY_ENC);
234
235 if (cdata->key_inline)
236 append_key_as_imm(desc, cdata->key_virt, enckeylen,
237 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
238 else
239 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
240 KEY_DEST_CLASS_REG);
241
242 /* Load Counter into CONTEXT1 reg */
243 if (is_rfc3686) {
244 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
245 LDST_CLASS_IND_CCB |
246 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
247 append_move(desc,
248 MOVE_SRC_OUTFIFO |
249 MOVE_DEST_CLASS1CTX |
250 (16 << MOVE_OFFSET_SHIFT) |
251 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
252 }
253
254 set_jump_tgt_here(desc, key_jump_cmd);
255 }
256
257 /**
258 * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor
259 * (non-protocol).
260 * @desc: pointer to buffer used for descriptor construction
261 * @cdata: pointer to block cipher transform definitions
262 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
263 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
264 * @adata: pointer to authentication transform definitions. Note that since a
265 * split key is to be used, the size of the split key itself is
266 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
267 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
268 * @icvsize: integrity check value (ICV) size (truncated or full)
269 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
270 * @nonce: pointer to rfc3686 nonce
271 * @ctx1_iv_off: IV offset in CONTEXT1 register
272 *
273 * Note: Requires an MDHA split key.
274 */
275 void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
276 struct alginfo *adata, unsigned int icvsize,
277 const bool is_rfc3686, u32 *nonce,
278 const u32 ctx1_iv_off)
279 {
280 /* Note: Context registers are saved. */
281 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
282
283 /* Class 2 operation */
284 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
285 OP_ALG_ENCRYPT);
286
287 /* Read and write assoclen bytes */
288 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
289 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
290
291 /* Skip assoc data */
292 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
293
294 /* read assoc before reading payload */
295 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
296 FIFOLDST_VLF);
297
298 /* Load Counter into CONTEXT1 reg */
299 if (is_rfc3686)
300 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
301 LDST_SRCDST_BYTE_CONTEXT |
302 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
303 LDST_OFFSET_SHIFT));
304
305 /* Class 1 operation */
306 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
307 OP_ALG_ENCRYPT);
308
309 /* Read and write cryptlen bytes */
310 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
311 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
312 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
313
314 /* Write ICV */
315 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
316 LDST_SRCDST_BYTE_CONTEXT);
317
318 #ifdef DEBUG
319 print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ",
320 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
321 #endif
322 }
323 EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
324
325 /**
326 * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor
327 * (non-protocol).
328 * @desc: pointer to buffer used for descriptor construction
329 * @cdata: pointer to block cipher transform definitions
330 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
331 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
332 * @adata: pointer to authentication transform definitions. Note that since a
333 * split key is to be used, the size of the split key itself is
334 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
335 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
336 * @ivsize: initialization vector size
337 * @icvsize: integrity check value (ICV) size (truncated or full)
338 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
339 * @nonce: pointer to rfc3686 nonce
340 * @ctx1_iv_off: IV offset in CONTEXT1 register
341 *
342 * Note: Requires an MDHA split key.
343 */
344 void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
345 struct alginfo *adata, unsigned int ivsize,
346 unsigned int icvsize, const bool geniv,
347 const bool is_rfc3686, u32 *nonce,
348 const u32 ctx1_iv_off)
349 {
350 /* Note: Context registers are saved. */
351 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
352
353 /* Class 2 operation */
354 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
355 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
356
357 /* Read and write assoclen bytes */
358 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
359 if (geniv)
360 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize);
361 else
362 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
363
364 /* Skip assoc data */
365 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
366
367 /* read assoc before reading payload */
368 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
369 KEY_VLF);
370
371 if (geniv) {
372 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
373 LDST_SRCDST_BYTE_CONTEXT |
374 (ctx1_iv_off << LDST_OFFSET_SHIFT));
375 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
376 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
377 }
378
379 /* Load Counter into CONTEXT1 reg */
380 if (is_rfc3686)
381 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
382 LDST_SRCDST_BYTE_CONTEXT |
383 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
384 LDST_OFFSET_SHIFT));
385
386 /* Choose operation */
387 if (ctx1_iv_off)
388 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
389 OP_ALG_DECRYPT);
390 else
391 append_dec_op1(desc, cdata->algtype);
392
393 /* Read and write cryptlen bytes */
394 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
395 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
396 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
397
398 /* Load ICV */
399 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
400 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
401
402 #ifdef DEBUG
403 print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ",
404 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
405 #endif
406 }
407 EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
408
409 /**
410 * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor
411 * (non-protocol) with HW-generated initialization
412 * vector.
413 * @desc: pointer to buffer used for descriptor construction
414 * @cdata: pointer to block cipher transform definitions
415 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
416 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
417 * @adata: pointer to authentication transform definitions. Note that since a
418 * split key is to be used, the size of the split key itself is
419 * specified. Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1,
420 * SHA224, SHA256, SHA384, SHA512} ANDed with OP_ALG_AAI_HMAC_PRECOMP.
421 * @ivsize: initialization vector size
422 * @icvsize: integrity check value (ICV) size (truncated or full)
423 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
424 * @nonce: pointer to rfc3686 nonce
425 * @ctx1_iv_off: IV offset in CONTEXT1 register
426 *
427 * Note: Requires an MDHA split key.
428 */
429 void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
430 struct alginfo *adata, unsigned int ivsize,
431 unsigned int icvsize, const bool is_rfc3686,
432 u32 *nonce, const u32 ctx1_iv_off)
433 {
434 u32 geniv, moveiv;
435
436 /* Note: Context registers are saved. */
437 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce);
438
439 if (is_rfc3686)
440 goto copy_iv;
441
442 /* Generate IV */
443 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
444 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
445 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
446 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
447 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
448 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
449 append_move(desc, MOVE_WAITCOMP |
450 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
451 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
452 (ivsize << MOVE_LEN_SHIFT));
453 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
454
455 copy_iv:
456 /* Copy IV to class 1 context */
457 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
458 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
459 (ivsize << MOVE_LEN_SHIFT));
460
461 /* Return to encryption */
462 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
463 OP_ALG_ENCRYPT);
464
465 /* Read and write assoclen bytes */
466 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
467 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
468
469 /* Skip assoc data */
470 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
471
472 /* read assoc before reading payload */
473 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
474 KEY_VLF);
475
476 /* Copy iv from outfifo to class 2 fifo */
477 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
478 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
479 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
480 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
481 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
482 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
483
484 /* Load Counter into CONTEXT1 reg */
485 if (is_rfc3686)
486 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
487 LDST_SRCDST_BYTE_CONTEXT |
488 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
489 LDST_OFFSET_SHIFT));
490
491 /* Class 1 operation */
492 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
493 OP_ALG_ENCRYPT);
494
495 /* Will write ivsize + cryptlen */
496 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
497
498 /* Not need to reload iv */
499 append_seq_fifo_load(desc, ivsize,
500 FIFOLD_CLASS_SKIP);
501
502 /* Will read cryptlen */
503 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
504 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
505 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
506 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
507
508 /* Write ICV */
509 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
510 LDST_SRCDST_BYTE_CONTEXT);
511
512 #ifdef DEBUG
513 print_hex_dump(KERN_ERR,
514 "aead givenc shdesc@" __stringify(__LINE__)": ",
515 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
516 #endif
517 }
518 EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
519
520 /**
521 * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor
522 * @desc: pointer to buffer used for descriptor construction
523 * @cdata: pointer to block cipher transform definitions
524 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
525 * @icvsize: integrity check value (ICV) size (truncated or full)
526 */
527 void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
528 unsigned int icvsize)
529 {
530 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
531 *zero_assoc_jump_cmd2;
532
533 init_sh_desc(desc, HDR_SHARE_SERIAL);
534
535 /* skip key loading if they are loaded due to sharing */
536 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
537 JUMP_COND_SHRD | JUMP_COND_SELF);
538 if (cdata->key_inline)
539 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
540 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
541 else
542 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
543 KEY_DEST_CLASS_REG);
544 set_jump_tgt_here(desc, key_jump_cmd);
545
546 /* class 1 operation */
547 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
548 OP_ALG_ENCRYPT);
549
550 /* if assoclen + cryptlen is ZERO, skip to ICV write */
551 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
552 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
553 JUMP_COND_MATH_Z);
554
555 /* if assoclen is ZERO, skip reading the assoc data */
556 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
557 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
558 JUMP_COND_MATH_Z);
559
560 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
561
562 /* skip assoc data */
563 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
564
565 /* cryptlen = seqinlen - assoclen */
566 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
567
568 /* if cryptlen is ZERO jump to zero-payload commands */
569 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
570 JUMP_COND_MATH_Z);
571
572 /* read assoc data */
573 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
574 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
575 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
576
577 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
578
579 /* write encrypted data */
580 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
581
582 /* read payload data */
583 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
584 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
585
586 /* jump the zero-payload commands */
587 append_jump(desc, JUMP_TEST_ALL | 2);
588
589 /* zero-payload commands */
590 set_jump_tgt_here(desc, zero_payload_jump_cmd);
591
592 /* read assoc data */
593 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
594 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
595
596 /* There is no input data */
597 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
598
599 /* write ICV */
600 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
601 LDST_SRCDST_BYTE_CONTEXT);
602
603 #ifdef DEBUG
604 print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ",
605 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
606 #endif
607 }
608 EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
609
610 /**
611 * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor
612 * @desc: pointer to buffer used for descriptor construction
613 * @cdata: pointer to block cipher transform definitions
614 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
615 * @icvsize: integrity check value (ICV) size (truncated or full)
616 */
617 void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
618 unsigned int icvsize)
619 {
620 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
621
622 init_sh_desc(desc, HDR_SHARE_SERIAL);
623
624 /* skip key loading if they are loaded due to sharing */
625 key_jump_cmd = append_jump(desc, JUMP_JSL |
626 JUMP_TEST_ALL | JUMP_COND_SHRD |
627 JUMP_COND_SELF);
628 if (cdata->key_inline)
629 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
630 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
631 else
632 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
633 KEY_DEST_CLASS_REG);
634 set_jump_tgt_here(desc, key_jump_cmd);
635
636 /* class 1 operation */
637 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
638 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
639
640 /* if assoclen is ZERO, skip reading the assoc data */
641 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
642 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
643 JUMP_COND_MATH_Z);
644
645 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
646
647 /* skip assoc data */
648 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
649
650 /* read assoc data */
651 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
652 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
653
654 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
655
656 /* cryptlen = seqoutlen - assoclen */
657 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
658
659 /* jump to zero-payload command if cryptlen is zero */
660 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
661 JUMP_COND_MATH_Z);
662
663 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
664
665 /* store encrypted data */
666 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
667
668 /* read payload data */
669 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
670 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
671
672 /* zero-payload command */
673 set_jump_tgt_here(desc, zero_payload_jump_cmd);
674
675 /* read ICV */
676 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
677 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
678
679 #ifdef DEBUG
680 print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ",
681 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
682 #endif
683 }
684 EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
685
686 /**
687 * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor
688 * (non-protocol).
689 * @desc: pointer to buffer used for descriptor construction
690 * @cdata: pointer to block cipher transform definitions
691 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
692 * @icvsize: integrity check value (ICV) size (truncated or full)
693 */
694 void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
695 unsigned int icvsize)
696 {
697 u32 *key_jump_cmd;
698
699 init_sh_desc(desc, HDR_SHARE_SERIAL);
700
701 /* Skip key loading if it is loaded due to sharing */
702 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
703 JUMP_COND_SHRD);
704 if (cdata->key_inline)
705 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
706 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
707 else
708 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
709 KEY_DEST_CLASS_REG);
710 set_jump_tgt_here(desc, key_jump_cmd);
711
712 /* Class 1 operation */
713 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
714 OP_ALG_ENCRYPT);
715
716 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
717 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
718
719 /* Read assoc data */
720 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
721 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
722
723 /* Skip IV */
724 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
725
726 /* Will read cryptlen bytes */
727 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
728
729 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
730 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
731
732 /* Skip assoc data */
733 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
734
735 /* cryptlen = seqoutlen - assoclen */
736 append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ);
737
738 /* Write encrypted data */
739 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
740
741 /* Read payload data */
742 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
743 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
744
745 /* Write ICV */
746 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
747 LDST_SRCDST_BYTE_CONTEXT);
748
749 #ifdef DEBUG
750 print_hex_dump(KERN_ERR,
751 "rfc4106 enc shdesc@" __stringify(__LINE__)": ",
752 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
753 #endif
754 }
755 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
756
757 /**
758 * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor
759 * (non-protocol).
760 * @desc: pointer to buffer used for descriptor construction
761 * @cdata: pointer to block cipher transform definitions
762 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
763 * @icvsize: integrity check value (ICV) size (truncated or full)
764 */
765 void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
766 unsigned int icvsize)
767 {
768 u32 *key_jump_cmd;
769
770 init_sh_desc(desc, HDR_SHARE_SERIAL);
771
772 /* Skip key loading if it is loaded due to sharing */
773 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
774 JUMP_COND_SHRD);
775 if (cdata->key_inline)
776 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
777 cdata->keylen, CLASS_1 |
778 KEY_DEST_CLASS_REG);
779 else
780 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
781 KEY_DEST_CLASS_REG);
782 set_jump_tgt_here(desc, key_jump_cmd);
783
784 /* Class 1 operation */
785 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
786 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
787
788 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8);
789 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
790
791 /* Read assoc data */
792 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
793 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
794
795 /* Skip IV */
796 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
797
798 /* Will read cryptlen bytes */
799 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
800
801 /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */
802 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
803
804 /* Skip assoc data */
805 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
806
807 /* Will write cryptlen bytes */
808 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
809
810 /* Store payload data */
811 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
812
813 /* Read encrypted data */
814 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
815 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
816
817 /* Read ICV */
818 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
819 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
820
821 #ifdef DEBUG
822 print_hex_dump(KERN_ERR,
823 "rfc4106 dec shdesc@" __stringify(__LINE__)": ",
824 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
825 #endif
826 }
827 EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
828
829 /**
830 * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor
831 * (non-protocol).
832 * @desc: pointer to buffer used for descriptor construction
833 * @cdata: pointer to block cipher transform definitions
834 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
835 * @icvsize: integrity check value (ICV) size (truncated or full)
836 */
837 void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
838 unsigned int icvsize)
839 {
840 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
841
842 init_sh_desc(desc, HDR_SHARE_SERIAL);
843
844 /* Skip key loading if it is loaded due to sharing */
845 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
846 JUMP_COND_SHRD);
847 if (cdata->key_inline)
848 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
849 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
850 else
851 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
852 KEY_DEST_CLASS_REG);
853 set_jump_tgt_here(desc, key_jump_cmd);
854
855 /* Class 1 operation */
856 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
857 OP_ALG_ENCRYPT);
858
859 /* assoclen + cryptlen = seqinlen */
860 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
861
862 /*
863 * MOVE_LEN opcode is not available in all SEC HW revisions,
864 * thus need to do some magic, i.e. self-patch the descriptor
865 * buffer.
866 */
867 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
868 (0x6 << MOVE_LEN_SHIFT));
869 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
870 (0x8 << MOVE_LEN_SHIFT));
871
872 /* Will read assoclen + cryptlen bytes */
873 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
874
875 /* Will write assoclen + cryptlen bytes */
876 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
877
878 /* Read and write assoclen + cryptlen bytes */
879 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
880
881 set_move_tgt_here(desc, read_move_cmd);
882 set_move_tgt_here(desc, write_move_cmd);
883 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
884 /* Move payload data to OFIFO */
885 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
886
887 /* Write ICV */
888 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
889 LDST_SRCDST_BYTE_CONTEXT);
890
891 #ifdef DEBUG
892 print_hex_dump(KERN_ERR,
893 "rfc4543 enc shdesc@" __stringify(__LINE__)": ",
894 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
895 #endif
896 }
897 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
898
899 /**
900 * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor
901 * (non-protocol).
902 * @desc: pointer to buffer used for descriptor construction
903 * @cdata: pointer to block cipher transform definitions
904 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM.
905 * @icvsize: integrity check value (ICV) size (truncated or full)
906 */
907 void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
908 unsigned int icvsize)
909 {
910 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
911
912 init_sh_desc(desc, HDR_SHARE_SERIAL);
913
914 /* Skip key loading if it is loaded due to sharing */
915 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
916 JUMP_COND_SHRD);
917 if (cdata->key_inline)
918 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
919 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
920 else
921 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
922 KEY_DEST_CLASS_REG);
923 set_jump_tgt_here(desc, key_jump_cmd);
924
925 /* Class 1 operation */
926 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
927 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
928
929 /* assoclen + cryptlen = seqoutlen */
930 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
931
932 /*
933 * MOVE_LEN opcode is not available in all SEC HW revisions,
934 * thus need to do some magic, i.e. self-patch the descriptor
935 * buffer.
936 */
937 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
938 (0x6 << MOVE_LEN_SHIFT));
939 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
940 (0x8 << MOVE_LEN_SHIFT));
941
942 /* Will read assoclen + cryptlen bytes */
943 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
944
945 /* Will write assoclen + cryptlen bytes */
946 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
947
948 /* Store payload data */
949 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
950
951 /* In-snoop assoclen + cryptlen data */
952 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
953 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
954
955 set_move_tgt_here(desc, read_move_cmd);
956 set_move_tgt_here(desc, write_move_cmd);
957 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
958 /* Move payload data to OFIFO */
959 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
960 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
961
962 /* Read ICV */
963 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
964 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
965
966 #ifdef DEBUG
967 print_hex_dump(KERN_ERR,
968 "rfc4543 dec shdesc@" __stringify(__LINE__)": ",
969 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
970 #endif
971 }
972 EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
973
974 /*
975 * For ablkcipher encrypt and decrypt, read from req->src and
976 * write to req->dst
977 */
978 static inline void ablkcipher_append_src_dst(u32 *desc)
979 {
980 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
981 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
982 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
983 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
984 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
985 }
986
987 /**
988 * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor
989 * @desc: pointer to buffer used for descriptor construction
990 * @cdata: pointer to block cipher transform definitions
991 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
992 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
993 * @ivsize: initialization vector size
994 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
995 * @ctx1_iv_off: IV offset in CONTEXT1 register
996 */
997 void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata,
998 unsigned int ivsize, const bool is_rfc3686,
999 const u32 ctx1_iv_off)
1000 {
1001 u32 *key_jump_cmd;
1002
1003 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1004 /* Skip if already shared */
1005 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1006 JUMP_COND_SHRD);
1007
1008 /* Load class1 key only */
1009 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1010 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1011
1012 /* Load nonce into CONTEXT1 reg */
1013 if (is_rfc3686) {
1014 u8 *nonce = cdata->key_virt + cdata->keylen;
1015
1016 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1017 LDST_CLASS_IND_CCB |
1018 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1019 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1020 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1021 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1022 }
1023
1024 set_jump_tgt_here(desc, key_jump_cmd);
1025
1026 /* Load iv */
1027 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1028 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1029
1030 /* Load counter into CONTEXT1 reg */
1031 if (is_rfc3686)
1032 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1033 LDST_SRCDST_BYTE_CONTEXT |
1034 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1035 LDST_OFFSET_SHIFT));
1036
1037 /* Load operation */
1038 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1039 OP_ALG_ENCRYPT);
1040
1041 /* Perform operation */
1042 ablkcipher_append_src_dst(desc);
1043
1044 #ifdef DEBUG
1045 print_hex_dump(KERN_ERR,
1046 "ablkcipher enc shdesc@" __stringify(__LINE__)": ",
1047 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1048 #endif
1049 }
1050 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap);
1051
1052 /**
1053 * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor
1054 * @desc: pointer to buffer used for descriptor construction
1055 * @cdata: pointer to block cipher transform definitions
1056 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1057 * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128.
1058 * @ivsize: initialization vector size
1059 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1060 * @ctx1_iv_off: IV offset in CONTEXT1 register
1061 */
1062 void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata,
1063 unsigned int ivsize, const bool is_rfc3686,
1064 const u32 ctx1_iv_off)
1065 {
1066 u32 *key_jump_cmd;
1067
1068 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1069 /* Skip if already shared */
1070 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1071 JUMP_COND_SHRD);
1072
1073 /* Load class1 key only */
1074 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1075 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1076
1077 /* Load nonce into CONTEXT1 reg */
1078 if (is_rfc3686) {
1079 u8 *nonce = cdata->key_virt + cdata->keylen;
1080
1081 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1082 LDST_CLASS_IND_CCB |
1083 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1084 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1085 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1086 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1087 }
1088
1089 set_jump_tgt_here(desc, key_jump_cmd);
1090
1091 /* load IV */
1092 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1093 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1094
1095 /* Load counter into CONTEXT1 reg */
1096 if (is_rfc3686)
1097 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1098 LDST_SRCDST_BYTE_CONTEXT |
1099 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1100 LDST_OFFSET_SHIFT));
1101
1102 /* Choose operation */
1103 if (ctx1_iv_off)
1104 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1105 OP_ALG_DECRYPT);
1106 else
1107 append_dec_op1(desc, cdata->algtype);
1108
1109 /* Perform operation */
1110 ablkcipher_append_src_dst(desc);
1111
1112 #ifdef DEBUG
1113 print_hex_dump(KERN_ERR,
1114 "ablkcipher dec shdesc@" __stringify(__LINE__)": ",
1115 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1116 #endif
1117 }
1118 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap);
1119
1120 /**
1121 * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor
1122 * with HW-generated initialization vector.
1123 * @desc: pointer to buffer used for descriptor construction
1124 * @cdata: pointer to block cipher transform definitions
1125 * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed
1126 * with OP_ALG_AAI_CBC.
1127 * @ivsize: initialization vector size
1128 * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template
1129 * @ctx1_iv_off: IV offset in CONTEXT1 register
1130 */
1131 void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata,
1132 unsigned int ivsize, const bool is_rfc3686,
1133 const u32 ctx1_iv_off)
1134 {
1135 u32 *key_jump_cmd, geniv;
1136
1137 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1138 /* Skip if already shared */
1139 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1140 JUMP_COND_SHRD);
1141
1142 /* Load class1 key only */
1143 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1144 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1145
1146 /* Load Nonce into CONTEXT1 reg */
1147 if (is_rfc3686) {
1148 u8 *nonce = cdata->key_virt + cdata->keylen;
1149
1150 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1151 LDST_CLASS_IND_CCB |
1152 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1153 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1154 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1155 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1156 }
1157 set_jump_tgt_here(desc, key_jump_cmd);
1158
1159 /* Generate IV */
1160 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
1161 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND |
1162 (ivsize << NFIFOENTRY_DLEN_SHIFT);
1163 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
1164 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
1165 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1166 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO |
1167 MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) |
1168 (ctx1_iv_off << MOVE_OFFSET_SHIFT));
1169 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1170
1171 /* Copy generated IV to memory */
1172 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1173 LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT));
1174
1175 /* Load Counter into CONTEXT1 reg */
1176 if (is_rfc3686)
1177 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1178 LDST_SRCDST_BYTE_CONTEXT |
1179 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1180 LDST_OFFSET_SHIFT));
1181
1182 if (ctx1_iv_off)
1183 append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP |
1184 (1 << JUMP_OFFSET_SHIFT));
1185
1186 /* Load operation */
1187 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1188 OP_ALG_ENCRYPT);
1189
1190 /* Perform operation */
1191 ablkcipher_append_src_dst(desc);
1192
1193 #ifdef DEBUG
1194 print_hex_dump(KERN_ERR,
1195 "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ",
1196 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1197 #endif
1198 }
1199 EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap);
1200
1201 /**
1202 * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared
1203 * descriptor
1204 * @desc: pointer to buffer used for descriptor construction
1205 * @cdata: pointer to block cipher transform definitions
1206 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1207 */
1208 void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata)
1209 {
1210 __be64 sector_size = cpu_to_be64(512);
1211 u32 *key_jump_cmd;
1212
1213 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1214 /* Skip if already shared */
1215 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1216 JUMP_COND_SHRD);
1217
1218 /* Load class1 keys only */
1219 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1220 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1221
1222 /* Load sector size with index 40 bytes (0x28) */
1223 append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1224 LDST_SRCDST_BYTE_CONTEXT |
1225 (0x28 << LDST_OFFSET_SHIFT));
1226
1227 set_jump_tgt_here(desc, key_jump_cmd);
1228
1229 /*
1230 * create sequence for loading the sector index
1231 * Upper 8B of IV - will be used as sector index
1232 * Lower 8B of IV - will be discarded
1233 */
1234 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1235 (0x20 << LDST_OFFSET_SHIFT));
1236 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1237
1238 /* Load operation */
1239 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1240 OP_ALG_ENCRYPT);
1241
1242 /* Perform operation */
1243 ablkcipher_append_src_dst(desc);
1244
1245 #ifdef DEBUG
1246 print_hex_dump(KERN_ERR,
1247 "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ",
1248 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1249 #endif
1250 }
1251 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap);
1252
1253 /**
1254 * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared
1255 * descriptor
1256 * @desc: pointer to buffer used for descriptor construction
1257 * @cdata: pointer to block cipher transform definitions
1258 * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS.
1259 */
1260 void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata)
1261 {
1262 __be64 sector_size = cpu_to_be64(512);
1263 u32 *key_jump_cmd;
1264
1265 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1266 /* Skip if already shared */
1267 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1268 JUMP_COND_SHRD);
1269
1270 /* Load class1 key only */
1271 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1272 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1273
1274 /* Load sector size with index 40 bytes (0x28) */
1275 append_load_as_imm(desc, (void *)&sector_size, 8, LDST_CLASS_1_CCB |
1276 LDST_SRCDST_BYTE_CONTEXT |
1277 (0x28 << LDST_OFFSET_SHIFT));
1278
1279 set_jump_tgt_here(desc, key_jump_cmd);
1280
1281 /*
1282 * create sequence for loading the sector index
1283 * Upper 8B of IV - will be used as sector index
1284 * Lower 8B of IV - will be discarded
1285 */
1286 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1287 (0x20 << LDST_OFFSET_SHIFT));
1288 append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP);
1289
1290 /* Load operation */
1291 append_dec_op1(desc, cdata->algtype);
1292
1293 /* Perform operation */
1294 ablkcipher_append_src_dst(desc);
1295
1296 #ifdef DEBUG
1297 print_hex_dump(KERN_ERR,
1298 "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ",
1299 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1);
1300 #endif
1301 }
1302 EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap);
1303
1304 MODULE_LICENSE("GPL");
1305 MODULE_DESCRIPTION("FSL CAAM descriptor support");
1306 MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
ubuntu-bionic-kernel mirror