]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - drivers/crypto/chelsio/chcr_algo.c
projects / mirror_ubuntu-hirsute-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge tag 'fsnotify_for_v5.7-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git...
[mirror_ubuntu-hirsute-kernel.git] / drivers / crypto / chelsio / chcr_algo.c
1 /*
2 * This file is part of the Chelsio T6 Crypto driver for Linux.
3 *
4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written and Maintained by:
35 * Manoj Malviya (manojmalviya@chelsio.com)
36 * Atul Gupta (atul.gupta@chelsio.com)
37 * Jitendra Lulla (jlulla@chelsio.com)
38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
39 * Harsh Jain (harsh@chelsio.com)
40 */
41
42 #define pr_fmt(fmt) "chcr:" fmt
43
44 #include <linux/kernel.h>
45 #include <linux/module.h>
46 #include <linux/crypto.h>
47 #include <linux/cryptohash.h>
48 #include <linux/skbuff.h>
49 #include <linux/rtnetlink.h>
50 #include <linux/highmem.h>
51 #include <linux/scatterlist.h>
52
53 #include <crypto/aes.h>
54 #include <crypto/algapi.h>
55 #include <crypto/hash.h>
56 #include <crypto/gcm.h>
57 #include <crypto/sha.h>
58 #include <crypto/authenc.h>
59 #include <crypto/ctr.h>
60 #include <crypto/gf128mul.h>
61 #include <crypto/internal/aead.h>
62 #include <crypto/null.h>
63 #include <crypto/internal/skcipher.h>
64 #include <crypto/aead.h>
65 #include <crypto/scatterwalk.h>
66 #include <crypto/internal/hash.h>
67
68 #include "t4fw_api.h"
69 #include "t4_msg.h"
70 #include "chcr_core.h"
71 #include "chcr_algo.h"
72 #include "chcr_crypto.h"
73
74 #define IV AES_BLOCK_SIZE
75
76 static unsigned int sgl_ent_len[] = {
77 0, 0, 16, 24, 40, 48, 64, 72, 88,
78 96, 112, 120, 136, 144, 160, 168, 184,
79 192, 208, 216, 232, 240, 256, 264, 280,
80 288, 304, 312, 328, 336, 352, 360, 376
81 };
82
83 static unsigned int dsgl_ent_len[] = {
84 0, 32, 32, 48, 48, 64, 64, 80, 80,
85 112, 112, 128, 128, 144, 144, 160, 160,
86 192, 192, 208, 208, 224, 224, 240, 240,
87 272, 272, 288, 288, 304, 304, 320, 320
88 };
89
90 static u32 round_constant[11] = {
91 0x01000000, 0x02000000, 0x04000000, 0x08000000,
92 0x10000000, 0x20000000, 0x40000000, 0x80000000,
93 0x1B000000, 0x36000000, 0x6C000000
94 };
95
96 static int chcr_handle_cipher_resp(struct skcipher_request *req,
97 unsigned char *input, int err);
98
99 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
100 {
101 return ctx->crypto_ctx->aeadctx;
102 }
103
104 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
105 {
106 return ctx->crypto_ctx->ablkctx;
107 }
108
109 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
110 {
111 return ctx->crypto_ctx->hmacctx;
112 }
113
114 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
115 {
116 return gctx->ctx->gcm;
117 }
118
119 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
120 {
121 return gctx->ctx->authenc;
122 }
123
124 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
125 {
126 return container_of(ctx->dev, struct uld_ctx, dev);
127 }
128
129 static inline int is_ofld_imm(const struct sk_buff *skb)
130 {
131 return (skb->len <= SGE_MAX_WR_LEN);
132 }
133
134 static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx)
135 {
136 memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr));
137 }
138
139 static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
140 unsigned int entlen,
141 unsigned int skip)
142 {
143 int nents = 0;
144 unsigned int less;
145 unsigned int skip_len = 0;
146
147 while (sg && skip) {
148 if (sg_dma_len(sg) <= skip) {
149 skip -= sg_dma_len(sg);
150 skip_len = 0;
151 sg = sg_next(sg);
152 } else {
153 skip_len = skip;
154 skip = 0;
155 }
156 }
157
158 while (sg && reqlen) {
159 less = min(reqlen, sg_dma_len(sg) - skip_len);
160 nents += DIV_ROUND_UP(less, entlen);
161 reqlen -= less;
162 skip_len = 0;
163 sg = sg_next(sg);
164 }
165 return nents;
166 }
167
168 static inline int get_aead_subtype(struct crypto_aead *aead)
169 {
170 struct aead_alg *alg = crypto_aead_alg(aead);
171 struct chcr_alg_template *chcr_crypto_alg =
172 container_of(alg, struct chcr_alg_template, alg.aead);
173 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
174 }
175
176 void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
177 {
178 u8 temp[SHA512_DIGEST_SIZE];
179 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
180 int authsize = crypto_aead_authsize(tfm);
181 struct cpl_fw6_pld *fw6_pld;
182 int cmp = 0;
183
184 fw6_pld = (struct cpl_fw6_pld *)input;
185 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
186 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
187 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
188 } else {
189
190 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
191 authsize, req->assoclen +
192 req->cryptlen - authsize);
193 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
194 }
195 if (cmp)
196 *err = -EBADMSG;
197 else
198 *err = 0;
199 }
200
201 static int chcr_inc_wrcount(struct chcr_dev *dev)
202 {
203 if (dev->state == CHCR_DETACH)
204 return 1;
205 atomic_inc(&dev->inflight);
206 return 0;
207 }
208
209 static inline void chcr_dec_wrcount(struct chcr_dev *dev)
210 {
211 atomic_dec(&dev->inflight);
212 }
213
214 static inline int chcr_handle_aead_resp(struct aead_request *req,
215 unsigned char *input,
216 int err)
217 {
218 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
219 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
220 struct chcr_dev *dev = a_ctx(tfm)->dev;
221
222 chcr_aead_common_exit(req);
223 if (reqctx->verify == VERIFY_SW) {
224 chcr_verify_tag(req, input, &err);
225 reqctx->verify = VERIFY_HW;
226 }
227 chcr_dec_wrcount(dev);
228 req->base.complete(&req->base, err);
229
230 return err;
231 }
232
233 static void get_aes_decrypt_key(unsigned char *dec_key,
234 const unsigned char *key,
235 unsigned int keylength)
236 {
237 u32 temp;
238 u32 w_ring[MAX_NK];
239 int i, j, k;
240 u8 nr, nk;
241
242 switch (keylength) {
243 case AES_KEYLENGTH_128BIT:
244 nk = KEYLENGTH_4BYTES;
245 nr = NUMBER_OF_ROUNDS_10;
246 break;
247 case AES_KEYLENGTH_192BIT:
248 nk = KEYLENGTH_6BYTES;
249 nr = NUMBER_OF_ROUNDS_12;
250 break;
251 case AES_KEYLENGTH_256BIT:
252 nk = KEYLENGTH_8BYTES;
253 nr = NUMBER_OF_ROUNDS_14;
254 break;
255 default:
256 return;
257 }
258 for (i = 0; i < nk; i++)
259 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
260
261 i = 0;
262 temp = w_ring[nk - 1];
263 while (i + nk < (nr + 1) * 4) {
264 if (!(i % nk)) {
265 /* RotWord(temp) */
266 temp = (temp << 8) | (temp >> 24);
267 temp = aes_ks_subword(temp);
268 temp ^= round_constant[i / nk];
269 } else if (nk == 8 && (i % 4 == 0)) {
270 temp = aes_ks_subword(temp);
271 }
272 w_ring[i % nk] ^= temp;
273 temp = w_ring[i % nk];
274 i++;
275 }
276 i--;
277 for (k = 0, j = i % nk; k < nk; k++) {
278 *((u32 *)dec_key + k) = htonl(w_ring[j]);
279 j--;
280 if (j < 0)
281 j += nk;
282 }
283 }
284
285 static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
286 {
287 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
288
289 switch (ds) {
290 case SHA1_DIGEST_SIZE:
291 base_hash = crypto_alloc_shash("sha1", 0, 0);
292 break;
293 case SHA224_DIGEST_SIZE:
294 base_hash = crypto_alloc_shash("sha224", 0, 0);
295 break;
296 case SHA256_DIGEST_SIZE:
297 base_hash = crypto_alloc_shash("sha256", 0, 0);
298 break;
299 case SHA384_DIGEST_SIZE:
300 base_hash = crypto_alloc_shash("sha384", 0, 0);
301 break;
302 case SHA512_DIGEST_SIZE:
303 base_hash = crypto_alloc_shash("sha512", 0, 0);
304 break;
305 }
306
307 return base_hash;
308 }
309
310 static int chcr_compute_partial_hash(struct shash_desc *desc,
311 char *iopad, char *result_hash,
312 int digest_size)
313 {
314 struct sha1_state sha1_st;
315 struct sha256_state sha256_st;
316 struct sha512_state sha512_st;
317 int error;
318
319 if (digest_size == SHA1_DIGEST_SIZE) {
320 error = crypto_shash_init(desc) ?:
321 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
322 crypto_shash_export(desc, (void *)&sha1_st);
323 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
324 } else if (digest_size == SHA224_DIGEST_SIZE) {
325 error = crypto_shash_init(desc) ?:
326 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
327 crypto_shash_export(desc, (void *)&sha256_st);
328 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
329
330 } else if (digest_size == SHA256_DIGEST_SIZE) {
331 error = crypto_shash_init(desc) ?:
332 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
333 crypto_shash_export(desc, (void *)&sha256_st);
334 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
335
336 } else if (digest_size == SHA384_DIGEST_SIZE) {
337 error = crypto_shash_init(desc) ?:
338 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
339 crypto_shash_export(desc, (void *)&sha512_st);
340 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
341
342 } else if (digest_size == SHA512_DIGEST_SIZE) {
343 error = crypto_shash_init(desc) ?:
344 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
345 crypto_shash_export(desc, (void *)&sha512_st);
346 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
347 } else {
348 error = -EINVAL;
349 pr_err("Unknown digest size %d\n", digest_size);
350 }
351 return error;
352 }
353
354 static void chcr_change_order(char *buf, int ds)
355 {
356 int i;
357
358 if (ds == SHA512_DIGEST_SIZE) {
359 for (i = 0; i < (ds / sizeof(u64)); i++)
360 *((__be64 *)buf + i) =
361 cpu_to_be64(*((u64 *)buf + i));
362 } else {
363 for (i = 0; i < (ds / sizeof(u32)); i++)
364 *((__be32 *)buf + i) =
365 cpu_to_be32(*((u32 *)buf + i));
366 }
367 }
368
369 static inline int is_hmac(struct crypto_tfm *tfm)
370 {
371 struct crypto_alg *alg = tfm->__crt_alg;
372 struct chcr_alg_template *chcr_crypto_alg =
373 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
374 alg.hash);
375 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
376 return 1;
377 return 0;
378 }
379
380 static inline void dsgl_walk_init(struct dsgl_walk *walk,
381 struct cpl_rx_phys_dsgl *dsgl)
382 {
383 walk->dsgl = dsgl;
384 walk->nents = 0;
385 walk->to = (struct phys_sge_pairs *)(dsgl + 1);
386 }
387
388 static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid,
389 int pci_chan_id)
390 {
391 struct cpl_rx_phys_dsgl *phys_cpl;
392
393 phys_cpl = walk->dsgl;
394
395 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
396 | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
397 phys_cpl->pcirlxorder_to_noofsgentr =
398 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
399 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
400 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
401 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
402 CPL_RX_PHYS_DSGL_DCAID_V(0) |
403 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents));
404 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
405 phys_cpl->rss_hdr_int.qid = htons(qid);
406 phys_cpl->rss_hdr_int.hash_val = 0;
407 phys_cpl->rss_hdr_int.channel = pci_chan_id;
408 }
409
410 static inline void dsgl_walk_add_page(struct dsgl_walk *walk,
411 size_t size,
412 dma_addr_t addr)
413 {
414 int j;
415
416 if (!size)
417 return;
418 j = walk->nents;
419 walk->to->len[j % 8] = htons(size);
420 walk->to->addr[j % 8] = cpu_to_be64(addr);
421 j++;
422 if ((j % 8) == 0)
423 walk->to++;
424 walk->nents = j;
425 }
426
427 static void dsgl_walk_add_sg(struct dsgl_walk *walk,
428 struct scatterlist *sg,
429 unsigned int slen,
430 unsigned int skip)
431 {
432 int skip_len = 0;
433 unsigned int left_size = slen, len = 0;
434 unsigned int j = walk->nents;
435 int offset, ent_len;
436
437 if (!slen)
438 return;
439 while (sg && skip) {
440 if (sg_dma_len(sg) <= skip) {
441 skip -= sg_dma_len(sg);
442 skip_len = 0;
443 sg = sg_next(sg);
444 } else {
445 skip_len = skip;
446 skip = 0;
447 }
448 }
449
450 while (left_size && sg) {
451 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len);
452 offset = 0;
453 while (len) {
454 ent_len = min_t(u32, len, CHCR_DST_SG_SIZE);
455 walk->to->len[j % 8] = htons(ent_len);
456 walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
457 offset + skip_len);
458 offset += ent_len;
459 len -= ent_len;
460 j++;
461 if ((j % 8) == 0)
462 walk->to++;
463 }
464 walk->last_sg = sg;
465 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) -
466 skip_len) + skip_len;
467 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len);
468 skip_len = 0;
469 sg = sg_next(sg);
470 }
471 walk->nents = j;
472 }
473
474 static inline void ulptx_walk_init(struct ulptx_walk *walk,
475 struct ulptx_sgl *ulp)
476 {
477 walk->sgl = ulp;
478 walk->nents = 0;
479 walk->pair_idx = 0;
480 walk->pair = ulp->sge;
481 walk->last_sg = NULL;
482 walk->last_sg_len = 0;
483 }
484
485 static inline void ulptx_walk_end(struct ulptx_walk *walk)
486 {
487 walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
488 ULPTX_NSGE_V(walk->nents));
489 }
490
491
492 static inline void ulptx_walk_add_page(struct ulptx_walk *walk,
493 size_t size,
494 dma_addr_t addr)
495 {
496 if (!size)
497 return;
498
499 if (walk->nents == 0) {
500 walk->sgl->len0 = cpu_to_be32(size);
501 walk->sgl->addr0 = cpu_to_be64(addr);
502 } else {
503 walk->pair->addr[walk->pair_idx] = cpu_to_be64(addr);
504 walk->pair->len[walk->pair_idx] = cpu_to_be32(size);
505 walk->pair_idx = !walk->pair_idx;
506 if (!walk->pair_idx)
507 walk->pair++;
508 }
509 walk->nents++;
510 }
511
512 static void ulptx_walk_add_sg(struct ulptx_walk *walk,
513 struct scatterlist *sg,
514 unsigned int len,
515 unsigned int skip)
516 {
517 int small;
518 int skip_len = 0;
519 unsigned int sgmin;
520
521 if (!len)
522 return;
523 while (sg && skip) {
524 if (sg_dma_len(sg) <= skip) {
525 skip -= sg_dma_len(sg);
526 skip_len = 0;
527 sg = sg_next(sg);
528 } else {
529 skip_len = skip;
530 skip = 0;
531 }
532 }
533 WARN(!sg, "SG should not be null here\n");
534 if (sg && (walk->nents == 0)) {
535 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len);
536 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
537 walk->sgl->len0 = cpu_to_be32(sgmin);
538 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len);
539 walk->nents++;
540 len -= sgmin;
541 walk->last_sg = sg;
542 walk->last_sg_len = sgmin + skip_len;
543 skip_len += sgmin;
544 if (sg_dma_len(sg) == skip_len) {
545 sg = sg_next(sg);
546 skip_len = 0;
547 }
548 }
549
550 while (sg && len) {
551 small = min(sg_dma_len(sg) - skip_len, len);
552 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
553 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin);
554 walk->pair->addr[walk->pair_idx] =
555 cpu_to_be64(sg_dma_address(sg) + skip_len);
556 walk->pair_idx = !walk->pair_idx;
557 walk->nents++;
558 if (!walk->pair_idx)
559 walk->pair++;
560 len -= sgmin;
561 skip_len += sgmin;
562 walk->last_sg = sg;
563 walk->last_sg_len = skip_len;
564 if (sg_dma_len(sg) == skip_len) {
565 sg = sg_next(sg);
566 skip_len = 0;
567 }
568 }
569 }
570
571 static inline int get_cryptoalg_subtype(struct crypto_skcipher *tfm)
572 {
573 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
574 struct chcr_alg_template *chcr_crypto_alg =
575 container_of(alg, struct chcr_alg_template, alg.skcipher);
576
577 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
578 }
579
580 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
581 {
582 struct adapter *adap = netdev2adap(dev);
583 struct sge_uld_txq_info *txq_info =
584 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
585 struct sge_uld_txq *txq;
586 int ret = 0;
587
588 local_bh_disable();
589 txq = &txq_info->uldtxq[idx];
590 spin_lock(&txq->sendq.lock);
591 if (txq->full)
592 ret = -1;
593 spin_unlock(&txq->sendq.lock);
594 local_bh_enable();
595 return ret;
596 }
597
598 static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
599 struct _key_ctx *key_ctx)
600 {
601 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
602 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
603 } else {
604 memcpy(key_ctx->key,
605 ablkctx->key + (ablkctx->enckey_len >> 1),
606 ablkctx->enckey_len >> 1);
607 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
608 ablkctx->rrkey, ablkctx->enckey_len >> 1);
609 }
610 return 0;
611 }
612
613 static int chcr_hash_ent_in_wr(struct scatterlist *src,
614 unsigned int minsg,
615 unsigned int space,
616 unsigned int srcskip)
617 {
618 int srclen = 0;
619 int srcsg = minsg;
620 int soffset = 0, sless;
621
622 if (sg_dma_len(src) == srcskip) {
623 src = sg_next(src);
624 srcskip = 0;
625 }
626 while (src && space > (sgl_ent_len[srcsg + 1])) {
627 sless = min_t(unsigned int, sg_dma_len(src) - soffset - srcskip,
628 CHCR_SRC_SG_SIZE);
629 srclen += sless;
630 soffset += sless;
631 srcsg++;
632 if (sg_dma_len(src) == (soffset + srcskip)) {
633 src = sg_next(src);
634 soffset = 0;
635 srcskip = 0;
636 }
637 }
638 return srclen;
639 }
640
641 static int chcr_sg_ent_in_wr(struct scatterlist *src,
642 struct scatterlist *dst,
643 unsigned int minsg,
644 unsigned int space,
645 unsigned int srcskip,
646 unsigned int dstskip)
647 {
648 int srclen = 0, dstlen = 0;
649 int srcsg = minsg, dstsg = minsg;
650 int offset = 0, soffset = 0, less, sless = 0;
651
652 if (sg_dma_len(src) == srcskip) {
653 src = sg_next(src);
654 srcskip = 0;
655 }
656 if (sg_dma_len(dst) == dstskip) {
657 dst = sg_next(dst);
658 dstskip = 0;
659 }
660
661 while (src && dst &&
662 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
663 sless = min_t(unsigned int, sg_dma_len(src) - srcskip - soffset,
664 CHCR_SRC_SG_SIZE);
665 srclen += sless;
666 srcsg++;
667 offset = 0;
668 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
669 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
670 if (srclen <= dstlen)
671 break;
672 less = min_t(unsigned int, sg_dma_len(dst) - offset -
673 dstskip, CHCR_DST_SG_SIZE);
674 dstlen += less;
675 offset += less;
676 if ((offset + dstskip) == sg_dma_len(dst)) {
677 dst = sg_next(dst);
678 offset = 0;
679 }
680 dstsg++;
681 dstskip = 0;
682 }
683 soffset += sless;
684 if ((soffset + srcskip) == sg_dma_len(src)) {
685 src = sg_next(src);
686 srcskip = 0;
687 soffset = 0;
688 }
689
690 }
691 return min(srclen, dstlen);
692 }
693
694 static int chcr_cipher_fallback(struct crypto_sync_skcipher *cipher,
695 u32 flags,
696 struct scatterlist *src,
697 struct scatterlist *dst,
698 unsigned int nbytes,
699 u8 *iv,
700 unsigned short op_type)
701 {
702 int err;
703
704 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
705
706 skcipher_request_set_sync_tfm(subreq, cipher);
707 skcipher_request_set_callback(subreq, flags, NULL, NULL);
708 skcipher_request_set_crypt(subreq, src, dst,
709 nbytes, iv);
710
711 err = op_type ? crypto_skcipher_decrypt(subreq) :
712 crypto_skcipher_encrypt(subreq);
713 skcipher_request_zero(subreq);
714
715 return err;
716
717 }
718
719 static inline int get_qidxs(struct crypto_async_request *req,
720 unsigned int *txqidx, unsigned int *rxqidx)
721 {
722 struct crypto_tfm *tfm = req->tfm;
723 int ret = 0;
724
725 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
726 case CRYPTO_ALG_TYPE_AEAD:
727 {
728 struct aead_request *aead_req =
729 container_of(req, struct aead_request, base);
730 struct chcr_aead_reqctx *reqctx = aead_request_ctx(aead_req);
731 *txqidx = reqctx->txqidx;
732 *rxqidx = reqctx->rxqidx;
733 break;
734 }
735 case CRYPTO_ALG_TYPE_SKCIPHER:
736 {
737 struct skcipher_request *sk_req =
738 container_of(req, struct skcipher_request, base);
739 struct chcr_skcipher_req_ctx *reqctx =
740 skcipher_request_ctx(sk_req);
741 *txqidx = reqctx->txqidx;
742 *rxqidx = reqctx->rxqidx;
743 break;
744 }
745 case CRYPTO_ALG_TYPE_AHASH:
746 {
747 struct ahash_request *ahash_req =
748 container_of(req, struct ahash_request, base);
749 struct chcr_ahash_req_ctx *reqctx =
750 ahash_request_ctx(ahash_req);
751 *txqidx = reqctx->txqidx;
752 *rxqidx = reqctx->rxqidx;
753 break;
754 }
755 default:
756 ret = -EINVAL;
757 /* should never get here */
758 BUG();
759 break;
760 }
761 return ret;
762 }
763
764 static inline void create_wreq(struct chcr_context *ctx,
765 struct chcr_wr *chcr_req,
766 struct crypto_async_request *req,
767 unsigned int imm,
768 int hash_sz,
769 unsigned int len16,
770 unsigned int sc_len,
771 unsigned int lcb)
772 {
773 struct uld_ctx *u_ctx = ULD_CTX(ctx);
774 unsigned int tx_channel_id, rx_channel_id;
775 unsigned int txqidx = 0, rxqidx = 0;
776 unsigned int qid, fid;
777
778 get_qidxs(req, &txqidx, &rxqidx);
779 qid = u_ctx->lldi.rxq_ids[rxqidx];
780 fid = u_ctx->lldi.rxq_ids[0];
781 tx_channel_id = txqidx / ctx->txq_perchan;
782 rx_channel_id = rxqidx / ctx->rxq_perchan;
783
784
785 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE;
786 chcr_req->wreq.pld_size_hash_size =
787 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
788 chcr_req->wreq.len16_pkd =
789 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16)));
790 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
791 chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(rx_channel_id, qid,
792 !!lcb, txqidx);
793
794 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(tx_channel_id, fid);
795 chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) -
796 ((sizeof(chcr_req->wreq)) >> 4)));
797 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm);
798 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
799 sizeof(chcr_req->key_ctx) + sc_len);
800 }
801
802 /**
803 * create_cipher_wr - form the WR for cipher operations
804 * @req: cipher req.
805 * @ctx: crypto driver context of the request.
806 * @qid: ingress qid where response of this WR should be received.
807 * @op_type: encryption or decryption
808 */
809 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
810 {
811 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req);
812 struct chcr_context *ctx = c_ctx(tfm);
813 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
814 struct sk_buff *skb = NULL;
815 struct chcr_wr *chcr_req;
816 struct cpl_rx_phys_dsgl *phys_cpl;
817 struct ulptx_sgl *ulptx;
818 struct chcr_skcipher_req_ctx *reqctx =
819 skcipher_request_ctx(wrparam->req);
820 unsigned int temp = 0, transhdr_len, dst_size;
821 int error;
822 int nents;
823 unsigned int kctx_len;
824 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
825 GFP_KERNEL : GFP_ATOMIC;
826 struct adapter *adap = padap(ctx->dev);
827 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
828
829 nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE,
830 reqctx->dst_ofst);
831 dst_size = get_space_for_phys_dsgl(nents);
832 kctx_len = roundup(ablkctx->enckey_len, 16);
833 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
834 nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes,
835 CHCR_SRC_SG_SIZE, reqctx->src_ofst);
836 temp = reqctx->imm ? roundup(wrparam->bytes, 16) :
837 (sgl_len(nents) * 8);
838 transhdr_len += temp;
839 transhdr_len = roundup(transhdr_len, 16);
840 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
841 if (!skb) {
842 error = -ENOMEM;
843 goto err;
844 }
845 chcr_req = __skb_put_zero(skb, transhdr_len);
846 chcr_req->sec_cpl.op_ivinsrtofst =
847 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
848
849 chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes);
850 chcr_req->sec_cpl.aadstart_cipherstop_hi =
851 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0);
852
853 chcr_req->sec_cpl.cipherstop_lo_authinsert =
854 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
855 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
856 ablkctx->ciph_mode,
857 0, 0, IV >> 1);
858 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
859 0, 1, dst_size);
860
861 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
862 if ((reqctx->op == CHCR_DECRYPT_OP) &&
863 (!(get_cryptoalg_subtype(tfm) ==
864 CRYPTO_ALG_SUB_TYPE_CTR)) &&
865 (!(get_cryptoalg_subtype(tfm) ==
866 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
867 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
868 } else {
869 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
870 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
871 memcpy(chcr_req->key_ctx.key, ablkctx->key,
872 ablkctx->enckey_len);
873 } else {
874 memcpy(chcr_req->key_ctx.key, ablkctx->key +
875 (ablkctx->enckey_len >> 1),
876 ablkctx->enckey_len >> 1);
877 memcpy(chcr_req->key_ctx.key +
878 (ablkctx->enckey_len >> 1),
879 ablkctx->key,
880 ablkctx->enckey_len >> 1);
881 }
882 }
883 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
884 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
885 chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam);
886 chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid);
887
888 atomic_inc(&adap->chcr_stats.cipher_rqst);
889 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV
890 + (reqctx->imm ? (wrparam->bytes) : 0);
891 create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0,
892 transhdr_len, temp,
893 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
894 reqctx->skb = skb;
895
896 if (reqctx->op && (ablkctx->ciph_mode ==
897 CHCR_SCMD_CIPHER_MODE_AES_CBC))
898 sg_pcopy_to_buffer(wrparam->req->src,
899 sg_nents(wrparam->req->src), wrparam->req->iv, 16,
900 reqctx->processed + wrparam->bytes - AES_BLOCK_SIZE);
901
902 return skb;
903 err:
904 return ERR_PTR(error);
905 }
906
907 static inline int chcr_keyctx_ck_size(unsigned int keylen)
908 {
909 int ck_size = 0;
910
911 if (keylen == AES_KEYSIZE_128)
912 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
913 else if (keylen == AES_KEYSIZE_192)
914 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
915 else if (keylen == AES_KEYSIZE_256)
916 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
917 else
918 ck_size = 0;
919
920 return ck_size;
921 }
922 static int chcr_cipher_fallback_setkey(struct crypto_skcipher *cipher,
923 const u8 *key,
924 unsigned int keylen)
925 {
926 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
927
928 crypto_sync_skcipher_clear_flags(ablkctx->sw_cipher,
929 CRYPTO_TFM_REQ_MASK);
930 crypto_sync_skcipher_set_flags(ablkctx->sw_cipher,
931 cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK);
932 return crypto_sync_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
933 }
934
935 static int chcr_aes_cbc_setkey(struct crypto_skcipher *cipher,
936 const u8 *key,
937 unsigned int keylen)
938 {
939 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
940 unsigned int ck_size, context_size;
941 u16 alignment = 0;
942 int err;
943
944 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
945 if (err)
946 goto badkey_err;
947
948 ck_size = chcr_keyctx_ck_size(keylen);
949 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
950 memcpy(ablkctx->key, key, keylen);
951 ablkctx->enckey_len = keylen;
952 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
953 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
954 keylen + alignment) >> 4;
955
956 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
957 0, 0, context_size);
958 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
959 return 0;
960 badkey_err:
961 ablkctx->enckey_len = 0;
962
963 return err;
964 }
965
966 static int chcr_aes_ctr_setkey(struct crypto_skcipher *cipher,
967 const u8 *key,
968 unsigned int keylen)
969 {
970 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
971 unsigned int ck_size, context_size;
972 u16 alignment = 0;
973 int err;
974
975 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
976 if (err)
977 goto badkey_err;
978 ck_size = chcr_keyctx_ck_size(keylen);
979 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
980 memcpy(ablkctx->key, key, keylen);
981 ablkctx->enckey_len = keylen;
982 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
983 keylen + alignment) >> 4;
984
985 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
986 0, 0, context_size);
987 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
988
989 return 0;
990 badkey_err:
991 ablkctx->enckey_len = 0;
992
993 return err;
994 }
995
996 static int chcr_aes_rfc3686_setkey(struct crypto_skcipher *cipher,
997 const u8 *key,
998 unsigned int keylen)
999 {
1000 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
1001 unsigned int ck_size, context_size;
1002 u16 alignment = 0;
1003 int err;
1004
1005 if (keylen < CTR_RFC3686_NONCE_SIZE)
1006 return -EINVAL;
1007 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
1008 CTR_RFC3686_NONCE_SIZE);
1009
1010 keylen -= CTR_RFC3686_NONCE_SIZE;
1011 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
1012 if (err)
1013 goto badkey_err;
1014
1015 ck_size = chcr_keyctx_ck_size(keylen);
1016 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
1017 memcpy(ablkctx->key, key, keylen);
1018 ablkctx->enckey_len = keylen;
1019 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
1020 keylen + alignment) >> 4;
1021
1022 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
1023 0, 0, context_size);
1024 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
1025
1026 return 0;
1027 badkey_err:
1028 ablkctx->enckey_len = 0;
1029
1030 return err;
1031 }
1032 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
1033 {
1034 unsigned int size = AES_BLOCK_SIZE;
1035 __be32 *b = (__be32 *)(dstiv + size);
1036 u32 c, prev;
1037
1038 memcpy(dstiv, srciv, AES_BLOCK_SIZE);
1039 for (; size >= 4; size -= 4) {
1040 prev = be32_to_cpu(*--b);
1041 c = prev + add;
1042 *b = cpu_to_be32(c);
1043 if (prev < c)
1044 break;
1045 add = 1;
1046 }
1047
1048 }
1049
1050 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
1051 {
1052 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
1053 u64 c;
1054 u32 temp = be32_to_cpu(*--b);
1055
1056 temp = ~temp;
1057 c = (u64)temp + 1; // No of block can processed withou overflow
1058 if ((bytes / AES_BLOCK_SIZE) > c)
1059 bytes = c * AES_BLOCK_SIZE;
1060 return bytes;
1061 }
1062
1063 static int chcr_update_tweak(struct skcipher_request *req, u8 *iv,
1064 u32 isfinal)
1065 {
1066 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1067 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1068 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1069 struct crypto_aes_ctx aes;
1070 int ret, i;
1071 u8 *key;
1072 unsigned int keylen;
1073 int round = reqctx->last_req_len / AES_BLOCK_SIZE;
1074 int round8 = round / 8;
1075
1076 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1077
1078 keylen = ablkctx->enckey_len / 2;
1079 key = ablkctx->key + keylen;
1080 ret = aes_expandkey(&aes, key, keylen);
1081 if (ret)
1082 return ret;
1083 aes_encrypt(&aes, iv, iv);
1084 for (i = 0; i < round8; i++)
1085 gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
1086
1087 for (i = 0; i < (round % 8); i++)
1088 gf128mul_x_ble((le128 *)iv, (le128 *)iv);
1089
1090 if (!isfinal)
1091 aes_decrypt(&aes, iv, iv);
1092
1093 memzero_explicit(&aes, sizeof(aes));
1094 return 0;
1095 }
1096
1097 static int chcr_update_cipher_iv(struct skcipher_request *req,
1098 struct cpl_fw6_pld *fw6_pld, u8 *iv)
1099 {
1100 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1101 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1102 int subtype = get_cryptoalg_subtype(tfm);
1103 int ret = 0;
1104
1105 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1106 ctr_add_iv(iv, req->iv, (reqctx->processed /
1107 AES_BLOCK_SIZE));
1108 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
1109 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1110 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
1111 AES_BLOCK_SIZE) + 1);
1112 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
1113 ret = chcr_update_tweak(req, iv, 0);
1114 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1115 if (reqctx->op)
1116 /*Updated before sending last WR*/
1117 memcpy(iv, req->iv, AES_BLOCK_SIZE);
1118 else
1119 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1120 }
1121
1122 return ret;
1123
1124 }
1125
1126 /* We need separate function for final iv because in rfc3686 Initial counter
1127 * starts from 1 and buffer size of iv is 8 byte only which remains constant
1128 * for subsequent update requests
1129 */
1130
1131 static int chcr_final_cipher_iv(struct skcipher_request *req,
1132 struct cpl_fw6_pld *fw6_pld, u8 *iv)
1133 {
1134 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1135 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1136 int subtype = get_cryptoalg_subtype(tfm);
1137 int ret = 0;
1138
1139 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
1140 ctr_add_iv(iv, req->iv, DIV_ROUND_UP(reqctx->processed,
1141 AES_BLOCK_SIZE));
1142 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) {
1143 if (!reqctx->partial_req)
1144 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1145 else
1146 ret = chcr_update_tweak(req, iv, 1);
1147 }
1148 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1149 /*Already updated for Decrypt*/
1150 if (!reqctx->op)
1151 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1152
1153 }
1154 return ret;
1155
1156 }
1157
1158 static int chcr_handle_cipher_resp(struct skcipher_request *req,
1159 unsigned char *input, int err)
1160 {
1161 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1162 struct chcr_context *ctx = c_ctx(tfm);
1163 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1164 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1165 struct sk_buff *skb;
1166 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
1167 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1168 struct cipher_wr_param wrparam;
1169 struct chcr_dev *dev = c_ctx(tfm)->dev;
1170 int bytes;
1171
1172 if (err)
1173 goto unmap;
1174 if (req->cryptlen == reqctx->processed) {
1175 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1176 req);
1177 err = chcr_final_cipher_iv(req, fw6_pld, req->iv);
1178 goto complete;
1179 }
1180
1181 if (!reqctx->imm) {
1182 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0,
1183 CIP_SPACE_LEFT(ablkctx->enckey_len),
1184 reqctx->src_ofst, reqctx->dst_ofst);
1185 if ((bytes + reqctx->processed) >= req->cryptlen)
1186 bytes = req->cryptlen - reqctx->processed;
1187 else
1188 bytes = rounddown(bytes, 16);
1189 } else {
1190 /*CTR mode counter overfloa*/
1191 bytes = req->cryptlen - reqctx->processed;
1192 }
1193 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
1194 if (err)
1195 goto unmap;
1196
1197 if (unlikely(bytes == 0)) {
1198 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1199 req);
1200 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1201 req->base.flags,
1202 req->src,
1203 req->dst,
1204 req->cryptlen,
1205 req->iv,
1206 reqctx->op);
1207 goto complete;
1208 }
1209
1210 if (get_cryptoalg_subtype(tfm) ==
1211 CRYPTO_ALG_SUB_TYPE_CTR)
1212 bytes = adjust_ctr_overflow(reqctx->iv, bytes);
1213 wrparam.qid = u_ctx->lldi.rxq_ids[reqctx->rxqidx];
1214 wrparam.req = req;
1215 wrparam.bytes = bytes;
1216 skb = create_cipher_wr(&wrparam);
1217 if (IS_ERR(skb)) {
1218 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
1219 err = PTR_ERR(skb);
1220 goto unmap;
1221 }
1222 skb->dev = u_ctx->lldi.ports[0];
1223 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1224 chcr_send_wr(skb);
1225 reqctx->last_req_len = bytes;
1226 reqctx->processed += bytes;
1227 if (get_cryptoalg_subtype(tfm) ==
1228 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1229 CRYPTO_TFM_REQ_MAY_SLEEP ) {
1230 complete(&ctx->cbc_aes_aio_done);
1231 }
1232 return 0;
1233 unmap:
1234 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1235 complete:
1236 if (get_cryptoalg_subtype(tfm) ==
1237 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1238 CRYPTO_TFM_REQ_MAY_SLEEP ) {
1239 complete(&ctx->cbc_aes_aio_done);
1240 }
1241 chcr_dec_wrcount(dev);
1242 req->base.complete(&req->base, err);
1243 return err;
1244 }
1245
1246 static int process_cipher(struct skcipher_request *req,
1247 unsigned short qid,
1248 struct sk_buff **skb,
1249 unsigned short op_type)
1250 {
1251 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1252 unsigned int ivsize = crypto_skcipher_ivsize(tfm);
1253 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1254 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
1255 struct cipher_wr_param wrparam;
1256 int bytes, err = -EINVAL;
1257
1258 reqctx->processed = 0;
1259 reqctx->partial_req = 0;
1260 if (!req->iv)
1261 goto error;
1262 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
1263 (req->cryptlen == 0) ||
1264 (req->cryptlen % crypto_skcipher_blocksize(tfm))) {
1265 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1266 ablkctx->enckey_len, req->cryptlen, ivsize);
1267 goto error;
1268 }
1269
1270 err = chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1271 if (err)
1272 goto error;
1273 if (req->cryptlen < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) +
1274 AES_MIN_KEY_SIZE +
1275 sizeof(struct cpl_rx_phys_dsgl) +
1276 /*Min dsgl size*/
1277 32))) {
1278 /* Can be sent as Imm*/
1279 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len;
1280
1281 dnents = sg_nents_xlen(req->dst, req->cryptlen,
1282 CHCR_DST_SG_SIZE, 0);
1283 phys_dsgl = get_space_for_phys_dsgl(dnents);
1284 kctx_len = roundup(ablkctx->enckey_len, 16);
1285 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
1286 reqctx->imm = (transhdr_len + IV + req->cryptlen) <=
1287 SGE_MAX_WR_LEN;
1288 bytes = IV + req->cryptlen;
1289
1290 } else {
1291 reqctx->imm = 0;
1292 }
1293
1294 if (!reqctx->imm) {
1295 bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0,
1296 CIP_SPACE_LEFT(ablkctx->enckey_len),
1297 0, 0);
1298 if ((bytes + reqctx->processed) >= req->cryptlen)
1299 bytes = req->cryptlen - reqctx->processed;
1300 else
1301 bytes = rounddown(bytes, 16);
1302 } else {
1303 bytes = req->cryptlen;
1304 }
1305 if (get_cryptoalg_subtype(tfm) ==
1306 CRYPTO_ALG_SUB_TYPE_CTR) {
1307 bytes = adjust_ctr_overflow(req->iv, bytes);
1308 }
1309 if (get_cryptoalg_subtype(tfm) ==
1310 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1311 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
1312 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv,
1313 CTR_RFC3686_IV_SIZE);
1314
1315 /* initialize counter portion of counter block */
1316 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1317 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1318
1319 } else {
1320
1321 memcpy(reqctx->iv, req->iv, IV);
1322 }
1323 if (unlikely(bytes == 0)) {
1324 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1325 req);
1326 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1327 req->base.flags,
1328 req->src,
1329 req->dst,
1330 req->cryptlen,
1331 reqctx->iv,
1332 op_type);
1333 goto error;
1334 }
1335 reqctx->op = op_type;
1336 reqctx->srcsg = req->src;
1337 reqctx->dstsg = req->dst;
1338 reqctx->src_ofst = 0;
1339 reqctx->dst_ofst = 0;
1340 wrparam.qid = qid;
1341 wrparam.req = req;
1342 wrparam.bytes = bytes;
1343 *skb = create_cipher_wr(&wrparam);
1344 if (IS_ERR(*skb)) {
1345 err = PTR_ERR(*skb);
1346 goto unmap;
1347 }
1348 reqctx->processed = bytes;
1349 reqctx->last_req_len = bytes;
1350 reqctx->partial_req = !!(req->cryptlen - reqctx->processed);
1351
1352 return 0;
1353 unmap:
1354 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1355 error:
1356 return err;
1357 }
1358
1359 static int chcr_aes_encrypt(struct skcipher_request *req)
1360 {
1361 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1362 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1363 struct chcr_dev *dev = c_ctx(tfm)->dev;
1364 struct sk_buff *skb = NULL;
1365 int err;
1366 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1367 struct chcr_context *ctx = c_ctx(tfm);
1368 unsigned int cpu;
1369
1370 cpu = get_cpu();
1371 reqctx->txqidx = cpu % ctx->ntxq;
1372 reqctx->rxqidx = cpu % ctx->nrxq;
1373 put_cpu();
1374
1375 err = chcr_inc_wrcount(dev);
1376 if (err)
1377 return -ENXIO;
1378 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1379 reqctx->txqidx) &&
1380 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1381 err = -ENOSPC;
1382 goto error;
1383 }
1384
1385 err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx],
1386 &skb, CHCR_ENCRYPT_OP);
1387 if (err || !skb)
1388 return err;
1389 skb->dev = u_ctx->lldi.ports[0];
1390 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1391 chcr_send_wr(skb);
1392 if (get_cryptoalg_subtype(tfm) ==
1393 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1394 CRYPTO_TFM_REQ_MAY_SLEEP ) {
1395 reqctx->partial_req = 1;
1396 wait_for_completion(&ctx->cbc_aes_aio_done);
1397 }
1398 return -EINPROGRESS;
1399 error:
1400 chcr_dec_wrcount(dev);
1401 return err;
1402 }
1403
1404 static int chcr_aes_decrypt(struct skcipher_request *req)
1405 {
1406 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1407 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1408 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1409 struct chcr_dev *dev = c_ctx(tfm)->dev;
1410 struct sk_buff *skb = NULL;
1411 int err;
1412 struct chcr_context *ctx = c_ctx(tfm);
1413 unsigned int cpu;
1414
1415 cpu = get_cpu();
1416 reqctx->txqidx = cpu % ctx->ntxq;
1417 reqctx->rxqidx = cpu % ctx->nrxq;
1418 put_cpu();
1419
1420 err = chcr_inc_wrcount(dev);
1421 if (err)
1422 return -ENXIO;
1423
1424 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1425 reqctx->txqidx) &&
1426 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))))
1427 return -ENOSPC;
1428 err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx],
1429 &skb, CHCR_DECRYPT_OP);
1430 if (err || !skb)
1431 return err;
1432 skb->dev = u_ctx->lldi.ports[0];
1433 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
1434 chcr_send_wr(skb);
1435 return -EINPROGRESS;
1436 }
1437 static int chcr_device_init(struct chcr_context *ctx)
1438 {
1439 struct uld_ctx *u_ctx = NULL;
1440 int txq_perchan, ntxq;
1441 int err = 0, rxq_perchan;
1442
1443 if (!ctx->dev) {
1444 u_ctx = assign_chcr_device();
1445 if (!u_ctx) {
1446 pr_err("chcr device assignment fails\n");
1447 goto out;
1448 }
1449 ctx->dev = &u_ctx->dev;
1450 ntxq = u_ctx->lldi.ntxq;
1451 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
1452 txq_perchan = ntxq / u_ctx->lldi.nchan;
1453 ctx->ntxq = ntxq;
1454 ctx->nrxq = u_ctx->lldi.nrxq;
1455 ctx->rxq_perchan = rxq_perchan;
1456 ctx->txq_perchan = txq_perchan;
1457 }
1458 out:
1459 return err;
1460 }
1461
1462 static int chcr_init_tfm(struct crypto_skcipher *tfm)
1463 {
1464 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1465 struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1466 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1467
1468 ablkctx->sw_cipher = crypto_alloc_sync_skcipher(alg->base.cra_name, 0,
1469 CRYPTO_ALG_NEED_FALLBACK);
1470 if (IS_ERR(ablkctx->sw_cipher)) {
1471 pr_err("failed to allocate fallback for %s\n", alg->base.cra_name);
1472 return PTR_ERR(ablkctx->sw_cipher);
1473 }
1474 init_completion(&ctx->cbc_aes_aio_done);
1475 crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx));
1476
1477 return chcr_device_init(ctx);
1478 }
1479
1480 static int chcr_rfc3686_init(struct crypto_skcipher *tfm)
1481 {
1482 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1483 struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1484 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1485
1486 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1487 * cannot be used as fallback in chcr_handle_cipher_response
1488 */
1489 ablkctx->sw_cipher = crypto_alloc_sync_skcipher("ctr(aes)", 0,
1490 CRYPTO_ALG_NEED_FALLBACK);
1491 if (IS_ERR(ablkctx->sw_cipher)) {
1492 pr_err("failed to allocate fallback for %s\n", alg->base.cra_name);
1493 return PTR_ERR(ablkctx->sw_cipher);
1494 }
1495 crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx));
1496 return chcr_device_init(ctx);
1497 }
1498
1499
1500 static void chcr_exit_tfm(struct crypto_skcipher *tfm)
1501 {
1502 struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
1503 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1504
1505 crypto_free_sync_skcipher(ablkctx->sw_cipher);
1506 }
1507
1508 static int get_alg_config(struct algo_param *params,
1509 unsigned int auth_size)
1510 {
1511 switch (auth_size) {
1512 case SHA1_DIGEST_SIZE:
1513 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1514 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1515 params->result_size = SHA1_DIGEST_SIZE;
1516 break;
1517 case SHA224_DIGEST_SIZE:
1518 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1519 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1520 params->result_size = SHA256_DIGEST_SIZE;
1521 break;
1522 case SHA256_DIGEST_SIZE:
1523 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1524 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1525 params->result_size = SHA256_DIGEST_SIZE;
1526 break;
1527 case SHA384_DIGEST_SIZE:
1528 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1529 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1530 params->result_size = SHA512_DIGEST_SIZE;
1531 break;
1532 case SHA512_DIGEST_SIZE:
1533 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1534 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1535 params->result_size = SHA512_DIGEST_SIZE;
1536 break;
1537 default:
1538 pr_err("chcr : ERROR, unsupported digest size\n");
1539 return -EINVAL;
1540 }
1541 return 0;
1542 }
1543
1544 static inline void chcr_free_shash(struct crypto_shash *base_hash)
1545 {
1546 crypto_free_shash(base_hash);
1547 }
1548
1549 /**
1550 * create_hash_wr - Create hash work request
1551 * @req - Cipher req base
1552 */
1553 static struct sk_buff *create_hash_wr(struct ahash_request *req,
1554 struct hash_wr_param *param)
1555 {
1556 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1557 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1558 struct chcr_context *ctx = h_ctx(tfm);
1559 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1560 struct sk_buff *skb = NULL;
1561 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1562 struct chcr_wr *chcr_req;
1563 struct ulptx_sgl *ulptx;
1564 unsigned int nents = 0, transhdr_len;
1565 unsigned int temp = 0;
1566 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1567 GFP_ATOMIC;
1568 struct adapter *adap = padap(h_ctx(tfm)->dev);
1569 int error = 0;
1570 unsigned int rx_channel_id = req_ctx->rxqidx / ctx->rxq_perchan;
1571
1572 transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len);
1573 req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len +
1574 param->sg_len) <= SGE_MAX_WR_LEN;
1575 nents = sg_nents_xlen(req_ctx->hctx_wr.srcsg, param->sg_len,
1576 CHCR_SRC_SG_SIZE, req_ctx->hctx_wr.src_ofst);
1577 nents += param->bfr_len ? 1 : 0;
1578 transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len +
1579 param->sg_len, 16) : (sgl_len(nents) * 8);
1580 transhdr_len = roundup(transhdr_len, 16);
1581
1582 skb = alloc_skb(transhdr_len, flags);
1583 if (!skb)
1584 return ERR_PTR(-ENOMEM);
1585 chcr_req = __skb_put_zero(skb, transhdr_len);
1586
1587 chcr_req->sec_cpl.op_ivinsrtofst =
1588 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 0);
1589
1590 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
1591
1592 chcr_req->sec_cpl.aadstart_cipherstop_hi =
1593 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
1594 chcr_req->sec_cpl.cipherstop_lo_authinsert =
1595 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
1596 chcr_req->sec_cpl.seqno_numivs =
1597 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
1598 param->opad_needed, 0);
1599
1600 chcr_req->sec_cpl.ivgen_hdrlen =
1601 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1602
1603 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1604 param->alg_prm.result_size);
1605
1606 if (param->opad_needed)
1607 memcpy(chcr_req->key_ctx.key +
1608 ((param->alg_prm.result_size <= 32) ? 32 :
1609 CHCR_HASH_MAX_DIGEST_SIZE),
1610 hmacctx->opad, param->alg_prm.result_size);
1611
1612 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
1613 param->alg_prm.mk_size, 0,
1614 param->opad_needed,
1615 ((param->kctx_len +
1616 sizeof(chcr_req->key_ctx)) >> 4));
1617 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
1618 ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len +
1619 DUMMY_BYTES);
1620 if (param->bfr_len != 0) {
1621 req_ctx->hctx_wr.dma_addr =
1622 dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr,
1623 param->bfr_len, DMA_TO_DEVICE);
1624 if (dma_mapping_error(&u_ctx->lldi.pdev->dev,
1625 req_ctx->hctx_wr. dma_addr)) {
1626 error = -ENOMEM;
1627 goto err;
1628 }
1629 req_ctx->hctx_wr.dma_len = param->bfr_len;
1630 } else {
1631 req_ctx->hctx_wr.dma_addr = 0;
1632 }
1633 chcr_add_hash_src_ent(req, ulptx, param);
1634 /* Request upto max wr size */
1635 temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ?
1636 (param->sg_len + param->bfr_len) : 0);
1637 atomic_inc(&adap->chcr_stats.digest_rqst);
1638 create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->hctx_wr.imm,
1639 param->hash_size, transhdr_len,
1640 temp, 0);
1641 req_ctx->hctx_wr.skb = skb;
1642 return skb;
1643 err:
1644 kfree_skb(skb);
1645 return ERR_PTR(error);
1646 }
1647
1648 static int chcr_ahash_update(struct ahash_request *req)
1649 {
1650 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1651 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1652 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1653 struct chcr_context *ctx = h_ctx(rtfm);
1654 struct chcr_dev *dev = h_ctx(rtfm)->dev;
1655 struct sk_buff *skb;
1656 u8 remainder = 0, bs;
1657 unsigned int nbytes = req->nbytes;
1658 struct hash_wr_param params;
1659 int error;
1660 unsigned int cpu;
1661
1662 cpu = get_cpu();
1663 req_ctx->txqidx = cpu % ctx->ntxq;
1664 req_ctx->rxqidx = cpu % ctx->nrxq;
1665 put_cpu();
1666
1667 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1668
1669 if (nbytes + req_ctx->reqlen >= bs) {
1670 remainder = (nbytes + req_ctx->reqlen) % bs;
1671 nbytes = nbytes + req_ctx->reqlen - remainder;
1672 } else {
1673 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
1674 + req_ctx->reqlen, nbytes, 0);
1675 req_ctx->reqlen += nbytes;
1676 return 0;
1677 }
1678 error = chcr_inc_wrcount(dev);
1679 if (error)
1680 return -ENXIO;
1681 /* Detach state for CHCR means lldi or padap is freed. Increasing
1682 * inflight count for dev guarantees that lldi and padap is valid
1683 */
1684 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1685 req_ctx->txqidx) &&
1686 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1687 error = -ENOSPC;
1688 goto err;
1689 }
1690
1691 chcr_init_hctx_per_wr(req_ctx);
1692 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1693 if (error) {
1694 error = -ENOMEM;
1695 goto err;
1696 }
1697 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1698 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1699 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1700 HASH_SPACE_LEFT(params.kctx_len), 0);
1701 if (params.sg_len > req->nbytes)
1702 params.sg_len = req->nbytes;
1703 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) -
1704 req_ctx->reqlen;
1705 params.opad_needed = 0;
1706 params.more = 1;
1707 params.last = 0;
1708 params.bfr_len = req_ctx->reqlen;
1709 params.scmd1 = 0;
1710 req_ctx->hctx_wr.srcsg = req->src;
1711
1712 params.hash_size = params.alg_prm.result_size;
1713 req_ctx->data_len += params.sg_len + params.bfr_len;
1714 skb = create_hash_wr(req, &params);
1715 if (IS_ERR(skb)) {
1716 error = PTR_ERR(skb);
1717 goto unmap;
1718 }
1719
1720 req_ctx->hctx_wr.processed += params.sg_len;
1721 if (remainder) {
1722 /* Swap buffers */
1723 swap(req_ctx->reqbfr, req_ctx->skbfr);
1724 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
1725 req_ctx->reqbfr, remainder, req->nbytes -
1726 remainder);
1727 }
1728 req_ctx->reqlen = remainder;
1729 skb->dev = u_ctx->lldi.ports[0];
1730 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1731 chcr_send_wr(skb);
1732 return -EINPROGRESS;
1733 unmap:
1734 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1735 err:
1736 chcr_dec_wrcount(dev);
1737 return error;
1738 }
1739
1740 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1741 {
1742 memset(bfr_ptr, 0, bs);
1743 *bfr_ptr = 0x80;
1744 if (bs == 64)
1745 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
1746 else
1747 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
1748 }
1749
1750 static int chcr_ahash_final(struct ahash_request *req)
1751 {
1752 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1753 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1754 struct chcr_dev *dev = h_ctx(rtfm)->dev;
1755 struct hash_wr_param params;
1756 struct sk_buff *skb;
1757 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1758 struct chcr_context *ctx = h_ctx(rtfm);
1759 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1760 int error = -EINVAL;
1761 unsigned int cpu;
1762
1763 cpu = get_cpu();
1764 req_ctx->txqidx = cpu % ctx->ntxq;
1765 req_ctx->rxqidx = cpu % ctx->nrxq;
1766 put_cpu();
1767
1768 error = chcr_inc_wrcount(dev);
1769 if (error)
1770 return -ENXIO;
1771
1772 chcr_init_hctx_per_wr(req_ctx);
1773 if (is_hmac(crypto_ahash_tfm(rtfm)))
1774 params.opad_needed = 1;
1775 else
1776 params.opad_needed = 0;
1777 params.sg_len = 0;
1778 req_ctx->hctx_wr.isfinal = 1;
1779 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1780 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1781 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1782 params.opad_needed = 1;
1783 params.kctx_len *= 2;
1784 } else {
1785 params.opad_needed = 0;
1786 }
1787
1788 req_ctx->hctx_wr.result = 1;
1789 params.bfr_len = req_ctx->reqlen;
1790 req_ctx->data_len += params.bfr_len + params.sg_len;
1791 req_ctx->hctx_wr.srcsg = req->src;
1792 if (req_ctx->reqlen == 0) {
1793 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1794 params.last = 0;
1795 params.more = 1;
1796 params.scmd1 = 0;
1797 params.bfr_len = bs;
1798
1799 } else {
1800 params.scmd1 = req_ctx->data_len;
1801 params.last = 1;
1802 params.more = 0;
1803 }
1804 params.hash_size = crypto_ahash_digestsize(rtfm);
1805 skb = create_hash_wr(req, &params);
1806 if (IS_ERR(skb)) {
1807 error = PTR_ERR(skb);
1808 goto err;
1809 }
1810 req_ctx->reqlen = 0;
1811 skb->dev = u_ctx->lldi.ports[0];
1812 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1813 chcr_send_wr(skb);
1814 return -EINPROGRESS;
1815 err:
1816 chcr_dec_wrcount(dev);
1817 return error;
1818 }
1819
1820 static int chcr_ahash_finup(struct ahash_request *req)
1821 {
1822 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1823 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1824 struct chcr_dev *dev = h_ctx(rtfm)->dev;
1825 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1826 struct chcr_context *ctx = h_ctx(rtfm);
1827 struct sk_buff *skb;
1828 struct hash_wr_param params;
1829 u8 bs;
1830 int error;
1831 unsigned int cpu;
1832
1833 cpu = get_cpu();
1834 req_ctx->txqidx = cpu % ctx->ntxq;
1835 req_ctx->rxqidx = cpu % ctx->nrxq;
1836 put_cpu();
1837
1838 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1839 error = chcr_inc_wrcount(dev);
1840 if (error)
1841 return -ENXIO;
1842
1843 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1844 req_ctx->txqidx) &&
1845 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1846 error = -ENOSPC;
1847 goto err;
1848 }
1849 chcr_init_hctx_per_wr(req_ctx);
1850 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1851 if (error) {
1852 error = -ENOMEM;
1853 goto err;
1854 }
1855
1856 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1857 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1858 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1859 params.kctx_len *= 2;
1860 params.opad_needed = 1;
1861 } else {
1862 params.opad_needed = 0;
1863 }
1864
1865 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1866 HASH_SPACE_LEFT(params.kctx_len), 0);
1867 if (params.sg_len < req->nbytes) {
1868 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1869 params.kctx_len /= 2;
1870 params.opad_needed = 0;
1871 }
1872 params.last = 0;
1873 params.more = 1;
1874 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs)
1875 - req_ctx->reqlen;
1876 params.hash_size = params.alg_prm.result_size;
1877 params.scmd1 = 0;
1878 } else {
1879 params.last = 1;
1880 params.more = 0;
1881 params.sg_len = req->nbytes;
1882 params.hash_size = crypto_ahash_digestsize(rtfm);
1883 params.scmd1 = req_ctx->data_len + req_ctx->reqlen +
1884 params.sg_len;
1885 }
1886 params.bfr_len = req_ctx->reqlen;
1887 req_ctx->data_len += params.bfr_len + params.sg_len;
1888 req_ctx->hctx_wr.result = 1;
1889 req_ctx->hctx_wr.srcsg = req->src;
1890 if ((req_ctx->reqlen + req->nbytes) == 0) {
1891 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1892 params.last = 0;
1893 params.more = 1;
1894 params.scmd1 = 0;
1895 params.bfr_len = bs;
1896 }
1897 skb = create_hash_wr(req, &params);
1898 if (IS_ERR(skb)) {
1899 error = PTR_ERR(skb);
1900 goto unmap;
1901 }
1902 req_ctx->reqlen = 0;
1903 req_ctx->hctx_wr.processed += params.sg_len;
1904 skb->dev = u_ctx->lldi.ports[0];
1905 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
1906 chcr_send_wr(skb);
1907 return -EINPROGRESS;
1908 unmap:
1909 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
1910 err:
1911 chcr_dec_wrcount(dev);
1912 return error;
1913 }
1914
1915 static int chcr_ahash_digest(struct ahash_request *req)
1916 {
1917 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1918 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1919 struct chcr_dev *dev = h_ctx(rtfm)->dev;
1920 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1921 struct chcr_context *ctx = h_ctx(rtfm);
1922 struct sk_buff *skb;
1923 struct hash_wr_param params;
1924 u8 bs;
1925 int error;
1926 unsigned int cpu;
1927
1928 cpu = get_cpu();
1929 req_ctx->txqidx = cpu % ctx->ntxq;
1930 req_ctx->rxqidx = cpu % ctx->nrxq;
1931 put_cpu();
1932
1933 rtfm->init(req);
1934 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1935 error = chcr_inc_wrcount(dev);
1936 if (error)
1937 return -ENXIO;
1938
1939 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1940 req_ctx->txqidx) &&
1941 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1942 error = -ENOSPC;
1943 goto err;
1944 }
1945
1946 chcr_init_hctx_per_wr(req_ctx);
1947 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
1948 if (error) {
1949 error = -ENOMEM;
1950 goto err;
1951 }
1952
1953 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1954 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1955 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1956 params.kctx_len *= 2;
1957 params.opad_needed = 1;
1958 } else {
1959 params.opad_needed = 0;
1960 }
1961 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1962 HASH_SPACE_LEFT(params.kctx_len), 0);
1963 if (params.sg_len < req->nbytes) {
1964 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1965 params.kctx_len /= 2;
1966 params.opad_needed = 0;
1967 }
1968 params.last = 0;
1969 params.more = 1;
1970 params.scmd1 = 0;
1971 params.sg_len = rounddown(params.sg_len, bs);
1972 params.hash_size = params.alg_prm.result_size;
1973 } else {
1974 params.sg_len = req->nbytes;
1975 params.hash_size = crypto_ahash_digestsize(rtfm);
1976 params.last = 1;
1977 params.more = 0;
1978 params.scmd1 = req->nbytes + req_ctx->data_len;
1979
1980 }
1981 params.bfr_len = 0;
1982 req_ctx->hctx_wr.result = 1;
1983 req_ctx->hctx_wr.srcsg = req->src;
1984 req_ctx->data_len += params.bfr_len + params.sg_len;
1985
1986 if (req->nbytes == 0) {
1987 create_last_hash_block(req_ctx->reqbfr, bs, 0);
1988 params.more = 1;
1989 params.bfr_len = bs;
1990 }
1991
1992 skb = create_hash_wr(req, &params);
1993 if (IS_ERR(skb)) {
1994 error = PTR_ERR(skb);
1995 goto unmap;
1996 }
1997 req_ctx->hctx_wr.processed += params.sg_len;
1998 skb->dev = u_ctx->lldi.ports[0];
1999 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
2000 chcr_send_wr(skb);
2001 return -EINPROGRESS;
2002 unmap:
2003 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
2004 err:
2005 chcr_dec_wrcount(dev);
2006 return error;
2007 }
2008
2009 static int chcr_ahash_continue(struct ahash_request *req)
2010 {
2011 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2012 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
2013 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
2014 struct chcr_context *ctx = h_ctx(rtfm);
2015 struct uld_ctx *u_ctx = ULD_CTX(ctx);
2016 struct sk_buff *skb;
2017 struct hash_wr_param params;
2018 u8 bs;
2019 int error;
2020 unsigned int cpu;
2021
2022 cpu = get_cpu();
2023 reqctx->txqidx = cpu % ctx->ntxq;
2024 reqctx->rxqidx = cpu % ctx->nrxq;
2025 put_cpu();
2026
2027 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
2028 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
2029 params.kctx_len = roundup(params.alg_prm.result_size, 16);
2030 if (is_hmac(crypto_ahash_tfm(rtfm))) {
2031 params.kctx_len *= 2;
2032 params.opad_needed = 1;
2033 } else {
2034 params.opad_needed = 0;
2035 }
2036 params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0,
2037 HASH_SPACE_LEFT(params.kctx_len),
2038 hctx_wr->src_ofst);
2039 if ((params.sg_len + hctx_wr->processed) > req->nbytes)
2040 params.sg_len = req->nbytes - hctx_wr->processed;
2041 if (!hctx_wr->result ||
2042 ((params.sg_len + hctx_wr->processed) < req->nbytes)) {
2043 if (is_hmac(crypto_ahash_tfm(rtfm))) {
2044 params.kctx_len /= 2;
2045 params.opad_needed = 0;
2046 }
2047 params.last = 0;
2048 params.more = 1;
2049 params.sg_len = rounddown(params.sg_len, bs);
2050 params.hash_size = params.alg_prm.result_size;
2051 params.scmd1 = 0;
2052 } else {
2053 params.last = 1;
2054 params.more = 0;
2055 params.hash_size = crypto_ahash_digestsize(rtfm);
2056 params.scmd1 = reqctx->data_len + params.sg_len;
2057 }
2058 params.bfr_len = 0;
2059 reqctx->data_len += params.sg_len;
2060 skb = create_hash_wr(req, &params);
2061 if (IS_ERR(skb)) {
2062 error = PTR_ERR(skb);
2063 goto err;
2064 }
2065 hctx_wr->processed += params.sg_len;
2066 skb->dev = u_ctx->lldi.ports[0];
2067 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
2068 chcr_send_wr(skb);
2069 return 0;
2070 err:
2071 return error;
2072 }
2073
2074 static inline void chcr_handle_ahash_resp(struct ahash_request *req,
2075 unsigned char *input,
2076 int err)
2077 {
2078 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2079 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
2080 int digestsize, updated_digestsize;
2081 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2082 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
2083 struct chcr_dev *dev = h_ctx(tfm)->dev;
2084
2085 if (input == NULL)
2086 goto out;
2087 digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
2088 updated_digestsize = digestsize;
2089 if (digestsize == SHA224_DIGEST_SIZE)
2090 updated_digestsize = SHA256_DIGEST_SIZE;
2091 else if (digestsize == SHA384_DIGEST_SIZE)
2092 updated_digestsize = SHA512_DIGEST_SIZE;
2093
2094 if (hctx_wr->dma_addr) {
2095 dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr,
2096 hctx_wr->dma_len, DMA_TO_DEVICE);
2097 hctx_wr->dma_addr = 0;
2098 }
2099 if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) ==
2100 req->nbytes)) {
2101 if (hctx_wr->result == 1) {
2102 hctx_wr->result = 0;
2103 memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
2104 digestsize);
2105 } else {
2106 memcpy(reqctx->partial_hash,
2107 input + sizeof(struct cpl_fw6_pld),
2108 updated_digestsize);
2109
2110 }
2111 goto unmap;
2112 }
2113 memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
2114 updated_digestsize);
2115
2116 err = chcr_ahash_continue(req);
2117 if (err)
2118 goto unmap;
2119 return;
2120 unmap:
2121 if (hctx_wr->is_sg_map)
2122 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
2123
2124
2125 out:
2126 chcr_dec_wrcount(dev);
2127 req->base.complete(&req->base, err);
2128 }
2129
2130 /*
2131 * chcr_handle_resp - Unmap the DMA buffers associated with the request
2132 * @req: crypto request
2133 */
2134 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
2135 int err)
2136 {
2137 struct crypto_tfm *tfm = req->tfm;
2138 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2139 struct adapter *adap = padap(ctx->dev);
2140
2141 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
2142 case CRYPTO_ALG_TYPE_AEAD:
2143 err = chcr_handle_aead_resp(aead_request_cast(req), input, err);
2144 break;
2145
2146 case CRYPTO_ALG_TYPE_SKCIPHER:
2147 chcr_handle_cipher_resp(skcipher_request_cast(req),
2148 input, err);
2149 break;
2150 case CRYPTO_ALG_TYPE_AHASH:
2151 chcr_handle_ahash_resp(ahash_request_cast(req), input, err);
2152 }
2153 atomic_inc(&adap->chcr_stats.complete);
2154 return err;
2155 }
2156 static int chcr_ahash_export(struct ahash_request *areq, void *out)
2157 {
2158 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2159 struct chcr_ahash_req_ctx *state = out;
2160
2161 state->reqlen = req_ctx->reqlen;
2162 state->data_len = req_ctx->data_len;
2163 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
2164 memcpy(state->partial_hash, req_ctx->partial_hash,
2165 CHCR_HASH_MAX_DIGEST_SIZE);
2166 chcr_init_hctx_per_wr(state);
2167 return 0;
2168 }
2169
2170 static int chcr_ahash_import(struct ahash_request *areq, const void *in)
2171 {
2172 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2173 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
2174
2175 req_ctx->reqlen = state->reqlen;
2176 req_ctx->data_len = state->data_len;
2177 req_ctx->reqbfr = req_ctx->bfr1;
2178 req_ctx->skbfr = req_ctx->bfr2;
2179 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
2180 memcpy(req_ctx->partial_hash, state->partial_hash,
2181 CHCR_HASH_MAX_DIGEST_SIZE);
2182 chcr_init_hctx_per_wr(req_ctx);
2183 return 0;
2184 }
2185
2186 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2187 unsigned int keylen)
2188 {
2189 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
2190 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2191 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2192 unsigned int i, err = 0, updated_digestsize;
2193
2194 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
2195
2196 /* use the key to calculate the ipad and opad. ipad will sent with the
2197 * first request's data. opad will be sent with the final hash result
2198 * ipad in hmacctx->ipad and opad in hmacctx->opad location
2199 */
2200 shash->tfm = hmacctx->base_hash;
2201 if (keylen > bs) {
2202 err = crypto_shash_digest(shash, key, keylen,
2203 hmacctx->ipad);
2204 if (err)
2205 goto out;
2206 keylen = digestsize;
2207 } else {
2208 memcpy(hmacctx->ipad, key, keylen);
2209 }
2210 memset(hmacctx->ipad + keylen, 0, bs - keylen);
2211 memcpy(hmacctx->opad, hmacctx->ipad, bs);
2212
2213 for (i = 0; i < bs / sizeof(int); i++) {
2214 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
2215 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
2216 }
2217
2218 updated_digestsize = digestsize;
2219 if (digestsize == SHA224_DIGEST_SIZE)
2220 updated_digestsize = SHA256_DIGEST_SIZE;
2221 else if (digestsize == SHA384_DIGEST_SIZE)
2222 updated_digestsize = SHA512_DIGEST_SIZE;
2223 err = chcr_compute_partial_hash(shash, hmacctx->ipad,
2224 hmacctx->ipad, digestsize);
2225 if (err)
2226 goto out;
2227 chcr_change_order(hmacctx->ipad, updated_digestsize);
2228
2229 err = chcr_compute_partial_hash(shash, hmacctx->opad,
2230 hmacctx->opad, digestsize);
2231 if (err)
2232 goto out;
2233 chcr_change_order(hmacctx->opad, updated_digestsize);
2234 out:
2235 return err;
2236 }
2237
2238 static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key,
2239 unsigned int key_len)
2240 {
2241 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
2242 unsigned short context_size = 0;
2243 int err;
2244
2245 err = chcr_cipher_fallback_setkey(cipher, key, key_len);
2246 if (err)
2247 goto badkey_err;
2248
2249 memcpy(ablkctx->key, key, key_len);
2250 ablkctx->enckey_len = key_len;
2251 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
2252 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
2253 ablkctx->key_ctx_hdr =
2254 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
2255 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
2256 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
2257 CHCR_KEYCTX_NO_KEY, 1,
2258 0, context_size);
2259 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
2260 return 0;
2261 badkey_err:
2262 ablkctx->enckey_len = 0;
2263
2264 return err;
2265 }
2266
2267 static int chcr_sha_init(struct ahash_request *areq)
2268 {
2269 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2270 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2271 int digestsize = crypto_ahash_digestsize(tfm);
2272
2273 req_ctx->data_len = 0;
2274 req_ctx->reqlen = 0;
2275 req_ctx->reqbfr = req_ctx->bfr1;
2276 req_ctx->skbfr = req_ctx->bfr2;
2277 copy_hash_init_values(req_ctx->partial_hash, digestsize);
2278
2279 return 0;
2280 }
2281
2282 static int chcr_sha_cra_init(struct crypto_tfm *tfm)
2283 {
2284 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2285 sizeof(struct chcr_ahash_req_ctx));
2286 return chcr_device_init(crypto_tfm_ctx(tfm));
2287 }
2288
2289 static int chcr_hmac_init(struct ahash_request *areq)
2290 {
2291 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2292 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
2293 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm));
2294 unsigned int digestsize = crypto_ahash_digestsize(rtfm);
2295 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
2296
2297 chcr_sha_init(areq);
2298 req_ctx->data_len = bs;
2299 if (is_hmac(crypto_ahash_tfm(rtfm))) {
2300 if (digestsize == SHA224_DIGEST_SIZE)
2301 memcpy(req_ctx->partial_hash, hmacctx->ipad,
2302 SHA256_DIGEST_SIZE);
2303 else if (digestsize == SHA384_DIGEST_SIZE)
2304 memcpy(req_ctx->partial_hash, hmacctx->ipad,
2305 SHA512_DIGEST_SIZE);
2306 else
2307 memcpy(req_ctx->partial_hash, hmacctx->ipad,
2308 digestsize);
2309 }
2310 return 0;
2311 }
2312
2313 static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
2314 {
2315 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2316 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
2317 unsigned int digestsize =
2318 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
2319
2320 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2321 sizeof(struct chcr_ahash_req_ctx));
2322 hmacctx->base_hash = chcr_alloc_shash(digestsize);
2323 if (IS_ERR(hmacctx->base_hash))
2324 return PTR_ERR(hmacctx->base_hash);
2325 return chcr_device_init(crypto_tfm_ctx(tfm));
2326 }
2327
2328 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
2329 {
2330 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2331 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
2332
2333 if (hmacctx->base_hash) {
2334 chcr_free_shash(hmacctx->base_hash);
2335 hmacctx->base_hash = NULL;
2336 }
2337 }
2338
2339 inline void chcr_aead_common_exit(struct aead_request *req)
2340 {
2341 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2342 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2343 struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
2344
2345 chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
2346 }
2347
2348 static int chcr_aead_common_init(struct aead_request *req)
2349 {
2350 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2351 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2352 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2353 unsigned int authsize = crypto_aead_authsize(tfm);
2354 int error = -EINVAL;
2355
2356 /* validate key size */
2357 if (aeadctx->enckey_len == 0)
2358 goto err;
2359 if (reqctx->op && req->cryptlen < authsize)
2360 goto err;
2361 if (reqctx->b0_len)
2362 reqctx->scratch_pad = reqctx->iv + IV;
2363 else
2364 reqctx->scratch_pad = NULL;
2365
2366 error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
2367 reqctx->op);
2368 if (error) {
2369 error = -ENOMEM;
2370 goto err;
2371 }
2372
2373 return 0;
2374 err:
2375 return error;
2376 }
2377
2378 static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents,
2379 int aadmax, int wrlen,
2380 unsigned short op_type)
2381 {
2382 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
2383
2384 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
2385 dst_nents > MAX_DSGL_ENT ||
2386 (req->assoclen > aadmax) ||
2387 (wrlen > SGE_MAX_WR_LEN))
2388 return 1;
2389 return 0;
2390 }
2391
2392 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
2393 {
2394 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2395 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2396 struct aead_request *subreq = aead_request_ctx(req);
2397
2398 aead_request_set_tfm(subreq, aeadctx->sw_cipher);
2399 aead_request_set_callback(subreq, req->base.flags,
2400 req->base.complete, req->base.data);
2401 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
2402 req->iv);
2403 aead_request_set_ad(subreq, req->assoclen);
2404 return op_type ? crypto_aead_decrypt(subreq) :
2405 crypto_aead_encrypt(subreq);
2406 }
2407
2408 static struct sk_buff *create_authenc_wr(struct aead_request *req,
2409 unsigned short qid,
2410 int size)
2411 {
2412 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2413 struct chcr_context *ctx = a_ctx(tfm);
2414 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2415 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2416 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2417 struct sk_buff *skb = NULL;
2418 struct chcr_wr *chcr_req;
2419 struct cpl_rx_phys_dsgl *phys_cpl;
2420 struct ulptx_sgl *ulptx;
2421 unsigned int transhdr_len;
2422 unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm);
2423 unsigned int kctx_len = 0, dnents, snents;
2424 unsigned int authsize = crypto_aead_authsize(tfm);
2425 int error = -EINVAL;
2426 u8 *ivptr;
2427 int null = 0;
2428 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2429 GFP_ATOMIC;
2430 struct adapter *adap = padap(ctx->dev);
2431 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2432
2433 if (req->cryptlen == 0)
2434 return NULL;
2435
2436 reqctx->b0_len = 0;
2437 error = chcr_aead_common_init(req);
2438 if (error)
2439 return ERR_PTR(error);
2440
2441 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL ||
2442 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2443 null = 1;
2444 }
2445 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen +
2446 (reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE, 0);
2447 dnents += MIN_AUTH_SG; // For IV
2448 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
2449 CHCR_SRC_SG_SIZE, 0);
2450 dst_size = get_space_for_phys_dsgl(dnents);
2451 kctx_len = (KEY_CONTEXT_CTX_LEN_G(ntohl(aeadctx->key_ctx_hdr)) << 4)
2452 - sizeof(chcr_req->key_ctx);
2453 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2454 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <
2455 SGE_MAX_WR_LEN;
2456 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16)
2457 : (sgl_len(snents) * 8);
2458 transhdr_len += temp;
2459 transhdr_len = roundup(transhdr_len, 16);
2460
2461 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
2462 transhdr_len, reqctx->op)) {
2463 atomic_inc(&adap->chcr_stats.fallback);
2464 chcr_aead_common_exit(req);
2465 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
2466 }
2467 skb = alloc_skb(transhdr_len, flags);
2468 if (!skb) {
2469 error = -ENOMEM;
2470 goto err;
2471 }
2472
2473 chcr_req = __skb_put_zero(skb, transhdr_len);
2474
2475 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
2476
2477 /*
2478 * Input order is AAD,IV and Payload. where IV should be included as
2479 * the part of authdata. All other fields should be filled according
2480 * to the hardware spec
2481 */
2482 chcr_req->sec_cpl.op_ivinsrtofst =
2483 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
2484 chcr_req->sec_cpl.pldlen = htonl(req->assoclen + IV + req->cryptlen);
2485 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2486 null ? 0 : 1 + IV,
2487 null ? 0 : IV + req->assoclen,
2488 req->assoclen + IV + 1,
2489 (temp & 0x1F0) >> 4);
2490 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2491 temp & 0xF,
2492 null ? 0 : req->assoclen + IV + 1,
2493 temp, temp);
2494 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL ||
2495 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA)
2496 temp = CHCR_SCMD_CIPHER_MODE_AES_CTR;
2497 else
2498 temp = CHCR_SCMD_CIPHER_MODE_AES_CBC;
2499 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op,
2500 (reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0,
2501 temp,
2502 actx->auth_mode, aeadctx->hmac_ctrl,
2503 IV >> 1);
2504 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2505 0, 0, dst_size);
2506
2507 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2508 if (reqctx->op == CHCR_ENCRYPT_OP ||
2509 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2510 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL)
2511 memcpy(chcr_req->key_ctx.key, aeadctx->key,
2512 aeadctx->enckey_len);
2513 else
2514 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2515 aeadctx->enckey_len);
2516
2517 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
2518 actx->h_iopad, kctx_len - roundup(aeadctx->enckey_len, 16));
2519 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2520 ivptr = (u8 *)(phys_cpl + 1) + dst_size;
2521 ulptx = (struct ulptx_sgl *)(ivptr + IV);
2522 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2523 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2524 memcpy(ivptr, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE);
2525 memcpy(ivptr + CTR_RFC3686_NONCE_SIZE, req->iv,
2526 CTR_RFC3686_IV_SIZE);
2527 *(__be32 *)(ivptr + CTR_RFC3686_NONCE_SIZE +
2528 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
2529 } else {
2530 memcpy(ivptr, req->iv, IV);
2531 }
2532 chcr_add_aead_dst_ent(req, phys_cpl, qid);
2533 chcr_add_aead_src_ent(req, ulptx);
2534 atomic_inc(&adap->chcr_stats.cipher_rqst);
2535 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
2536 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0);
2537 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
2538 transhdr_len, temp, 0);
2539 reqctx->skb = skb;
2540
2541 return skb;
2542 err:
2543 chcr_aead_common_exit(req);
2544
2545 return ERR_PTR(error);
2546 }
2547
2548 int chcr_aead_dma_map(struct device *dev,
2549 struct aead_request *req,
2550 unsigned short op_type)
2551 {
2552 int error;
2553 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2554 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2555 unsigned int authsize = crypto_aead_authsize(tfm);
2556 int dst_size;
2557
2558 dst_size = req->assoclen + req->cryptlen + (op_type ?
2559 -authsize : authsize);
2560 if (!req->cryptlen || !dst_size)
2561 return 0;
2562 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len),
2563 DMA_BIDIRECTIONAL);
2564 if (dma_mapping_error(dev, reqctx->iv_dma))
2565 return -ENOMEM;
2566 if (reqctx->b0_len)
2567 reqctx->b0_dma = reqctx->iv_dma + IV;
2568 else
2569 reqctx->b0_dma = 0;
2570 if (req->src == req->dst) {
2571 error = dma_map_sg(dev, req->src,
2572 sg_nents_for_len(req->src, dst_size),
2573 DMA_BIDIRECTIONAL);
2574 if (!error)
2575 goto err;
2576 } else {
2577 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2578 DMA_TO_DEVICE);
2579 if (!error)
2580 goto err;
2581 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2582 DMA_FROM_DEVICE);
2583 if (!error) {
2584 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2585 DMA_TO_DEVICE);
2586 goto err;
2587 }
2588 }
2589
2590 return 0;
2591 err:
2592 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
2593 return -ENOMEM;
2594 }
2595
2596 void chcr_aead_dma_unmap(struct device *dev,
2597 struct aead_request *req,
2598 unsigned short op_type)
2599 {
2600 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2601 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2602 unsigned int authsize = crypto_aead_authsize(tfm);
2603 int dst_size;
2604
2605 dst_size = req->assoclen + req->cryptlen + (op_type ?
2606 -authsize : authsize);
2607 if (!req->cryptlen || !dst_size)
2608 return;
2609
2610 dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len),
2611 DMA_BIDIRECTIONAL);
2612 if (req->src == req->dst) {
2613 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2614 DMA_BIDIRECTIONAL);
2615 } else {
2616 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2617 DMA_TO_DEVICE);
2618 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2619 DMA_FROM_DEVICE);
2620 }
2621 }
2622
2623 void chcr_add_aead_src_ent(struct aead_request *req,
2624 struct ulptx_sgl *ulptx)
2625 {
2626 struct ulptx_walk ulp_walk;
2627 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2628
2629 if (reqctx->imm) {
2630 u8 *buf = (u8 *)ulptx;
2631
2632 if (reqctx->b0_len) {
2633 memcpy(buf, reqctx->scratch_pad, reqctx->b0_len);
2634 buf += reqctx->b0_len;
2635 }
2636 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2637 buf, req->cryptlen + req->assoclen, 0);
2638 } else {
2639 ulptx_walk_init(&ulp_walk, ulptx);
2640 if (reqctx->b0_len)
2641 ulptx_walk_add_page(&ulp_walk, reqctx->b0_len,
2642 reqctx->b0_dma);
2643 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen +
2644 req->assoclen, 0);
2645 ulptx_walk_end(&ulp_walk);
2646 }
2647 }
2648
2649 void chcr_add_aead_dst_ent(struct aead_request *req,
2650 struct cpl_rx_phys_dsgl *phys_cpl,
2651 unsigned short qid)
2652 {
2653 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2654 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2655 struct dsgl_walk dsgl_walk;
2656 unsigned int authsize = crypto_aead_authsize(tfm);
2657 struct chcr_context *ctx = a_ctx(tfm);
2658 u32 temp;
2659 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2660
2661 dsgl_walk_init(&dsgl_walk, phys_cpl);
2662 dsgl_walk_add_page(&dsgl_walk, IV + reqctx->b0_len, reqctx->iv_dma);
2663 temp = req->assoclen + req->cryptlen +
2664 (reqctx->op ? -authsize : authsize);
2665 dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, 0);
2666 dsgl_walk_end(&dsgl_walk, qid, rx_channel_id);
2667 }
2668
2669 void chcr_add_cipher_src_ent(struct skcipher_request *req,
2670 void *ulptx,
2671 struct cipher_wr_param *wrparam)
2672 {
2673 struct ulptx_walk ulp_walk;
2674 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2675 u8 *buf = ulptx;
2676
2677 memcpy(buf, reqctx->iv, IV);
2678 buf += IV;
2679 if (reqctx->imm) {
2680 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2681 buf, wrparam->bytes, reqctx->processed);
2682 } else {
2683 ulptx_walk_init(&ulp_walk, (struct ulptx_sgl *)buf);
2684 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes,
2685 reqctx->src_ofst);
2686 reqctx->srcsg = ulp_walk.last_sg;
2687 reqctx->src_ofst = ulp_walk.last_sg_len;
2688 ulptx_walk_end(&ulp_walk);
2689 }
2690 }
2691
2692 void chcr_add_cipher_dst_ent(struct skcipher_request *req,
2693 struct cpl_rx_phys_dsgl *phys_cpl,
2694 struct cipher_wr_param *wrparam,
2695 unsigned short qid)
2696 {
2697 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2698 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req);
2699 struct chcr_context *ctx = c_ctx(tfm);
2700 struct dsgl_walk dsgl_walk;
2701 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2702
2703 dsgl_walk_init(&dsgl_walk, phys_cpl);
2704 dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes,
2705 reqctx->dst_ofst);
2706 reqctx->dstsg = dsgl_walk.last_sg;
2707 reqctx->dst_ofst = dsgl_walk.last_sg_len;
2708 dsgl_walk_end(&dsgl_walk, qid, rx_channel_id);
2709 }
2710
2711 void chcr_add_hash_src_ent(struct ahash_request *req,
2712 struct ulptx_sgl *ulptx,
2713 struct hash_wr_param *param)
2714 {
2715 struct ulptx_walk ulp_walk;
2716 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2717
2718 if (reqctx->hctx_wr.imm) {
2719 u8 *buf = (u8 *)ulptx;
2720
2721 if (param->bfr_len) {
2722 memcpy(buf, reqctx->reqbfr, param->bfr_len);
2723 buf += param->bfr_len;
2724 }
2725
2726 sg_pcopy_to_buffer(reqctx->hctx_wr.srcsg,
2727 sg_nents(reqctx->hctx_wr.srcsg), buf,
2728 param->sg_len, 0);
2729 } else {
2730 ulptx_walk_init(&ulp_walk, ulptx);
2731 if (param->bfr_len)
2732 ulptx_walk_add_page(&ulp_walk, param->bfr_len,
2733 reqctx->hctx_wr.dma_addr);
2734 ulptx_walk_add_sg(&ulp_walk, reqctx->hctx_wr.srcsg,
2735 param->sg_len, reqctx->hctx_wr.src_ofst);
2736 reqctx->hctx_wr.srcsg = ulp_walk.last_sg;
2737 reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len;
2738 ulptx_walk_end(&ulp_walk);
2739 }
2740 }
2741
2742 int chcr_hash_dma_map(struct device *dev,
2743 struct ahash_request *req)
2744 {
2745 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2746 int error = 0;
2747
2748 if (!req->nbytes)
2749 return 0;
2750 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2751 DMA_TO_DEVICE);
2752 if (!error)
2753 return -ENOMEM;
2754 req_ctx->hctx_wr.is_sg_map = 1;
2755 return 0;
2756 }
2757
2758 void chcr_hash_dma_unmap(struct device *dev,
2759 struct ahash_request *req)
2760 {
2761 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2762
2763 if (!req->nbytes)
2764 return;
2765
2766 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2767 DMA_TO_DEVICE);
2768 req_ctx->hctx_wr.is_sg_map = 0;
2769
2770 }
2771
2772 int chcr_cipher_dma_map(struct device *dev,
2773 struct skcipher_request *req)
2774 {
2775 int error;
2776
2777 if (req->src == req->dst) {
2778 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2779 DMA_BIDIRECTIONAL);
2780 if (!error)
2781 goto err;
2782 } else {
2783 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2784 DMA_TO_DEVICE);
2785 if (!error)
2786 goto err;
2787 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2788 DMA_FROM_DEVICE);
2789 if (!error) {
2790 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2791 DMA_TO_DEVICE);
2792 goto err;
2793 }
2794 }
2795
2796 return 0;
2797 err:
2798 return -ENOMEM;
2799 }
2800
2801 void chcr_cipher_dma_unmap(struct device *dev,
2802 struct skcipher_request *req)
2803 {
2804 if (req->src == req->dst) {
2805 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2806 DMA_BIDIRECTIONAL);
2807 } else {
2808 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2809 DMA_TO_DEVICE);
2810 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2811 DMA_FROM_DEVICE);
2812 }
2813 }
2814
2815 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2816 {
2817 __be32 data;
2818
2819 memset(block, 0, csize);
2820 block += csize;
2821
2822 if (csize >= 4)
2823 csize = 4;
2824 else if (msglen > (unsigned int)(1 << (8 * csize)))
2825 return -EOVERFLOW;
2826
2827 data = cpu_to_be32(msglen);
2828 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2829
2830 return 0;
2831 }
2832
2833 static int generate_b0(struct aead_request *req, u8 *ivptr,
2834 unsigned short op_type)
2835 {
2836 unsigned int l, lp, m;
2837 int rc;
2838 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2839 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2840 u8 *b0 = reqctx->scratch_pad;
2841
2842 m = crypto_aead_authsize(aead);
2843
2844 memcpy(b0, ivptr, 16);
2845
2846 lp = b0[0];
2847 l = lp + 1;
2848
2849 /* set m, bits 3-5 */
2850 *b0 |= (8 * ((m - 2) / 2));
2851
2852 /* set adata, bit 6, if associated data is used */
2853 if (req->assoclen)
2854 *b0 |= 64;
2855 rc = set_msg_len(b0 + 16 - l,
2856 (op_type == CHCR_DECRYPT_OP) ?
2857 req->cryptlen - m : req->cryptlen, l);
2858
2859 return rc;
2860 }
2861
2862 static inline int crypto_ccm_check_iv(const u8 *iv)
2863 {
2864 /* 2 <= L <= 8, so 1 <= L' <= 7. */
2865 if (iv[0] < 1 || iv[0] > 7)
2866 return -EINVAL;
2867
2868 return 0;
2869 }
2870
2871 static int ccm_format_packet(struct aead_request *req,
2872 u8 *ivptr,
2873 unsigned int sub_type,
2874 unsigned short op_type,
2875 unsigned int assoclen)
2876 {
2877 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2878 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2879 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2880 int rc = 0;
2881
2882 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2883 ivptr[0] = 3;
2884 memcpy(ivptr + 1, &aeadctx->salt[0], 3);
2885 memcpy(ivptr + 4, req->iv, 8);
2886 memset(ivptr + 12, 0, 4);
2887 } else {
2888 memcpy(ivptr, req->iv, 16);
2889 }
2890 if (assoclen)
2891 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2892 htons(assoclen);
2893
2894 rc = generate_b0(req, ivptr, op_type);
2895 /* zero the ctr value */
2896 memset(ivptr + 15 - ivptr[0], 0, ivptr[0] + 1);
2897 return rc;
2898 }
2899
2900 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2901 unsigned int dst_size,
2902 struct aead_request *req,
2903 unsigned short op_type)
2904 {
2905 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2906 struct chcr_context *ctx = a_ctx(tfm);
2907 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2908 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2909 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2910 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
2911 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2912 unsigned int ccm_xtra;
2913 unsigned char tag_offset = 0, auth_offset = 0;
2914 unsigned int assoclen;
2915
2916 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2917 assoclen = req->assoclen - 8;
2918 else
2919 assoclen = req->assoclen;
2920 ccm_xtra = CCM_B0_SIZE +
2921 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2922
2923 auth_offset = req->cryptlen ?
2924 (req->assoclen + IV + 1 + ccm_xtra) : 0;
2925 if (op_type == CHCR_DECRYPT_OP) {
2926 if (crypto_aead_authsize(tfm) != req->cryptlen)
2927 tag_offset = crypto_aead_authsize(tfm);
2928 else
2929 auth_offset = 0;
2930 }
2931
2932 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
2933 sec_cpl->pldlen =
2934 htonl(req->assoclen + IV + req->cryptlen + ccm_xtra);
2935 /* For CCM there wil be b0 always. So AAD start will be 1 always */
2936 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2937 1 + IV, IV + assoclen + ccm_xtra,
2938 req->assoclen + IV + 1 + ccm_xtra, 0);
2939
2940 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2941 auth_offset, tag_offset,
2942 (op_type == CHCR_ENCRYPT_OP) ? 0 :
2943 crypto_aead_authsize(tfm));
2944 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2945 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
2946 cipher_mode, mac_mode,
2947 aeadctx->hmac_ctrl, IV >> 1);
2948
2949 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2950 0, dst_size);
2951 }
2952
2953 static int aead_ccm_validate_input(unsigned short op_type,
2954 struct aead_request *req,
2955 struct chcr_aead_ctx *aeadctx,
2956 unsigned int sub_type)
2957 {
2958 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2959 if (crypto_ccm_check_iv(req->iv)) {
2960 pr_err("CCM: IV check fails\n");
2961 return -EINVAL;
2962 }
2963 } else {
2964 if (req->assoclen != 16 && req->assoclen != 20) {
2965 pr_err("RFC4309: Invalid AAD length %d\n",
2966 req->assoclen);
2967 return -EINVAL;
2968 }
2969 }
2970 return 0;
2971 }
2972
2973 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
2974 unsigned short qid,
2975 int size)
2976 {
2977 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2978 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2979 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2980 struct sk_buff *skb = NULL;
2981 struct chcr_wr *chcr_req;
2982 struct cpl_rx_phys_dsgl *phys_cpl;
2983 struct ulptx_sgl *ulptx;
2984 unsigned int transhdr_len;
2985 unsigned int dst_size = 0, kctx_len, dnents, temp, snents;
2986 unsigned int sub_type, assoclen = req->assoclen;
2987 unsigned int authsize = crypto_aead_authsize(tfm);
2988 int error = -EINVAL;
2989 u8 *ivptr;
2990 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2991 GFP_ATOMIC;
2992 struct adapter *adap = padap(a_ctx(tfm)->dev);
2993
2994 sub_type = get_aead_subtype(tfm);
2995 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2996 assoclen -= 8;
2997 reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
2998 error = chcr_aead_common_init(req);
2999 if (error)
3000 return ERR_PTR(error);
3001
3002 error = aead_ccm_validate_input(reqctx->op, req, aeadctx, sub_type);
3003 if (error)
3004 goto err;
3005 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen
3006 + (reqctx->op ? -authsize : authsize),
3007 CHCR_DST_SG_SIZE, 0);
3008 dnents += MIN_CCM_SG; // For IV and B0
3009 dst_size = get_space_for_phys_dsgl(dnents);
3010 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
3011 CHCR_SRC_SG_SIZE, 0);
3012 snents += MIN_CCM_SG; //For B0
3013 kctx_len = roundup(aeadctx->enckey_len, 16) * 2;
3014 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
3015 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen +
3016 reqctx->b0_len) <= SGE_MAX_WR_LEN;
3017 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen +
3018 reqctx->b0_len, 16) :
3019 (sgl_len(snents) * 8);
3020 transhdr_len += temp;
3021 transhdr_len = roundup(transhdr_len, 16);
3022
3023 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE -
3024 reqctx->b0_len, transhdr_len, reqctx->op)) {
3025 atomic_inc(&adap->chcr_stats.fallback);
3026 chcr_aead_common_exit(req);
3027 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
3028 }
3029 skb = alloc_skb(transhdr_len, flags);
3030
3031 if (!skb) {
3032 error = -ENOMEM;
3033 goto err;
3034 }
3035
3036 chcr_req = __skb_put_zero(skb, transhdr_len);
3037
3038 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, reqctx->op);
3039
3040 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3041 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
3042 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3043 aeadctx->key, aeadctx->enckey_len);
3044
3045 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
3046 ivptr = (u8 *)(phys_cpl + 1) + dst_size;
3047 ulptx = (struct ulptx_sgl *)(ivptr + IV);
3048 error = ccm_format_packet(req, ivptr, sub_type, reqctx->op, assoclen);
3049 if (error)
3050 goto dstmap_fail;
3051 chcr_add_aead_dst_ent(req, phys_cpl, qid);
3052 chcr_add_aead_src_ent(req, ulptx);
3053
3054 atomic_inc(&adap->chcr_stats.aead_rqst);
3055 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
3056 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen +
3057 reqctx->b0_len) : 0);
3058 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0,
3059 transhdr_len, temp, 0);
3060 reqctx->skb = skb;
3061
3062 return skb;
3063 dstmap_fail:
3064 kfree_skb(skb);
3065 err:
3066 chcr_aead_common_exit(req);
3067 return ERR_PTR(error);
3068 }
3069
3070 static struct sk_buff *create_gcm_wr(struct aead_request *req,
3071 unsigned short qid,
3072 int size)
3073 {
3074 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3075 struct chcr_context *ctx = a_ctx(tfm);
3076 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3077 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3078 struct sk_buff *skb = NULL;
3079 struct chcr_wr *chcr_req;
3080 struct cpl_rx_phys_dsgl *phys_cpl;
3081 struct ulptx_sgl *ulptx;
3082 unsigned int transhdr_len, dnents = 0, snents;
3083 unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen;
3084 unsigned int authsize = crypto_aead_authsize(tfm);
3085 int error = -EINVAL;
3086 u8 *ivptr;
3087 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
3088 GFP_ATOMIC;
3089 struct adapter *adap = padap(ctx->dev);
3090 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
3091
3092 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
3093 assoclen = req->assoclen - 8;
3094
3095 reqctx->b0_len = 0;
3096 error = chcr_aead_common_init(req);
3097 if (error)
3098 return ERR_PTR(error);
3099 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen +
3100 (reqctx->op ? -authsize : authsize),
3101 CHCR_DST_SG_SIZE, 0);
3102 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
3103 CHCR_SRC_SG_SIZE, 0);
3104 dnents += MIN_GCM_SG; // For IV
3105 dst_size = get_space_for_phys_dsgl(dnents);
3106 kctx_len = roundup(aeadctx->enckey_len, 16) + AEAD_H_SIZE;
3107 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
3108 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <=
3109 SGE_MAX_WR_LEN;
3110 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) :
3111 (sgl_len(snents) * 8);
3112 transhdr_len += temp;
3113 transhdr_len = roundup(transhdr_len, 16);
3114 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
3115 transhdr_len, reqctx->op)) {
3116
3117 atomic_inc(&adap->chcr_stats.fallback);
3118 chcr_aead_common_exit(req);
3119 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
3120 }
3121 skb = alloc_skb(transhdr_len, flags);
3122 if (!skb) {
3123 error = -ENOMEM;
3124 goto err;
3125 }
3126
3127 chcr_req = __skb_put_zero(skb, transhdr_len);
3128
3129 //Offset of tag from end
3130 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
3131 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
3132 rx_channel_id, 2, 1);
3133 chcr_req->sec_cpl.pldlen =
3134 htonl(req->assoclen + IV + req->cryptlen);
3135 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
3136 assoclen ? 1 + IV : 0,
3137 assoclen ? IV + assoclen : 0,
3138 req->assoclen + IV + 1, 0);
3139 chcr_req->sec_cpl.cipherstop_lo_authinsert =
3140 FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + IV + 1,
3141 temp, temp);
3142 chcr_req->sec_cpl.seqno_numivs =
3143 FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op ==
3144 CHCR_ENCRYPT_OP) ? 1 : 0,
3145 CHCR_SCMD_CIPHER_MODE_AES_GCM,
3146 CHCR_SCMD_AUTH_MODE_GHASH,
3147 aeadctx->hmac_ctrl, IV >> 1);
3148 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
3149 0, 0, dst_size);
3150 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3151 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
3152 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3153 GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
3154
3155 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
3156 ivptr = (u8 *)(phys_cpl + 1) + dst_size;
3157 /* prepare a 16 byte iv */
3158 /* S A L T | IV | 0x00000001 */
3159 if (get_aead_subtype(tfm) ==
3160 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
3161 memcpy(ivptr, aeadctx->salt, 4);
3162 memcpy(ivptr + 4, req->iv, GCM_RFC4106_IV_SIZE);
3163 } else {
3164 memcpy(ivptr, req->iv, GCM_AES_IV_SIZE);
3165 }
3166 *((unsigned int *)(ivptr + 12)) = htonl(0x01);
3167
3168 ulptx = (struct ulptx_sgl *)(ivptr + 16);
3169
3170 chcr_add_aead_dst_ent(req, phys_cpl, qid);
3171 chcr_add_aead_src_ent(req, ulptx);
3172 atomic_inc(&adap->chcr_stats.aead_rqst);
3173 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
3174 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0);
3175 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
3176 transhdr_len, temp, reqctx->verify);
3177 reqctx->skb = skb;
3178 return skb;
3179
3180 err:
3181 chcr_aead_common_exit(req);
3182 return ERR_PTR(error);
3183 }
3184
3185
3186
3187 static int chcr_aead_cra_init(struct crypto_aead *tfm)
3188 {
3189 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3190 struct aead_alg *alg = crypto_aead_alg(tfm);
3191
3192 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
3193 CRYPTO_ALG_NEED_FALLBACK |
3194 CRYPTO_ALG_ASYNC);
3195 if (IS_ERR(aeadctx->sw_cipher))
3196 return PTR_ERR(aeadctx->sw_cipher);
3197 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
3198 sizeof(struct aead_request) +
3199 crypto_aead_reqsize(aeadctx->sw_cipher)));
3200 return chcr_device_init(a_ctx(tfm));
3201 }
3202
3203 static void chcr_aead_cra_exit(struct crypto_aead *tfm)
3204 {
3205 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3206
3207 crypto_free_aead(aeadctx->sw_cipher);
3208 }
3209
3210 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
3211 unsigned int authsize)
3212 {
3213 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3214
3215 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
3216 aeadctx->mayverify = VERIFY_HW;
3217 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3218 }
3219 static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
3220 unsigned int authsize)
3221 {
3222 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3223 u32 maxauth = crypto_aead_maxauthsize(tfm);
3224
3225 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
3226 * true for sha1. authsize == 12 condition should be before
3227 * authsize == (maxauth >> 1)
3228 */
3229 if (authsize == ICV_4) {
3230 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3231 aeadctx->mayverify = VERIFY_HW;
3232 } else if (authsize == ICV_6) {
3233 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3234 aeadctx->mayverify = VERIFY_HW;
3235 } else if (authsize == ICV_10) {
3236 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3237 aeadctx->mayverify = VERIFY_HW;
3238 } else if (authsize == ICV_12) {
3239 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3240 aeadctx->mayverify = VERIFY_HW;
3241 } else if (authsize == ICV_14) {
3242 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3243 aeadctx->mayverify = VERIFY_HW;
3244 } else if (authsize == (maxauth >> 1)) {
3245 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3246 aeadctx->mayverify = VERIFY_HW;
3247 } else if (authsize == maxauth) {
3248 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3249 aeadctx->mayverify = VERIFY_HW;
3250 } else {
3251 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3252 aeadctx->mayverify = VERIFY_SW;
3253 }
3254 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3255 }
3256
3257
3258 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
3259 {
3260 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3261
3262 switch (authsize) {
3263 case ICV_4:
3264 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3265 aeadctx->mayverify = VERIFY_HW;
3266 break;
3267 case ICV_8:
3268 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3269 aeadctx->mayverify = VERIFY_HW;
3270 break;
3271 case ICV_12:
3272 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3273 aeadctx->mayverify = VERIFY_HW;
3274 break;
3275 case ICV_14:
3276 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3277 aeadctx->mayverify = VERIFY_HW;
3278 break;
3279 case ICV_16:
3280 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3281 aeadctx->mayverify = VERIFY_HW;
3282 break;
3283 case ICV_13:
3284 case ICV_15:
3285 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3286 aeadctx->mayverify = VERIFY_SW;
3287 break;
3288 default:
3289 return -EINVAL;
3290 }
3291 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3292 }
3293
3294 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
3295 unsigned int authsize)
3296 {
3297 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3298
3299 switch (authsize) {
3300 case ICV_8:
3301 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3302 aeadctx->mayverify = VERIFY_HW;
3303 break;
3304 case ICV_12:
3305 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3306 aeadctx->mayverify = VERIFY_HW;
3307 break;
3308 case ICV_16:
3309 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3310 aeadctx->mayverify = VERIFY_HW;
3311 break;
3312 default:
3313 return -EINVAL;
3314 }
3315 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3316 }
3317
3318 static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
3319 unsigned int authsize)
3320 {
3321 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
3322
3323 switch (authsize) {
3324 case ICV_4:
3325 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3326 aeadctx->mayverify = VERIFY_HW;
3327 break;
3328 case ICV_6:
3329 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3330 aeadctx->mayverify = VERIFY_HW;
3331 break;
3332 case ICV_8:
3333 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3334 aeadctx->mayverify = VERIFY_HW;
3335 break;
3336 case ICV_10:
3337 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3338 aeadctx->mayverify = VERIFY_HW;
3339 break;
3340 case ICV_12:
3341 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3342 aeadctx->mayverify = VERIFY_HW;
3343 break;
3344 case ICV_14:
3345 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3346 aeadctx->mayverify = VERIFY_HW;
3347 break;
3348 case ICV_16:
3349 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3350 aeadctx->mayverify = VERIFY_HW;
3351 break;
3352 default:
3353 return -EINVAL;
3354 }
3355 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
3356 }
3357
3358 static int chcr_ccm_common_setkey(struct crypto_aead *aead,
3359 const u8 *key,
3360 unsigned int keylen)
3361 {
3362 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3363 unsigned char ck_size, mk_size;
3364 int key_ctx_size = 0;
3365
3366 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) * 2;
3367 if (keylen == AES_KEYSIZE_128) {
3368 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3369 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
3370 } else if (keylen == AES_KEYSIZE_192) {
3371 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3372 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
3373 } else if (keylen == AES_KEYSIZE_256) {
3374 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3375 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
3376 } else {
3377 aeadctx->enckey_len = 0;
3378 return -EINVAL;
3379 }
3380 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
3381 key_ctx_size >> 4);
3382 memcpy(aeadctx->key, key, keylen);
3383 aeadctx->enckey_len = keylen;
3384
3385 return 0;
3386 }
3387
3388 static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
3389 const u8 *key,
3390 unsigned int keylen)
3391 {
3392 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3393 int error;
3394
3395 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3396 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3397 CRYPTO_TFM_REQ_MASK);
3398 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3399 if (error)
3400 return error;
3401 return chcr_ccm_common_setkey(aead, key, keylen);
3402 }
3403
3404 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
3405 unsigned int keylen)
3406 {
3407 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3408 int error;
3409
3410 if (keylen < 3) {
3411 aeadctx->enckey_len = 0;
3412 return -EINVAL;
3413 }
3414 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3415 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3416 CRYPTO_TFM_REQ_MASK);
3417 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3418 if (error)
3419 return error;
3420 keylen -= 3;
3421 memcpy(aeadctx->salt, key + keylen, 3);
3422 return chcr_ccm_common_setkey(aead, key, keylen);
3423 }
3424
3425 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
3426 unsigned int keylen)
3427 {
3428 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
3429 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
3430 unsigned int ck_size;
3431 int ret = 0, key_ctx_size = 0;
3432 struct crypto_aes_ctx aes;
3433
3434 aeadctx->enckey_len = 0;
3435 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3436 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
3437 & CRYPTO_TFM_REQ_MASK);
3438 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3439 if (ret)
3440 goto out;
3441
3442 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3443 keylen > 3) {
3444 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
3445 memcpy(aeadctx->salt, key + keylen, 4);
3446 }
3447 if (keylen == AES_KEYSIZE_128) {
3448 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3449 } else if (keylen == AES_KEYSIZE_192) {
3450 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3451 } else if (keylen == AES_KEYSIZE_256) {
3452 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3453 } else {
3454 pr_err("GCM: Invalid key length %d\n", keylen);
3455 ret = -EINVAL;
3456 goto out;
3457 }
3458
3459 memcpy(aeadctx->key, key, keylen);
3460 aeadctx->enckey_len = keylen;
3461 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) +
3462 AEAD_H_SIZE;
3463 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
3464 CHCR_KEYCTX_MAC_KEY_SIZE_128,
3465 0, 0,
3466 key_ctx_size >> 4);
3467 /* Calculate the H = CIPH(K, 0 repeated 16 times).
3468 * It will go in key context
3469 */
3470 ret = aes_expandkey(&aes, key, keylen);
3471 if (ret) {
3472 aeadctx->enckey_len = 0;
3473 goto out;
3474 }
3475 memset(gctx->ghash_h, 0, AEAD_H_SIZE);
3476 aes_encrypt(&aes, gctx->ghash_h, gctx->ghash_h);
3477 memzero_explicit(&aes, sizeof(aes));
3478
3479 out:
3480 return ret;
3481 }
3482
3483 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
3484 unsigned int keylen)
3485 {
3486 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
3487 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3488 /* it contains auth and cipher key both*/
3489 struct crypto_authenc_keys keys;
3490 unsigned int bs, subtype;
3491 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
3492 int err = 0, i, key_ctx_len = 0;
3493 unsigned char ck_size = 0;
3494 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
3495 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
3496 struct algo_param param;
3497 int align;
3498 u8 *o_ptr = NULL;
3499
3500 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3501 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3502 & CRYPTO_TFM_REQ_MASK);
3503 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3504 if (err)
3505 goto out;
3506
3507 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
3508 goto out;
3509
3510 if (get_alg_config(&param, max_authsize)) {
3511 pr_err("chcr : Unsupported digest size\n");
3512 goto out;
3513 }
3514 subtype = get_aead_subtype(authenc);
3515 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3516 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3517 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3518 goto out;
3519 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3520 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3521 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3522 }
3523 if (keys.enckeylen == AES_KEYSIZE_128) {
3524 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3525 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3526 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3527 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3528 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3529 } else {
3530 pr_err("chcr : Unsupported cipher key\n");
3531 goto out;
3532 }
3533
3534 /* Copy only encryption key. We use authkey to generate h(ipad) and
3535 * h(opad) so authkey is not needed again. authkeylen size have the
3536 * size of the hash digest size.
3537 */
3538 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3539 aeadctx->enckey_len = keys.enckeylen;
3540 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3541 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3542
3543 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3544 aeadctx->enckey_len << 3);
3545 }
3546 base_hash = chcr_alloc_shash(max_authsize);
3547 if (IS_ERR(base_hash)) {
3548 pr_err("chcr : Base driver cannot be loaded\n");
3549 aeadctx->enckey_len = 0;
3550 memzero_explicit(&keys, sizeof(keys));
3551 return -EINVAL;
3552 }
3553 {
3554 SHASH_DESC_ON_STACK(shash, base_hash);
3555
3556 shash->tfm = base_hash;
3557 bs = crypto_shash_blocksize(base_hash);
3558 align = KEYCTX_ALIGN_PAD(max_authsize);
3559 o_ptr = actx->h_iopad + param.result_size + align;
3560
3561 if (keys.authkeylen > bs) {
3562 err = crypto_shash_digest(shash, keys.authkey,
3563 keys.authkeylen,
3564 o_ptr);
3565 if (err) {
3566 pr_err("chcr : Base driver cannot be loaded\n");
3567 goto out;
3568 }
3569 keys.authkeylen = max_authsize;
3570 } else
3571 memcpy(o_ptr, keys.authkey, keys.authkeylen);
3572
3573 /* Compute the ipad-digest*/
3574 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3575 memcpy(pad, o_ptr, keys.authkeylen);
3576 for (i = 0; i < bs >> 2; i++)
3577 *((unsigned int *)pad + i) ^= IPAD_DATA;
3578
3579 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
3580 max_authsize))
3581 goto out;
3582 /* Compute the opad-digest */
3583 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3584 memcpy(pad, o_ptr, keys.authkeylen);
3585 for (i = 0; i < bs >> 2; i++)
3586 *((unsigned int *)pad + i) ^= OPAD_DATA;
3587
3588 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
3589 goto out;
3590
3591 /* convert the ipad and opad digest to network order */
3592 chcr_change_order(actx->h_iopad, param.result_size);
3593 chcr_change_order(o_ptr, param.result_size);
3594 key_ctx_len = sizeof(struct _key_ctx) +
3595 roundup(keys.enckeylen, 16) +
3596 (param.result_size + align) * 2;
3597 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
3598 0, 1, key_ctx_len >> 4);
3599 actx->auth_mode = param.auth_mode;
3600 chcr_free_shash(base_hash);
3601
3602 memzero_explicit(&keys, sizeof(keys));
3603 return 0;
3604 }
3605 out:
3606 aeadctx->enckey_len = 0;
3607 memzero_explicit(&keys, sizeof(keys));
3608 if (!IS_ERR(base_hash))
3609 chcr_free_shash(base_hash);
3610 return -EINVAL;
3611 }
3612
3613 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3614 const u8 *key, unsigned int keylen)
3615 {
3616 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
3617 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3618 struct crypto_authenc_keys keys;
3619 int err;
3620 /* it contains auth and cipher key both*/
3621 unsigned int subtype;
3622 int key_ctx_len = 0;
3623 unsigned char ck_size = 0;
3624
3625 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3626 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3627 & CRYPTO_TFM_REQ_MASK);
3628 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3629 if (err)
3630 goto out;
3631
3632 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
3633 goto out;
3634
3635 subtype = get_aead_subtype(authenc);
3636 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3637 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3638 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3639 goto out;
3640 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3641 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3642 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3643 }
3644 if (keys.enckeylen == AES_KEYSIZE_128) {
3645 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3646 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3647 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3648 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3649 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3650 } else {
3651 pr_err("chcr : Unsupported cipher key %d\n", keys.enckeylen);
3652 goto out;
3653 }
3654 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3655 aeadctx->enckey_len = keys.enckeylen;
3656 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3657 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3658 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3659 aeadctx->enckey_len << 3);
3660 }
3661 key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16);
3662
3663 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3664 0, key_ctx_len >> 4);
3665 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
3666 memzero_explicit(&keys, sizeof(keys));
3667 return 0;
3668 out:
3669 aeadctx->enckey_len = 0;
3670 memzero_explicit(&keys, sizeof(keys));
3671 return -EINVAL;
3672 }
3673
3674 static int chcr_aead_op(struct aead_request *req,
3675 int size,
3676 create_wr_t create_wr_fn)
3677 {
3678 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3679 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3680 struct chcr_context *ctx = a_ctx(tfm);
3681 struct uld_ctx *u_ctx = ULD_CTX(ctx);
3682 struct sk_buff *skb;
3683 struct chcr_dev *cdev;
3684
3685 cdev = a_ctx(tfm)->dev;
3686 if (!cdev) {
3687 pr_err("chcr : %s : No crypto device.\n", __func__);
3688 return -ENXIO;
3689 }
3690
3691 if (chcr_inc_wrcount(cdev)) {
3692 /* Detach state for CHCR means lldi or padap is freed.
3693 * We cannot increment fallback here.
3694 */
3695 return chcr_aead_fallback(req, reqctx->op);
3696 }
3697
3698 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
3699 reqctx->txqidx) &&
3700 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) {
3701 chcr_dec_wrcount(cdev);
3702 return -ENOSPC;
3703 }
3704
3705 /* Form a WR from req */
3706 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], size);
3707
3708 if (IS_ERR_OR_NULL(skb)) {
3709 chcr_dec_wrcount(cdev);
3710 return PTR_ERR_OR_ZERO(skb);
3711 }
3712
3713 skb->dev = u_ctx->lldi.ports[0];
3714 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
3715 chcr_send_wr(skb);
3716 return -EINPROGRESS;
3717 }
3718
3719 static int chcr_aead_encrypt(struct aead_request *req)
3720 {
3721 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3722 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3723 struct chcr_context *ctx = a_ctx(tfm);
3724 unsigned int cpu;
3725
3726 cpu = get_cpu();
3727 reqctx->txqidx = cpu % ctx->ntxq;
3728 reqctx->rxqidx = cpu % ctx->nrxq;
3729 put_cpu();
3730
3731 reqctx->verify = VERIFY_HW;
3732 reqctx->op = CHCR_ENCRYPT_OP;
3733
3734 switch (get_aead_subtype(tfm)) {
3735 case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3736 case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3737 case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3738 case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
3739 return chcr_aead_op(req, 0, create_authenc_wr);
3740 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3741 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3742 return chcr_aead_op(req, 0, create_aead_ccm_wr);
3743 default:
3744 return chcr_aead_op(req, 0, create_gcm_wr);
3745 }
3746 }
3747
3748 static int chcr_aead_decrypt(struct aead_request *req)
3749 {
3750 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3751 struct chcr_context *ctx = a_ctx(tfm);
3752 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3753 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3754 int size;
3755 unsigned int cpu;
3756
3757 cpu = get_cpu();
3758 reqctx->txqidx = cpu % ctx->ntxq;
3759 reqctx->rxqidx = cpu % ctx->nrxq;
3760 put_cpu();
3761
3762 if (aeadctx->mayverify == VERIFY_SW) {
3763 size = crypto_aead_maxauthsize(tfm);
3764 reqctx->verify = VERIFY_SW;
3765 } else {
3766 size = 0;
3767 reqctx->verify = VERIFY_HW;
3768 }
3769 reqctx->op = CHCR_DECRYPT_OP;
3770 switch (get_aead_subtype(tfm)) {
3771 case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3772 case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3773 case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3774 case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
3775 return chcr_aead_op(req, size, create_authenc_wr);
3776 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3777 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3778 return chcr_aead_op(req, size, create_aead_ccm_wr);
3779 default:
3780 return chcr_aead_op(req, size, create_gcm_wr);
3781 }
3782 }
3783
3784 static struct chcr_alg_template driver_algs[] = {
3785 /* AES-CBC */
3786 {
3787 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
3788 .is_registered = 0,
3789 .alg.skcipher = {
3790 .base.cra_name = "cbc(aes)",
3791 .base.cra_driver_name = "cbc-aes-chcr",
3792 .base.cra_blocksize = AES_BLOCK_SIZE,
3793
3794 .init = chcr_init_tfm,
3795 .exit = chcr_exit_tfm,
3796 .min_keysize = AES_MIN_KEY_SIZE,
3797 .max_keysize = AES_MAX_KEY_SIZE,
3798 .ivsize = AES_BLOCK_SIZE,
3799 .setkey = chcr_aes_cbc_setkey,
3800 .encrypt = chcr_aes_encrypt,
3801 .decrypt = chcr_aes_decrypt,
3802 }
3803 },
3804 {
3805 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
3806 .is_registered = 0,
3807 .alg.skcipher = {
3808 .base.cra_name = "xts(aes)",
3809 .base.cra_driver_name = "xts-aes-chcr",
3810 .base.cra_blocksize = AES_BLOCK_SIZE,
3811
3812 .init = chcr_init_tfm,
3813 .exit = chcr_exit_tfm,
3814 .min_keysize = 2 * AES_MIN_KEY_SIZE,
3815 .max_keysize = 2 * AES_MAX_KEY_SIZE,
3816 .ivsize = AES_BLOCK_SIZE,
3817 .setkey = chcr_aes_xts_setkey,
3818 .encrypt = chcr_aes_encrypt,
3819 .decrypt = chcr_aes_decrypt,
3820 }
3821 },
3822 {
3823 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
3824 .is_registered = 0,
3825 .alg.skcipher = {
3826 .base.cra_name = "ctr(aes)",
3827 .base.cra_driver_name = "ctr-aes-chcr",
3828 .base.cra_blocksize = 1,
3829
3830 .init = chcr_init_tfm,
3831 .exit = chcr_exit_tfm,
3832 .min_keysize = AES_MIN_KEY_SIZE,
3833 .max_keysize = AES_MAX_KEY_SIZE,
3834 .ivsize = AES_BLOCK_SIZE,
3835 .setkey = chcr_aes_ctr_setkey,
3836 .encrypt = chcr_aes_encrypt,
3837 .decrypt = chcr_aes_decrypt,
3838 }
3839 },
3840 {
3841 .type = CRYPTO_ALG_TYPE_SKCIPHER |
3842 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3843 .is_registered = 0,
3844 .alg.skcipher = {
3845 .base.cra_name = "rfc3686(ctr(aes))",
3846 .base.cra_driver_name = "rfc3686-ctr-aes-chcr",
3847 .base.cra_blocksize = 1,
3848
3849 .init = chcr_rfc3686_init,
3850 .exit = chcr_exit_tfm,
3851 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3852 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3853 .ivsize = CTR_RFC3686_IV_SIZE,
3854 .setkey = chcr_aes_rfc3686_setkey,
3855 .encrypt = chcr_aes_encrypt,
3856 .decrypt = chcr_aes_decrypt,
3857 }
3858 },
3859 /* SHA */
3860 {
3861 .type = CRYPTO_ALG_TYPE_AHASH,
3862 .is_registered = 0,
3863 .alg.hash = {
3864 .halg.digestsize = SHA1_DIGEST_SIZE,
3865 .halg.base = {
3866 .cra_name = "sha1",
3867 .cra_driver_name = "sha1-chcr",
3868 .cra_blocksize = SHA1_BLOCK_SIZE,
3869 }
3870 }
3871 },
3872 {
3873 .type = CRYPTO_ALG_TYPE_AHASH,
3874 .is_registered = 0,
3875 .alg.hash = {
3876 .halg.digestsize = SHA256_DIGEST_SIZE,
3877 .halg.base = {
3878 .cra_name = "sha256",
3879 .cra_driver_name = "sha256-chcr",
3880 .cra_blocksize = SHA256_BLOCK_SIZE,
3881 }
3882 }
3883 },
3884 {
3885 .type = CRYPTO_ALG_TYPE_AHASH,
3886 .is_registered = 0,
3887 .alg.hash = {
3888 .halg.digestsize = SHA224_DIGEST_SIZE,
3889 .halg.base = {
3890 .cra_name = "sha224",
3891 .cra_driver_name = "sha224-chcr",
3892 .cra_blocksize = SHA224_BLOCK_SIZE,
3893 }
3894 }
3895 },
3896 {
3897 .type = CRYPTO_ALG_TYPE_AHASH,
3898 .is_registered = 0,
3899 .alg.hash = {
3900 .halg.digestsize = SHA384_DIGEST_SIZE,
3901 .halg.base = {
3902 .cra_name = "sha384",
3903 .cra_driver_name = "sha384-chcr",
3904 .cra_blocksize = SHA384_BLOCK_SIZE,
3905 }
3906 }
3907 },
3908 {
3909 .type = CRYPTO_ALG_TYPE_AHASH,
3910 .is_registered = 0,
3911 .alg.hash = {
3912 .halg.digestsize = SHA512_DIGEST_SIZE,
3913 .halg.base = {
3914 .cra_name = "sha512",
3915 .cra_driver_name = "sha512-chcr",
3916 .cra_blocksize = SHA512_BLOCK_SIZE,
3917 }
3918 }
3919 },
3920 /* HMAC */
3921 {
3922 .type = CRYPTO_ALG_TYPE_HMAC,
3923 .is_registered = 0,
3924 .alg.hash = {
3925 .halg.digestsize = SHA1_DIGEST_SIZE,
3926 .halg.base = {
3927 .cra_name = "hmac(sha1)",
3928 .cra_driver_name = "hmac-sha1-chcr",
3929 .cra_blocksize = SHA1_BLOCK_SIZE,
3930 }
3931 }
3932 },
3933 {
3934 .type = CRYPTO_ALG_TYPE_HMAC,
3935 .is_registered = 0,
3936 .alg.hash = {
3937 .halg.digestsize = SHA224_DIGEST_SIZE,
3938 .halg.base = {
3939 .cra_name = "hmac(sha224)",
3940 .cra_driver_name = "hmac-sha224-chcr",
3941 .cra_blocksize = SHA224_BLOCK_SIZE,
3942 }
3943 }
3944 },
3945 {
3946 .type = CRYPTO_ALG_TYPE_HMAC,
3947 .is_registered = 0,
3948 .alg.hash = {
3949 .halg.digestsize = SHA256_DIGEST_SIZE,
3950 .halg.base = {
3951 .cra_name = "hmac(sha256)",
3952 .cra_driver_name = "hmac-sha256-chcr",
3953 .cra_blocksize = SHA256_BLOCK_SIZE,
3954 }
3955 }
3956 },
3957 {
3958 .type = CRYPTO_ALG_TYPE_HMAC,
3959 .is_registered = 0,
3960 .alg.hash = {
3961 .halg.digestsize = SHA384_DIGEST_SIZE,
3962 .halg.base = {
3963 .cra_name = "hmac(sha384)",
3964 .cra_driver_name = "hmac-sha384-chcr",
3965 .cra_blocksize = SHA384_BLOCK_SIZE,
3966 }
3967 }
3968 },
3969 {
3970 .type = CRYPTO_ALG_TYPE_HMAC,
3971 .is_registered = 0,
3972 .alg.hash = {
3973 .halg.digestsize = SHA512_DIGEST_SIZE,
3974 .halg.base = {
3975 .cra_name = "hmac(sha512)",
3976 .cra_driver_name = "hmac-sha512-chcr",
3977 .cra_blocksize = SHA512_BLOCK_SIZE,
3978 }
3979 }
3980 },
3981 /* Add AEAD Algorithms */
3982 {
3983 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
3984 .is_registered = 0,
3985 .alg.aead = {
3986 .base = {
3987 .cra_name = "gcm(aes)",
3988 .cra_driver_name = "gcm-aes-chcr",
3989 .cra_blocksize = 1,
3990 .cra_priority = CHCR_AEAD_PRIORITY,
3991 .cra_ctxsize = sizeof(struct chcr_context) +
3992 sizeof(struct chcr_aead_ctx) +
3993 sizeof(struct chcr_gcm_ctx),
3994 },
3995 .ivsize = GCM_AES_IV_SIZE,
3996 .maxauthsize = GHASH_DIGEST_SIZE,
3997 .setkey = chcr_gcm_setkey,
3998 .setauthsize = chcr_gcm_setauthsize,
3999 }
4000 },
4001 {
4002 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
4003 .is_registered = 0,
4004 .alg.aead = {
4005 .base = {
4006 .cra_name = "rfc4106(gcm(aes))",
4007 .cra_driver_name = "rfc4106-gcm-aes-chcr",
4008 .cra_blocksize = 1,
4009 .cra_priority = CHCR_AEAD_PRIORITY + 1,
4010 .cra_ctxsize = sizeof(struct chcr_context) +
4011 sizeof(struct chcr_aead_ctx) +
4012 sizeof(struct chcr_gcm_ctx),
4013
4014 },
4015 .ivsize = GCM_RFC4106_IV_SIZE,
4016 .maxauthsize = GHASH_DIGEST_SIZE,
4017 .setkey = chcr_gcm_setkey,
4018 .setauthsize = chcr_4106_4309_setauthsize,
4019 }
4020 },
4021 {
4022 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
4023 .is_registered = 0,
4024 .alg.aead = {
4025 .base = {
4026 .cra_name = "ccm(aes)",
4027 .cra_driver_name = "ccm-aes-chcr",
4028 .cra_blocksize = 1,
4029 .cra_priority = CHCR_AEAD_PRIORITY,
4030 .cra_ctxsize = sizeof(struct chcr_context) +
4031 sizeof(struct chcr_aead_ctx),
4032
4033 },
4034 .ivsize = AES_BLOCK_SIZE,
4035 .maxauthsize = GHASH_DIGEST_SIZE,
4036 .setkey = chcr_aead_ccm_setkey,
4037 .setauthsize = chcr_ccm_setauthsize,
4038 }
4039 },
4040 {
4041 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
4042 .is_registered = 0,
4043 .alg.aead = {
4044 .base = {
4045 .cra_name = "rfc4309(ccm(aes))",
4046 .cra_driver_name = "rfc4309-ccm-aes-chcr",
4047 .cra_blocksize = 1,
4048 .cra_priority = CHCR_AEAD_PRIORITY + 1,
4049 .cra_ctxsize = sizeof(struct chcr_context) +
4050 sizeof(struct chcr_aead_ctx),
4051
4052 },
4053 .ivsize = 8,
4054 .maxauthsize = GHASH_DIGEST_SIZE,
4055 .setkey = chcr_aead_rfc4309_setkey,
4056 .setauthsize = chcr_4106_4309_setauthsize,
4057 }
4058 },
4059 {
4060 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4061 .is_registered = 0,
4062 .alg.aead = {
4063 .base = {
4064 .cra_name = "authenc(hmac(sha1),cbc(aes))",
4065 .cra_driver_name =
4066 "authenc-hmac-sha1-cbc-aes-chcr",
4067 .cra_blocksize = AES_BLOCK_SIZE,
4068 .cra_priority = CHCR_AEAD_PRIORITY,
4069 .cra_ctxsize = sizeof(struct chcr_context) +
4070 sizeof(struct chcr_aead_ctx) +
4071 sizeof(struct chcr_authenc_ctx),
4072
4073 },
4074 .ivsize = AES_BLOCK_SIZE,
4075 .maxauthsize = SHA1_DIGEST_SIZE,
4076 .setkey = chcr_authenc_setkey,
4077 .setauthsize = chcr_authenc_setauthsize,
4078 }
4079 },
4080 {
4081 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4082 .is_registered = 0,
4083 .alg.aead = {
4084 .base = {
4085
4086 .cra_name = "authenc(hmac(sha256),cbc(aes))",
4087 .cra_driver_name =
4088 "authenc-hmac-sha256-cbc-aes-chcr",
4089 .cra_blocksize = AES_BLOCK_SIZE,
4090 .cra_priority = CHCR_AEAD_PRIORITY,
4091 .cra_ctxsize = sizeof(struct chcr_context) +
4092 sizeof(struct chcr_aead_ctx) +
4093 sizeof(struct chcr_authenc_ctx),
4094
4095 },
4096 .ivsize = AES_BLOCK_SIZE,
4097 .maxauthsize = SHA256_DIGEST_SIZE,
4098 .setkey = chcr_authenc_setkey,
4099 .setauthsize = chcr_authenc_setauthsize,
4100 }
4101 },
4102 {
4103 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4104 .is_registered = 0,
4105 .alg.aead = {
4106 .base = {
4107 .cra_name = "authenc(hmac(sha224),cbc(aes))",
4108 .cra_driver_name =
4109 "authenc-hmac-sha224-cbc-aes-chcr",
4110 .cra_blocksize = AES_BLOCK_SIZE,
4111 .cra_priority = CHCR_AEAD_PRIORITY,
4112 .cra_ctxsize = sizeof(struct chcr_context) +
4113 sizeof(struct chcr_aead_ctx) +
4114 sizeof(struct chcr_authenc_ctx),
4115 },
4116 .ivsize = AES_BLOCK_SIZE,
4117 .maxauthsize = SHA224_DIGEST_SIZE,
4118 .setkey = chcr_authenc_setkey,
4119 .setauthsize = chcr_authenc_setauthsize,
4120 }
4121 },
4122 {
4123 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4124 .is_registered = 0,
4125 .alg.aead = {
4126 .base = {
4127 .cra_name = "authenc(hmac(sha384),cbc(aes))",
4128 .cra_driver_name =
4129 "authenc-hmac-sha384-cbc-aes-chcr",
4130 .cra_blocksize = AES_BLOCK_SIZE,
4131 .cra_priority = CHCR_AEAD_PRIORITY,
4132 .cra_ctxsize = sizeof(struct chcr_context) +
4133 sizeof(struct chcr_aead_ctx) +
4134 sizeof(struct chcr_authenc_ctx),
4135
4136 },
4137 .ivsize = AES_BLOCK_SIZE,
4138 .maxauthsize = SHA384_DIGEST_SIZE,
4139 .setkey = chcr_authenc_setkey,
4140 .setauthsize = chcr_authenc_setauthsize,
4141 }
4142 },
4143 {
4144 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
4145 .is_registered = 0,
4146 .alg.aead = {
4147 .base = {
4148 .cra_name = "authenc(hmac(sha512),cbc(aes))",
4149 .cra_driver_name =
4150 "authenc-hmac-sha512-cbc-aes-chcr",
4151 .cra_blocksize = AES_BLOCK_SIZE,
4152 .cra_priority = CHCR_AEAD_PRIORITY,
4153 .cra_ctxsize = sizeof(struct chcr_context) +
4154 sizeof(struct chcr_aead_ctx) +
4155 sizeof(struct chcr_authenc_ctx),
4156
4157 },
4158 .ivsize = AES_BLOCK_SIZE,
4159 .maxauthsize = SHA512_DIGEST_SIZE,
4160 .setkey = chcr_authenc_setkey,
4161 .setauthsize = chcr_authenc_setauthsize,
4162 }
4163 },
4164 {
4165 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL,
4166 .is_registered = 0,
4167 .alg.aead = {
4168 .base = {
4169 .cra_name = "authenc(digest_null,cbc(aes))",
4170 .cra_driver_name =
4171 "authenc-digest_null-cbc-aes-chcr",
4172 .cra_blocksize = AES_BLOCK_SIZE,
4173 .cra_priority = CHCR_AEAD_PRIORITY,
4174 .cra_ctxsize = sizeof(struct chcr_context) +
4175 sizeof(struct chcr_aead_ctx) +
4176 sizeof(struct chcr_authenc_ctx),
4177
4178 },
4179 .ivsize = AES_BLOCK_SIZE,
4180 .maxauthsize = 0,
4181 .setkey = chcr_aead_digest_null_setkey,
4182 .setauthsize = chcr_authenc_null_setauthsize,
4183 }
4184 },
4185 {
4186 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4187 .is_registered = 0,
4188 .alg.aead = {
4189 .base = {
4190 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4191 .cra_driver_name =
4192 "authenc-hmac-sha1-rfc3686-ctr-aes-chcr",
4193 .cra_blocksize = 1,
4194 .cra_priority = CHCR_AEAD_PRIORITY,
4195 .cra_ctxsize = sizeof(struct chcr_context) +
4196 sizeof(struct chcr_aead_ctx) +
4197 sizeof(struct chcr_authenc_ctx),
4198
4199 },
4200 .ivsize = CTR_RFC3686_IV_SIZE,
4201 .maxauthsize = SHA1_DIGEST_SIZE,
4202 .setkey = chcr_authenc_setkey,
4203 .setauthsize = chcr_authenc_setauthsize,
4204 }
4205 },
4206 {
4207 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4208 .is_registered = 0,
4209 .alg.aead = {
4210 .base = {
4211
4212 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4213 .cra_driver_name =
4214 "authenc-hmac-sha256-rfc3686-ctr-aes-chcr",
4215 .cra_blocksize = 1,
4216 .cra_priority = CHCR_AEAD_PRIORITY,
4217 .cra_ctxsize = sizeof(struct chcr_context) +
4218 sizeof(struct chcr_aead_ctx) +
4219 sizeof(struct chcr_authenc_ctx),
4220
4221 },
4222 .ivsize = CTR_RFC3686_IV_SIZE,
4223 .maxauthsize = SHA256_DIGEST_SIZE,
4224 .setkey = chcr_authenc_setkey,
4225 .setauthsize = chcr_authenc_setauthsize,
4226 }
4227 },
4228 {
4229 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4230 .is_registered = 0,
4231 .alg.aead = {
4232 .base = {
4233 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
4234 .cra_driver_name =
4235 "authenc-hmac-sha224-rfc3686-ctr-aes-chcr",
4236 .cra_blocksize = 1,
4237 .cra_priority = CHCR_AEAD_PRIORITY,
4238 .cra_ctxsize = sizeof(struct chcr_context) +
4239 sizeof(struct chcr_aead_ctx) +
4240 sizeof(struct chcr_authenc_ctx),
4241 },
4242 .ivsize = CTR_RFC3686_IV_SIZE,
4243 .maxauthsize = SHA224_DIGEST_SIZE,
4244 .setkey = chcr_authenc_setkey,
4245 .setauthsize = chcr_authenc_setauthsize,
4246 }
4247 },
4248 {
4249 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4250 .is_registered = 0,
4251 .alg.aead = {
4252 .base = {
4253 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4254 .cra_driver_name =
4255 "authenc-hmac-sha384-rfc3686-ctr-aes-chcr",
4256 .cra_blocksize = 1,
4257 .cra_priority = CHCR_AEAD_PRIORITY,
4258 .cra_ctxsize = sizeof(struct chcr_context) +
4259 sizeof(struct chcr_aead_ctx) +
4260 sizeof(struct chcr_authenc_ctx),
4261
4262 },
4263 .ivsize = CTR_RFC3686_IV_SIZE,
4264 .maxauthsize = SHA384_DIGEST_SIZE,
4265 .setkey = chcr_authenc_setkey,
4266 .setauthsize = chcr_authenc_setauthsize,
4267 }
4268 },
4269 {
4270 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4271 .is_registered = 0,
4272 .alg.aead = {
4273 .base = {
4274 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4275 .cra_driver_name =
4276 "authenc-hmac-sha512-rfc3686-ctr-aes-chcr",
4277 .cra_blocksize = 1,
4278 .cra_priority = CHCR_AEAD_PRIORITY,
4279 .cra_ctxsize = sizeof(struct chcr_context) +
4280 sizeof(struct chcr_aead_ctx) +
4281 sizeof(struct chcr_authenc_ctx),
4282
4283 },
4284 .ivsize = CTR_RFC3686_IV_SIZE,
4285 .maxauthsize = SHA512_DIGEST_SIZE,
4286 .setkey = chcr_authenc_setkey,
4287 .setauthsize = chcr_authenc_setauthsize,
4288 }
4289 },
4290 {
4291 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL,
4292 .is_registered = 0,
4293 .alg.aead = {
4294 .base = {
4295 .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))",
4296 .cra_driver_name =
4297 "authenc-digest_null-rfc3686-ctr-aes-chcr",
4298 .cra_blocksize = 1,
4299 .cra_priority = CHCR_AEAD_PRIORITY,
4300 .cra_ctxsize = sizeof(struct chcr_context) +
4301 sizeof(struct chcr_aead_ctx) +
4302 sizeof(struct chcr_authenc_ctx),
4303
4304 },
4305 .ivsize = CTR_RFC3686_IV_SIZE,
4306 .maxauthsize = 0,
4307 .setkey = chcr_aead_digest_null_setkey,
4308 .setauthsize = chcr_authenc_null_setauthsize,
4309 }
4310 },
4311 };
4312
4313 /*
4314 * chcr_unregister_alg - Deregister crypto algorithms with
4315 * kernel framework.
4316 */
4317 static int chcr_unregister_alg(void)
4318 {
4319 int i;
4320
4321 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4322 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4323 case CRYPTO_ALG_TYPE_SKCIPHER:
4324 if (driver_algs[i].is_registered)
4325 crypto_unregister_skcipher(
4326 &driver_algs[i].alg.skcipher);
4327 break;
4328 case CRYPTO_ALG_TYPE_AEAD:
4329 if (driver_algs[i].is_registered)
4330 crypto_unregister_aead(
4331 &driver_algs[i].alg.aead);
4332 break;
4333 case CRYPTO_ALG_TYPE_AHASH:
4334 if (driver_algs[i].is_registered)
4335 crypto_unregister_ahash(
4336 &driver_algs[i].alg.hash);
4337 break;
4338 }
4339 driver_algs[i].is_registered = 0;
4340 }
4341 return 0;
4342 }
4343
4344 #define SZ_AHASH_CTX sizeof(struct chcr_context)
4345 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
4346 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
4347
4348 /*
4349 * chcr_register_alg - Register crypto algorithms with kernel framework.
4350 */
4351 static int chcr_register_alg(void)
4352 {
4353 struct crypto_alg ai;
4354 struct ahash_alg *a_hash;
4355 int err = 0, i;
4356 char *name = NULL;
4357
4358 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4359 if (driver_algs[i].is_registered)
4360 continue;
4361 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
4362 case CRYPTO_ALG_TYPE_SKCIPHER:
4363 driver_algs[i].alg.skcipher.base.cra_priority =
4364 CHCR_CRA_PRIORITY;
4365 driver_algs[i].alg.skcipher.base.cra_module = THIS_MODULE;
4366 driver_algs[i].alg.skcipher.base.cra_flags =
4367 CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
4368 CRYPTO_ALG_NEED_FALLBACK;
4369 driver_algs[i].alg.skcipher.base.cra_ctxsize =
4370 sizeof(struct chcr_context) +
4371 sizeof(struct ablk_ctx);
4372 driver_algs[i].alg.skcipher.base.cra_alignmask = 0;
4373
4374 err = crypto_register_skcipher(&driver_algs[i].alg.skcipher);
4375 name = driver_algs[i].alg.skcipher.base.cra_driver_name;
4376 break;
4377 case CRYPTO_ALG_TYPE_AEAD:
4378 driver_algs[i].alg.aead.base.cra_flags =
4379 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
4380 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
4381 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
4382 driver_algs[i].alg.aead.init = chcr_aead_cra_init;
4383 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
4384 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
4385 err = crypto_register_aead(&driver_algs[i].alg.aead);
4386 name = driver_algs[i].alg.aead.base.cra_driver_name;
4387 break;
4388 case CRYPTO_ALG_TYPE_AHASH:
4389 a_hash = &driver_algs[i].alg.hash;
4390 a_hash->update = chcr_ahash_update;
4391 a_hash->final = chcr_ahash_final;
4392 a_hash->finup = chcr_ahash_finup;
4393 a_hash->digest = chcr_ahash_digest;
4394 a_hash->export = chcr_ahash_export;
4395 a_hash->import = chcr_ahash_import;
4396 a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
4397 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
4398 a_hash->halg.base.cra_module = THIS_MODULE;
4399 a_hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC;
4400 a_hash->halg.base.cra_alignmask = 0;
4401 a_hash->halg.base.cra_exit = NULL;
4402
4403 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
4404 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
4405 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
4406 a_hash->init = chcr_hmac_init;
4407 a_hash->setkey = chcr_ahash_setkey;
4408 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
4409 } else {
4410 a_hash->init = chcr_sha_init;
4411 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
4412 a_hash->halg.base.cra_init = chcr_sha_cra_init;
4413 }
4414 err = crypto_register_ahash(&driver_algs[i].alg.hash);
4415 ai = driver_algs[i].alg.hash.halg.base;
4416 name = ai.cra_driver_name;
4417 break;
4418 }
4419 if (err) {
4420 pr_err("chcr : %s : Algorithm registration failed\n",
4421 name);
4422 goto register_err;
4423 } else {
4424 driver_algs[i].is_registered = 1;
4425 }
4426 }
4427 return 0;
4428
4429 register_err:
4430 chcr_unregister_alg();
4431 return err;
4432 }
4433
4434 /*
4435 * start_crypto - Register the crypto algorithms.
4436 * This should called once when the first device comesup. After this
4437 * kernel will start calling driver APIs for crypto operations.
4438 */
4439 int start_crypto(void)
4440 {
4441 return chcr_register_alg();
4442 }
4443
4444 /*
4445 * stop_crypto - Deregister all the crypto algorithms with kernel.
4446 * This should be called once when the last device goes down. After this
4447 * kernel will not call the driver API for crypto operations.
4448 */
4449 int stop_crypto(void)
4450 {
4451 chcr_unregister_alg();
4452 return 0;
4453 }
Ubuntu Hirsute Linux kernel mirror