]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blob - drivers/crypto/chelsio/chcr_algo.c
projects / mirror_ubuntu-artful-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
scsi: qedf: Fix a potential NULL pointer dereference
[mirror_ubuntu-artful-kernel.git] / drivers / crypto / chelsio / chcr_algo.c
1 /*
2 * This file is part of the Chelsio T6 Crypto driver for Linux.
3 *
4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written and Maintained by:
35 * Manoj Malviya (manojmalviya@chelsio.com)
36 * Atul Gupta (atul.gupta@chelsio.com)
37 * Jitendra Lulla (jlulla@chelsio.com)
38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
39 * Harsh Jain (harsh@chelsio.com)
40 */
41
42 #define pr_fmt(fmt) "chcr:" fmt
43
44 #include <linux/kernel.h>
45 #include <linux/module.h>
46 #include <linux/crypto.h>
47 #include <linux/cryptohash.h>
48 #include <linux/skbuff.h>
49 #include <linux/rtnetlink.h>
50 #include <linux/highmem.h>
51 #include <linux/scatterlist.h>
52
53 #include <crypto/aes.h>
54 #include <crypto/algapi.h>
55 #include <crypto/hash.h>
56 #include <crypto/sha.h>
57 #include <crypto/authenc.h>
58 #include <crypto/ctr.h>
59 #include <crypto/gf128mul.h>
60 #include <crypto/internal/aead.h>
61 #include <crypto/null.h>
62 #include <crypto/internal/skcipher.h>
63 #include <crypto/aead.h>
64 #include <crypto/scatterwalk.h>
65 #include <crypto/internal/hash.h>
66
67 #include "t4fw_api.h"
68 #include "t4_msg.h"
69 #include "chcr_core.h"
70 #include "chcr_algo.h"
71 #include "chcr_crypto.h"
72
73 static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
74 {
75 return ctx->crypto_ctx->aeadctx;
76 }
77
78 static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
79 {
80 return ctx->crypto_ctx->ablkctx;
81 }
82
83 static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
84 {
85 return ctx->crypto_ctx->hmacctx;
86 }
87
88 static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
89 {
90 return gctx->ctx->gcm;
91 }
92
93 static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
94 {
95 return gctx->ctx->authenc;
96 }
97
98 static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
99 {
100 return ctx->dev->u_ctx;
101 }
102
103 static inline int is_ofld_imm(const struct sk_buff *skb)
104 {
105 return (skb->len <= CRYPTO_MAX_IMM_TX_PKT_LEN);
106 }
107
108 /*
109 * sgl_len - calculates the size of an SGL of the given capacity
110 * @n: the number of SGL entries
111 * Calculates the number of flits needed for a scatter/gather list that
112 * can hold the given number of entries.
113 */
114 static inline unsigned int sgl_len(unsigned int n)
115 {
116 n--;
117 return (3 * n) / 2 + (n & 1) + 2;
118 }
119
120 static void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
121 {
122 u8 temp[SHA512_DIGEST_SIZE];
123 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
124 int authsize = crypto_aead_authsize(tfm);
125 struct cpl_fw6_pld *fw6_pld;
126 int cmp = 0;
127
128 fw6_pld = (struct cpl_fw6_pld *)input;
129 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
130 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
131 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
132 } else {
133
134 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
135 authsize, req->assoclen +
136 req->cryptlen - authsize);
137 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
138 }
139 if (cmp)
140 *err = -EBADMSG;
141 else
142 *err = 0;
143 }
144
145 /*
146 * chcr_handle_resp - Unmap the DMA buffers associated with the request
147 * @req: crypto request
148 */
149 int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
150 int err)
151 {
152 struct crypto_tfm *tfm = req->tfm;
153 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
154 struct uld_ctx *u_ctx = ULD_CTX(ctx);
155 struct chcr_req_ctx ctx_req;
156 unsigned int digestsize, updated_digestsize;
157 struct adapter *adap = padap(ctx->dev);
158
159 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
160 case CRYPTO_ALG_TYPE_AEAD:
161 ctx_req.req.aead_req = aead_request_cast(req);
162 ctx_req.ctx.reqctx = aead_request_ctx(ctx_req.req.aead_req);
163 dma_unmap_sg(&u_ctx->lldi.pdev->dev, ctx_req.ctx.reqctx->dst,
164 ctx_req.ctx.reqctx->dst_nents, DMA_FROM_DEVICE);
165 if (ctx_req.ctx.reqctx->skb) {
166 kfree_skb(ctx_req.ctx.reqctx->skb);
167 ctx_req.ctx.reqctx->skb = NULL;
168 }
169 free_new_sg(ctx_req.ctx.reqctx->newdstsg);
170 ctx_req.ctx.reqctx->newdstsg = NULL;
171 if (ctx_req.ctx.reqctx->verify == VERIFY_SW) {
172 chcr_verify_tag(ctx_req.req.aead_req, input,
173 &err);
174 ctx_req.ctx.reqctx->verify = VERIFY_HW;
175 }
176 ctx_req.req.aead_req->base.complete(req, err);
177 break;
178
179 case CRYPTO_ALG_TYPE_ABLKCIPHER:
180 err = chcr_handle_cipher_resp(ablkcipher_request_cast(req),
181 input, err);
182 break;
183
184 case CRYPTO_ALG_TYPE_AHASH:
185 ctx_req.req.ahash_req = ahash_request_cast(req);
186 ctx_req.ctx.ahash_ctx =
187 ahash_request_ctx(ctx_req.req.ahash_req);
188 digestsize =
189 crypto_ahash_digestsize(crypto_ahash_reqtfm(
190 ctx_req.req.ahash_req));
191 updated_digestsize = digestsize;
192 if (digestsize == SHA224_DIGEST_SIZE)
193 updated_digestsize = SHA256_DIGEST_SIZE;
194 else if (digestsize == SHA384_DIGEST_SIZE)
195 updated_digestsize = SHA512_DIGEST_SIZE;
196 if (ctx_req.ctx.ahash_ctx->skb) {
197 kfree_skb(ctx_req.ctx.ahash_ctx->skb);
198 ctx_req.ctx.ahash_ctx->skb = NULL;
199 }
200 if (ctx_req.ctx.ahash_ctx->result == 1) {
201 ctx_req.ctx.ahash_ctx->result = 0;
202 memcpy(ctx_req.req.ahash_req->result, input +
203 sizeof(struct cpl_fw6_pld),
204 digestsize);
205 } else {
206 memcpy(ctx_req.ctx.ahash_ctx->partial_hash, input +
207 sizeof(struct cpl_fw6_pld),
208 updated_digestsize);
209 }
210 ctx_req.req.ahash_req->base.complete(req, err);
211 break;
212 }
213 atomic_inc(&adap->chcr_stats.complete);
214 return err;
215 }
216
217 /*
218 * calc_tx_flits_ofld - calculate # of flits for an offload packet
219 * @skb: the packet
220 * Returns the number of flits needed for the given offload packet.
221 * These packets are already fully constructed and no additional headers
222 * will be added.
223 */
224 static inline unsigned int calc_tx_flits_ofld(const struct sk_buff *skb)
225 {
226 unsigned int flits, cnt;
227
228 if (is_ofld_imm(skb))
229 return DIV_ROUND_UP(skb->len, 8);
230
231 flits = skb_transport_offset(skb) / 8; /* headers */
232 cnt = skb_shinfo(skb)->nr_frags;
233 if (skb_tail_pointer(skb) != skb_transport_header(skb))
234 cnt++;
235 return flits + sgl_len(cnt);
236 }
237
238 static inline void get_aes_decrypt_key(unsigned char *dec_key,
239 const unsigned char *key,
240 unsigned int keylength)
241 {
242 u32 temp;
243 u32 w_ring[MAX_NK];
244 int i, j, k;
245 u8 nr, nk;
246
247 switch (keylength) {
248 case AES_KEYLENGTH_128BIT:
249 nk = KEYLENGTH_4BYTES;
250 nr = NUMBER_OF_ROUNDS_10;
251 break;
252 case AES_KEYLENGTH_192BIT:
253 nk = KEYLENGTH_6BYTES;
254 nr = NUMBER_OF_ROUNDS_12;
255 break;
256 case AES_KEYLENGTH_256BIT:
257 nk = KEYLENGTH_8BYTES;
258 nr = NUMBER_OF_ROUNDS_14;
259 break;
260 default:
261 return;
262 }
263 for (i = 0; i < nk; i++)
264 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
265
266 i = 0;
267 temp = w_ring[nk - 1];
268 while (i + nk < (nr + 1) * 4) {
269 if (!(i % nk)) {
270 /* RotWord(temp) */
271 temp = (temp << 8) | (temp >> 24);
272 temp = aes_ks_subword(temp);
273 temp ^= round_constant[i / nk];
274 } else if (nk == 8 && (i % 4 == 0)) {
275 temp = aes_ks_subword(temp);
276 }
277 w_ring[i % nk] ^= temp;
278 temp = w_ring[i % nk];
279 i++;
280 }
281 i--;
282 for (k = 0, j = i % nk; k < nk; k++) {
283 *((u32 *)dec_key + k) = htonl(w_ring[j]);
284 j--;
285 if (j < 0)
286 j += nk;
287 }
288 }
289
290 static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
291 {
292 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
293
294 switch (ds) {
295 case SHA1_DIGEST_SIZE:
296 base_hash = crypto_alloc_shash("sha1", 0, 0);
297 break;
298 case SHA224_DIGEST_SIZE:
299 base_hash = crypto_alloc_shash("sha224", 0, 0);
300 break;
301 case SHA256_DIGEST_SIZE:
302 base_hash = crypto_alloc_shash("sha256", 0, 0);
303 break;
304 case SHA384_DIGEST_SIZE:
305 base_hash = crypto_alloc_shash("sha384", 0, 0);
306 break;
307 case SHA512_DIGEST_SIZE:
308 base_hash = crypto_alloc_shash("sha512", 0, 0);
309 break;
310 }
311
312 return base_hash;
313 }
314
315 static int chcr_compute_partial_hash(struct shash_desc *desc,
316 char *iopad, char *result_hash,
317 int digest_size)
318 {
319 struct sha1_state sha1_st;
320 struct sha256_state sha256_st;
321 struct sha512_state sha512_st;
322 int error;
323
324 if (digest_size == SHA1_DIGEST_SIZE) {
325 error = crypto_shash_init(desc) ?:
326 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
327 crypto_shash_export(desc, (void *)&sha1_st);
328 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
329 } else if (digest_size == SHA224_DIGEST_SIZE) {
330 error = crypto_shash_init(desc) ?:
331 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
332 crypto_shash_export(desc, (void *)&sha256_st);
333 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
334
335 } else if (digest_size == SHA256_DIGEST_SIZE) {
336 error = crypto_shash_init(desc) ?:
337 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
338 crypto_shash_export(desc, (void *)&sha256_st);
339 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
340
341 } else if (digest_size == SHA384_DIGEST_SIZE) {
342 error = crypto_shash_init(desc) ?:
343 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
344 crypto_shash_export(desc, (void *)&sha512_st);
345 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
346
347 } else if (digest_size == SHA512_DIGEST_SIZE) {
348 error = crypto_shash_init(desc) ?:
349 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
350 crypto_shash_export(desc, (void *)&sha512_st);
351 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
352 } else {
353 error = -EINVAL;
354 pr_err("Unknown digest size %d\n", digest_size);
355 }
356 return error;
357 }
358
359 static void chcr_change_order(char *buf, int ds)
360 {
361 int i;
362
363 if (ds == SHA512_DIGEST_SIZE) {
364 for (i = 0; i < (ds / sizeof(u64)); i++)
365 *((__be64 *)buf + i) =
366 cpu_to_be64(*((u64 *)buf + i));
367 } else {
368 for (i = 0; i < (ds / sizeof(u32)); i++)
369 *((__be32 *)buf + i) =
370 cpu_to_be32(*((u32 *)buf + i));
371 }
372 }
373
374 static inline int is_hmac(struct crypto_tfm *tfm)
375 {
376 struct crypto_alg *alg = tfm->__crt_alg;
377 struct chcr_alg_template *chcr_crypto_alg =
378 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
379 alg.hash);
380 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
381 return 1;
382 return 0;
383 }
384
385 static void write_phys_cpl(struct cpl_rx_phys_dsgl *phys_cpl,
386 struct scatterlist *sg,
387 struct phys_sge_parm *sg_param)
388 {
389 struct phys_sge_pairs *to;
390 unsigned int len = 0, left_size = sg_param->obsize;
391 unsigned int nents = sg_param->nents, i, j = 0;
392
393 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
394 | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
395 phys_cpl->pcirlxorder_to_noofsgentr =
396 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
397 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
398 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
399 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
400 CPL_RX_PHYS_DSGL_DCAID_V(0) |
401 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(nents));
402 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
403 phys_cpl->rss_hdr_int.qid = htons(sg_param->qid);
404 phys_cpl->rss_hdr_int.hash_val = 0;
405 to = (struct phys_sge_pairs *)((unsigned char *)phys_cpl +
406 sizeof(struct cpl_rx_phys_dsgl));
407 for (i = 0; nents && left_size; to++) {
408 for (j = 0; j < 8 && nents && left_size; j++, nents--) {
409 len = min(left_size, sg_dma_len(sg));
410 to->len[j] = htons(len);
411 to->addr[j] = cpu_to_be64(sg_dma_address(sg));
412 left_size -= len;
413 sg = sg_next(sg);
414 }
415 }
416 }
417
418 static inline int map_writesg_phys_cpl(struct device *dev,
419 struct cpl_rx_phys_dsgl *phys_cpl,
420 struct scatterlist *sg,
421 struct phys_sge_parm *sg_param)
422 {
423 if (!sg || !sg_param->nents)
424 return -EINVAL;
425
426 sg_param->nents = dma_map_sg(dev, sg, sg_param->nents, DMA_FROM_DEVICE);
427 if (sg_param->nents == 0) {
428 pr_err("CHCR : DMA mapping failed\n");
429 return -EINVAL;
430 }
431 write_phys_cpl(phys_cpl, sg, sg_param);
432 return 0;
433 }
434
435 static inline int get_aead_subtype(struct crypto_aead *aead)
436 {
437 struct aead_alg *alg = crypto_aead_alg(aead);
438 struct chcr_alg_template *chcr_crypto_alg =
439 container_of(alg, struct chcr_alg_template, alg.aead);
440 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
441 }
442
443 static inline int get_cryptoalg_subtype(struct crypto_tfm *tfm)
444 {
445 struct crypto_alg *alg = tfm->__crt_alg;
446 struct chcr_alg_template *chcr_crypto_alg =
447 container_of(alg, struct chcr_alg_template, alg.crypto);
448
449 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
450 }
451
452 static inline void write_buffer_to_skb(struct sk_buff *skb,
453 unsigned int *frags,
454 char *bfr,
455 u8 bfr_len)
456 {
457 skb->len += bfr_len;
458 skb->data_len += bfr_len;
459 skb->truesize += bfr_len;
460 get_page(virt_to_page(bfr));
461 skb_fill_page_desc(skb, *frags, virt_to_page(bfr),
462 offset_in_page(bfr), bfr_len);
463 (*frags)++;
464 }
465
466
467 static inline void
468 write_sg_to_skb(struct sk_buff *skb, unsigned int *frags,
469 struct scatterlist *sg, unsigned int count)
470 {
471 struct page *spage;
472 unsigned int page_len;
473
474 skb->len += count;
475 skb->data_len += count;
476 skb->truesize += count;
477
478 while (count > 0) {
479 if (!sg || (!(sg->length)))
480 break;
481 spage = sg_page(sg);
482 get_page(spage);
483 page_len = min(sg->length, count);
484 skb_fill_page_desc(skb, *frags, spage, sg->offset, page_len);
485 (*frags)++;
486 count -= page_len;
487 sg = sg_next(sg);
488 }
489 }
490
491 static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
492 {
493 struct adapter *adap = netdev2adap(dev);
494 struct sge_uld_txq_info *txq_info =
495 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
496 struct sge_uld_txq *txq;
497 int ret = 0;
498
499 local_bh_disable();
500 txq = &txq_info->uldtxq[idx];
501 spin_lock(&txq->sendq.lock);
502 if (txq->full)
503 ret = -1;
504 spin_unlock(&txq->sendq.lock);
505 local_bh_enable();
506 return ret;
507 }
508
509 static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
510 struct _key_ctx *key_ctx)
511 {
512 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
513 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
514 } else {
515 memcpy(key_ctx->key,
516 ablkctx->key + (ablkctx->enckey_len >> 1),
517 ablkctx->enckey_len >> 1);
518 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
519 ablkctx->rrkey, ablkctx->enckey_len >> 1);
520 }
521 return 0;
522 }
523 static int chcr_sg_ent_in_wr(struct scatterlist *src,
524 struct scatterlist *dst,
525 unsigned int minsg,
526 unsigned int space,
527 short int *sent,
528 short int *dent)
529 {
530 int srclen = 0, dstlen = 0;
531 int srcsg = minsg, dstsg = 0;
532
533 *sent = 0;
534 *dent = 0;
535 while (src && dst && ((srcsg + 1) <= MAX_SKB_FRAGS) &&
536 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
537 srclen += src->length;
538 srcsg++;
539 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
540 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
541 if (srclen <= dstlen)
542 break;
543 dstlen += dst->length;
544 dst = sg_next(dst);
545 dstsg++;
546 }
547 src = sg_next(src);
548 }
549 *sent = srcsg - minsg;
550 *dent = dstsg;
551 return min(srclen, dstlen);
552 }
553
554 static int chcr_cipher_fallback(struct crypto_skcipher *cipher,
555 u32 flags,
556 struct scatterlist *src,
557 struct scatterlist *dst,
558 unsigned int nbytes,
559 u8 *iv,
560 unsigned short op_type)
561 {
562 int err;
563
564 SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
565 skcipher_request_set_tfm(subreq, cipher);
566 skcipher_request_set_callback(subreq, flags, NULL, NULL);
567 skcipher_request_set_crypt(subreq, src, dst,
568 nbytes, iv);
569
570 err = op_type ? crypto_skcipher_decrypt(subreq) :
571 crypto_skcipher_encrypt(subreq);
572 skcipher_request_zero(subreq);
573
574 return err;
575
576 }
577 static inline void create_wreq(struct chcr_context *ctx,
578 struct chcr_wr *chcr_req,
579 void *req, struct sk_buff *skb,
580 int kctx_len, int hash_sz,
581 int is_iv,
582 unsigned int sc_len,
583 unsigned int lcb)
584 {
585 struct uld_ctx *u_ctx = ULD_CTX(ctx);
586 int iv_loc = IV_DSGL;
587 int qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
588 unsigned int immdatalen = 0, nr_frags = 0;
589
590 if (is_ofld_imm(skb)) {
591 immdatalen = skb->data_len;
592 iv_loc = IV_IMMEDIATE;
593 } else {
594 nr_frags = skb_shinfo(skb)->nr_frags;
595 }
596
597 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE(immdatalen,
598 ((sizeof(chcr_req->key_ctx) + kctx_len) >> 4));
599 chcr_req->wreq.pld_size_hash_size =
600 htonl(FW_CRYPTO_LOOKASIDE_WR_PLD_SIZE_V(sgl_lengths[nr_frags]) |
601 FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
602 chcr_req->wreq.len16_pkd =
603 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(
604 (calc_tx_flits_ofld(skb) * 8), 16)));
605 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
606 chcr_req->wreq.rx_chid_to_rx_q_id =
607 FILL_WR_RX_Q_ID(ctx->dev->rx_channel_id, qid,
608 is_iv ? iv_loc : IV_NOP, !!lcb,
609 ctx->tx_qidx);
610
611 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(ctx->dev->tx_channel_id,
612 qid);
613 chcr_req->ulptx.len = htonl((DIV_ROUND_UP((calc_tx_flits_ofld(skb) * 8),
614 16) - ((sizeof(chcr_req->wreq)) >> 4)));
615
616 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(immdatalen);
617 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
618 sizeof(chcr_req->key_ctx) +
619 kctx_len + sc_len + immdatalen);
620 }
621
622 /**
623 * create_cipher_wr - form the WR for cipher operations
624 * @req: cipher req.
625 * @ctx: crypto driver context of the request.
626 * @qid: ingress qid where response of this WR should be received.
627 * @op_type: encryption or decryption
628 */
629 static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
630 {
631 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(wrparam->req);
632 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
633 struct uld_ctx *u_ctx = ULD_CTX(ctx);
634 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
635 struct sk_buff *skb = NULL;
636 struct chcr_wr *chcr_req;
637 struct cpl_rx_phys_dsgl *phys_cpl;
638 struct chcr_blkcipher_req_ctx *reqctx =
639 ablkcipher_request_ctx(wrparam->req);
640 struct phys_sge_parm sg_param;
641 unsigned int frags = 0, transhdr_len, phys_dsgl;
642 int error;
643 unsigned int ivsize = AES_BLOCK_SIZE, kctx_len;
644 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
645 GFP_KERNEL : GFP_ATOMIC;
646 struct adapter *adap = padap(ctx->dev);
647
648 phys_dsgl = get_space_for_phys_dsgl(reqctx->dst_nents);
649
650 kctx_len = (DIV_ROUND_UP(ablkctx->enckey_len, 16) * 16);
651 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
652 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
653 if (!skb) {
654 error = -ENOMEM;
655 goto err;
656 }
657 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
658 chcr_req = __skb_put_zero(skb, transhdr_len);
659 chcr_req->sec_cpl.op_ivinsrtofst =
660 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 1);
661
662 chcr_req->sec_cpl.pldlen = htonl(ivsize + wrparam->bytes);
663 chcr_req->sec_cpl.aadstart_cipherstop_hi =
664 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, ivsize + 1, 0);
665
666 chcr_req->sec_cpl.cipherstop_lo_authinsert =
667 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
668 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
669 ablkctx->ciph_mode,
670 0, 0, ivsize >> 1);
671 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
672 0, 1, phys_dsgl);
673
674 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
675 if ((reqctx->op == CHCR_DECRYPT_OP) &&
676 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
677 CRYPTO_ALG_SUB_TYPE_CTR)) &&
678 (!(get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
679 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
680 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
681 } else {
682 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
683 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
684 memcpy(chcr_req->key_ctx.key, ablkctx->key,
685 ablkctx->enckey_len);
686 } else {
687 memcpy(chcr_req->key_ctx.key, ablkctx->key +
688 (ablkctx->enckey_len >> 1),
689 ablkctx->enckey_len >> 1);
690 memcpy(chcr_req->key_ctx.key +
691 (ablkctx->enckey_len >> 1),
692 ablkctx->key,
693 ablkctx->enckey_len >> 1);
694 }
695 }
696 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
697 sg_param.nents = reqctx->dst_nents;
698 sg_param.obsize = wrparam->bytes;
699 sg_param.qid = wrparam->qid;
700 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
701 reqctx->dst, &sg_param);
702 if (error)
703 goto map_fail1;
704
705 skb_set_transport_header(skb, transhdr_len);
706 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
707 write_sg_to_skb(skb, &frags, wrparam->srcsg, wrparam->bytes);
708 atomic_inc(&adap->chcr_stats.cipher_rqst);
709 create_wreq(ctx, chcr_req, &(wrparam->req->base), skb, kctx_len, 0, 1,
710 sizeof(struct cpl_rx_phys_dsgl) + phys_dsgl,
711 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
712 reqctx->skb = skb;
713 skb_get(skb);
714 return skb;
715 map_fail1:
716 kfree_skb(skb);
717 err:
718 return ERR_PTR(error);
719 }
720
721 static inline int chcr_keyctx_ck_size(unsigned int keylen)
722 {
723 int ck_size = 0;
724
725 if (keylen == AES_KEYSIZE_128)
726 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
727 else if (keylen == AES_KEYSIZE_192)
728 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
729 else if (keylen == AES_KEYSIZE_256)
730 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
731 else
732 ck_size = 0;
733
734 return ck_size;
735 }
736 static int chcr_cipher_fallback_setkey(struct crypto_ablkcipher *cipher,
737 const u8 *key,
738 unsigned int keylen)
739 {
740 struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher);
741 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
742 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
743 int err = 0;
744
745 crypto_skcipher_clear_flags(ablkctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
746 crypto_skcipher_set_flags(ablkctx->sw_cipher, cipher->base.crt_flags &
747 CRYPTO_TFM_REQ_MASK);
748 err = crypto_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
749 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
750 tfm->crt_flags |=
751 crypto_skcipher_get_flags(ablkctx->sw_cipher) &
752 CRYPTO_TFM_RES_MASK;
753 return err;
754 }
755
756 static int chcr_aes_cbc_setkey(struct crypto_ablkcipher *cipher,
757 const u8 *key,
758 unsigned int keylen)
759 {
760 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
761 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
762 unsigned int ck_size, context_size;
763 u16 alignment = 0;
764 int err;
765
766 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
767 if (err)
768 goto badkey_err;
769
770 ck_size = chcr_keyctx_ck_size(keylen);
771 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
772 memcpy(ablkctx->key, key, keylen);
773 ablkctx->enckey_len = keylen;
774 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
775 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
776 keylen + alignment) >> 4;
777
778 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
779 0, 0, context_size);
780 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
781 return 0;
782 badkey_err:
783 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
784 ablkctx->enckey_len = 0;
785
786 return err;
787 }
788
789 static int chcr_aes_ctr_setkey(struct crypto_ablkcipher *cipher,
790 const u8 *key,
791 unsigned int keylen)
792 {
793 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
794 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
795 unsigned int ck_size, context_size;
796 u16 alignment = 0;
797 int err;
798
799 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
800 if (err)
801 goto badkey_err;
802 ck_size = chcr_keyctx_ck_size(keylen);
803 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
804 memcpy(ablkctx->key, key, keylen);
805 ablkctx->enckey_len = keylen;
806 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
807 keylen + alignment) >> 4;
808
809 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
810 0, 0, context_size);
811 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
812
813 return 0;
814 badkey_err:
815 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
816 ablkctx->enckey_len = 0;
817
818 return err;
819 }
820
821 static int chcr_aes_rfc3686_setkey(struct crypto_ablkcipher *cipher,
822 const u8 *key,
823 unsigned int keylen)
824 {
825 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
826 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
827 unsigned int ck_size, context_size;
828 u16 alignment = 0;
829 int err;
830
831 if (keylen < CTR_RFC3686_NONCE_SIZE)
832 return -EINVAL;
833 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
834 CTR_RFC3686_NONCE_SIZE);
835
836 keylen -= CTR_RFC3686_NONCE_SIZE;
837 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
838 if (err)
839 goto badkey_err;
840
841 ck_size = chcr_keyctx_ck_size(keylen);
842 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
843 memcpy(ablkctx->key, key, keylen);
844 ablkctx->enckey_len = keylen;
845 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
846 keylen + alignment) >> 4;
847
848 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
849 0, 0, context_size);
850 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
851
852 return 0;
853 badkey_err:
854 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
855 ablkctx->enckey_len = 0;
856
857 return err;
858 }
859 static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
860 {
861 unsigned int size = AES_BLOCK_SIZE;
862 __be32 *b = (__be32 *)(dstiv + size);
863 u32 c, prev;
864
865 memcpy(dstiv, srciv, AES_BLOCK_SIZE);
866 for (; size >= 4; size -= 4) {
867 prev = be32_to_cpu(*--b);
868 c = prev + add;
869 *b = cpu_to_be32(c);
870 if (prev < c)
871 break;
872 add = 1;
873 }
874
875 }
876
877 static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
878 {
879 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
880 u64 c;
881 u32 temp = be32_to_cpu(*--b);
882
883 temp = ~temp;
884 c = (u64)temp + 1; // No of block can processed withou overflow
885 if ((bytes / AES_BLOCK_SIZE) > c)
886 bytes = c * AES_BLOCK_SIZE;
887 return bytes;
888 }
889
890 static int chcr_update_tweak(struct ablkcipher_request *req, u8 *iv)
891 {
892 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
893 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
894 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
895 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
896 struct crypto_cipher *cipher;
897 int ret, i;
898 u8 *key;
899 unsigned int keylen;
900
901 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
902 memcpy(iv, req->info, AES_BLOCK_SIZE);
903
904 if (IS_ERR(cipher)) {
905 ret = -ENOMEM;
906 goto out;
907 }
908 keylen = ablkctx->enckey_len / 2;
909 key = ablkctx->key + keylen;
910 ret = crypto_cipher_setkey(cipher, key, keylen);
911 if (ret)
912 goto out1;
913
914 crypto_cipher_encrypt_one(cipher, iv, iv);
915 for (i = 0; i < (reqctx->processed / AES_BLOCK_SIZE); i++)
916 gf128mul_x_ble((le128 *)iv, (le128 *)iv);
917
918 crypto_cipher_decrypt_one(cipher, iv, iv);
919 out1:
920 crypto_free_cipher(cipher);
921 out:
922 return ret;
923 }
924
925 static int chcr_update_cipher_iv(struct ablkcipher_request *req,
926 struct cpl_fw6_pld *fw6_pld, u8 *iv)
927 {
928 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
929 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
930 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
931 int ret = 0;
932
933 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
934 ctr_add_iv(iv, req->info, (reqctx->processed /
935 AES_BLOCK_SIZE));
936 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
937 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
938 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
939 AES_BLOCK_SIZE) + 1);
940 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
941 ret = chcr_update_tweak(req, iv);
942 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
943 if (reqctx->op)
944 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
945 16,
946 reqctx->processed - AES_BLOCK_SIZE);
947 else
948 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
949 }
950
951 return ret;
952
953 }
954
955 /* We need separate function for final iv because in rfc3686 Initial counter
956 * starts from 1 and buffer size of iv is 8 byte only which remains constant
957 * for subsequent update requests
958 */
959
960 static int chcr_final_cipher_iv(struct ablkcipher_request *req,
961 struct cpl_fw6_pld *fw6_pld, u8 *iv)
962 {
963 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
964 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
965 int subtype = get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm));
966 int ret = 0;
967
968 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
969 ctr_add_iv(iv, req->info, (reqctx->processed /
970 AES_BLOCK_SIZE));
971 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
972 ret = chcr_update_tweak(req, iv);
973 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
974 if (reqctx->op)
975 sg_pcopy_to_buffer(req->src, sg_nents(req->src), iv,
976 16,
977 reqctx->processed - AES_BLOCK_SIZE);
978 else
979 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
980
981 }
982 return ret;
983
984 }
985
986
987 static int chcr_handle_cipher_resp(struct ablkcipher_request *req,
988 unsigned char *input, int err)
989 {
990 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
991 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
992 struct uld_ctx *u_ctx = ULD_CTX(ctx);
993 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
994 struct sk_buff *skb;
995 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
996 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
997 struct cipher_wr_param wrparam;
998 int bytes;
999
1000 dma_unmap_sg(&u_ctx->lldi.pdev->dev, reqctx->dst, reqctx->dst_nents,
1001 DMA_FROM_DEVICE);
1002
1003 if (reqctx->skb) {
1004 kfree_skb(reqctx->skb);
1005 reqctx->skb = NULL;
1006 }
1007 if (err)
1008 goto complete;
1009
1010 if (req->nbytes == reqctx->processed) {
1011 err = chcr_final_cipher_iv(req, fw6_pld, req->info);
1012 goto complete;
1013 }
1014
1015 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1016 ctx->tx_qidx))) {
1017 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) {
1018 err = -EBUSY;
1019 goto complete;
1020 }
1021
1022 }
1023 wrparam.srcsg = scatterwalk_ffwd(reqctx->srcffwd, req->src,
1024 reqctx->processed);
1025 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd, reqctx->dstsg,
1026 reqctx->processed);
1027 if (!wrparam.srcsg || !reqctx->dst) {
1028 pr_err("Input sg list length less that nbytes\n");
1029 err = -EINVAL;
1030 goto complete;
1031 }
1032 bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dst, 1,
1033 SPACE_LEFT(ablkctx->enckey_len),
1034 &wrparam.snent, &reqctx->dst_nents);
1035 if ((bytes + reqctx->processed) >= req->nbytes)
1036 bytes = req->nbytes - reqctx->processed;
1037 else
1038 bytes = ROUND_16(bytes);
1039 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
1040 if (err)
1041 goto complete;
1042
1043 if (unlikely(bytes == 0)) {
1044 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1045 req->base.flags,
1046 wrparam.srcsg,
1047 reqctx->dst,
1048 req->nbytes - reqctx->processed,
1049 reqctx->iv,
1050 reqctx->op);
1051 goto complete;
1052 }
1053
1054 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1055 CRYPTO_ALG_SUB_TYPE_CTR)
1056 bytes = adjust_ctr_overflow(reqctx->iv, bytes);
1057 reqctx->processed += bytes;
1058 wrparam.qid = u_ctx->lldi.rxq_ids[ctx->rx_qidx];
1059 wrparam.req = req;
1060 wrparam.bytes = bytes;
1061 skb = create_cipher_wr(&wrparam);
1062 if (IS_ERR(skb)) {
1063 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
1064 err = PTR_ERR(skb);
1065 goto complete;
1066 }
1067 skb->dev = u_ctx->lldi.ports[0];
1068 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1069 chcr_send_wr(skb);
1070 return 0;
1071 complete:
1072 free_new_sg(reqctx->newdstsg);
1073 reqctx->newdstsg = NULL;
1074 req->base.complete(&req->base, err);
1075 return err;
1076 }
1077
1078 static int process_cipher(struct ablkcipher_request *req,
1079 unsigned short qid,
1080 struct sk_buff **skb,
1081 unsigned short op_type)
1082 {
1083 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1084 unsigned int ivsize = crypto_ablkcipher_ivsize(tfm);
1085 struct chcr_blkcipher_req_ctx *reqctx = ablkcipher_request_ctx(req);
1086 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1087 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1088 struct cipher_wr_param wrparam;
1089 int bytes, nents, err = -EINVAL;
1090
1091 reqctx->newdstsg = NULL;
1092 reqctx->processed = 0;
1093 if (!req->info)
1094 goto error;
1095 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
1096 (req->nbytes == 0) ||
1097 (req->nbytes % crypto_ablkcipher_blocksize(tfm))) {
1098 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1099 ablkctx->enckey_len, req->nbytes, ivsize);
1100 goto error;
1101 }
1102 wrparam.srcsg = req->src;
1103 if (is_newsg(req->dst, &nents)) {
1104 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
1105 if (IS_ERR(reqctx->newdstsg))
1106 return PTR_ERR(reqctx->newdstsg);
1107 reqctx->dstsg = reqctx->newdstsg;
1108 } else {
1109 reqctx->dstsg = req->dst;
1110 }
1111 bytes = chcr_sg_ent_in_wr(wrparam.srcsg, reqctx->dstsg, MIN_CIPHER_SG,
1112 SPACE_LEFT(ablkctx->enckey_len),
1113 &wrparam.snent,
1114 &reqctx->dst_nents);
1115 if ((bytes + reqctx->processed) >= req->nbytes)
1116 bytes = req->nbytes - reqctx->processed;
1117 else
1118 bytes = ROUND_16(bytes);
1119 if (unlikely(bytes > req->nbytes))
1120 bytes = req->nbytes;
1121 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1122 CRYPTO_ALG_SUB_TYPE_CTR) {
1123 bytes = adjust_ctr_overflow(req->info, bytes);
1124 }
1125 if (get_cryptoalg_subtype(crypto_ablkcipher_tfm(tfm)) ==
1126 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1127 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
1128 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->info,
1129 CTR_RFC3686_IV_SIZE);
1130
1131 /* initialize counter portion of counter block */
1132 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1133 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1134
1135 } else {
1136
1137 memcpy(reqctx->iv, req->info, ivsize);
1138 }
1139 if (unlikely(bytes == 0)) {
1140 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1141 req->base.flags,
1142 req->src,
1143 req->dst,
1144 req->nbytes,
1145 req->info,
1146 op_type);
1147 goto error;
1148 }
1149 reqctx->processed = bytes;
1150 reqctx->dst = reqctx->dstsg;
1151 reqctx->op = op_type;
1152 wrparam.qid = qid;
1153 wrparam.req = req;
1154 wrparam.bytes = bytes;
1155 *skb = create_cipher_wr(&wrparam);
1156 if (IS_ERR(*skb)) {
1157 err = PTR_ERR(*skb);
1158 goto error;
1159 }
1160
1161 return 0;
1162 error:
1163 free_new_sg(reqctx->newdstsg);
1164 reqctx->newdstsg = NULL;
1165 return err;
1166 }
1167
1168 static int chcr_aes_encrypt(struct ablkcipher_request *req)
1169 {
1170 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1171 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1172 struct sk_buff *skb = NULL;
1173 int err;
1174 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1175
1176 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1177 ctx->tx_qidx))) {
1178 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1179 return -EBUSY;
1180 }
1181
1182 err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb,
1183 CHCR_ENCRYPT_OP);
1184 if (err || !skb)
1185 return err;
1186 skb->dev = u_ctx->lldi.ports[0];
1187 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1188 chcr_send_wr(skb);
1189 return -EINPROGRESS;
1190 }
1191
1192 static int chcr_aes_decrypt(struct ablkcipher_request *req)
1193 {
1194 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
1195 struct chcr_context *ctx = crypto_ablkcipher_ctx(tfm);
1196 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1197 struct sk_buff *skb = NULL;
1198 int err;
1199
1200 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1201 ctx->tx_qidx))) {
1202 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1203 return -EBUSY;
1204 }
1205
1206 err = process_cipher(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], &skb,
1207 CHCR_DECRYPT_OP);
1208 if (err || !skb)
1209 return err;
1210 skb->dev = u_ctx->lldi.ports[0];
1211 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1212 chcr_send_wr(skb);
1213 return -EINPROGRESS;
1214 }
1215
1216 static int chcr_device_init(struct chcr_context *ctx)
1217 {
1218 struct uld_ctx *u_ctx = NULL;
1219 struct adapter *adap;
1220 unsigned int id;
1221 int txq_perchan, txq_idx, ntxq;
1222 int err = 0, rxq_perchan, rxq_idx;
1223
1224 id = smp_processor_id();
1225 if (!ctx->dev) {
1226 u_ctx = assign_chcr_device();
1227 if (!u_ctx) {
1228 pr_err("chcr device assignment fails\n");
1229 goto out;
1230 }
1231 ctx->dev = u_ctx->dev;
1232 adap = padap(ctx->dev);
1233 ntxq = min_not_zero((unsigned int)u_ctx->lldi.nrxq,
1234 adap->vres.ncrypto_fc);
1235 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
1236 txq_perchan = ntxq / u_ctx->lldi.nchan;
1237 rxq_idx = ctx->dev->tx_channel_id * rxq_perchan;
1238 rxq_idx += id % rxq_perchan;
1239 txq_idx = ctx->dev->tx_channel_id * txq_perchan;
1240 txq_idx += id % txq_perchan;
1241 spin_lock(&ctx->dev->lock_chcr_dev);
1242 ctx->rx_qidx = rxq_idx;
1243 ctx->tx_qidx = txq_idx;
1244 ctx->dev->tx_channel_id = !ctx->dev->tx_channel_id;
1245 ctx->dev->rx_channel_id = 0;
1246 spin_unlock(&ctx->dev->lock_chcr_dev);
1247 }
1248 out:
1249 return err;
1250 }
1251
1252 static int chcr_cra_init(struct crypto_tfm *tfm)
1253 {
1254 struct crypto_alg *alg = tfm->__crt_alg;
1255 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1256 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1257
1258 ablkctx->sw_cipher = crypto_alloc_skcipher(alg->cra_name, 0,
1259 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1260 if (IS_ERR(ablkctx->sw_cipher)) {
1261 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1262 return PTR_ERR(ablkctx->sw_cipher);
1263 }
1264 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1265 return chcr_device_init(crypto_tfm_ctx(tfm));
1266 }
1267
1268 static int chcr_rfc3686_init(struct crypto_tfm *tfm)
1269 {
1270 struct crypto_alg *alg = tfm->__crt_alg;
1271 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1272 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1273
1274 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1275 * cannot be used as fallback in chcr_handle_cipher_response
1276 */
1277 ablkctx->sw_cipher = crypto_alloc_skcipher("ctr(aes)", 0,
1278 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
1279 if (IS_ERR(ablkctx->sw_cipher)) {
1280 pr_err("failed to allocate fallback for %s\n", alg->cra_name);
1281 return PTR_ERR(ablkctx->sw_cipher);
1282 }
1283 tfm->crt_ablkcipher.reqsize = sizeof(struct chcr_blkcipher_req_ctx);
1284 return chcr_device_init(crypto_tfm_ctx(tfm));
1285 }
1286
1287
1288 static void chcr_cra_exit(struct crypto_tfm *tfm)
1289 {
1290 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1291 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1292
1293 crypto_free_skcipher(ablkctx->sw_cipher);
1294 }
1295
1296 static int get_alg_config(struct algo_param *params,
1297 unsigned int auth_size)
1298 {
1299 switch (auth_size) {
1300 case SHA1_DIGEST_SIZE:
1301 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1302 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1303 params->result_size = SHA1_DIGEST_SIZE;
1304 break;
1305 case SHA224_DIGEST_SIZE:
1306 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1307 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1308 params->result_size = SHA256_DIGEST_SIZE;
1309 break;
1310 case SHA256_DIGEST_SIZE:
1311 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1312 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1313 params->result_size = SHA256_DIGEST_SIZE;
1314 break;
1315 case SHA384_DIGEST_SIZE:
1316 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1317 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1318 params->result_size = SHA512_DIGEST_SIZE;
1319 break;
1320 case SHA512_DIGEST_SIZE:
1321 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1322 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1323 params->result_size = SHA512_DIGEST_SIZE;
1324 break;
1325 default:
1326 pr_err("chcr : ERROR, unsupported digest size\n");
1327 return -EINVAL;
1328 }
1329 return 0;
1330 }
1331
1332 static inline void chcr_free_shash(struct crypto_shash *base_hash)
1333 {
1334 crypto_free_shash(base_hash);
1335 }
1336
1337 /**
1338 * create_hash_wr - Create hash work request
1339 * @req - Cipher req base
1340 */
1341 static struct sk_buff *create_hash_wr(struct ahash_request *req,
1342 struct hash_wr_param *param)
1343 {
1344 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1345 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
1346 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1347 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1348 struct sk_buff *skb = NULL;
1349 struct chcr_wr *chcr_req;
1350 unsigned int frags = 0, transhdr_len, iopad_alignment = 0;
1351 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1352 unsigned int kctx_len = 0;
1353 u8 hash_size_in_response = 0;
1354 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1355 GFP_ATOMIC;
1356 struct adapter *adap = padap(ctx->dev);
1357
1358 iopad_alignment = KEYCTX_ALIGN_PAD(digestsize);
1359 kctx_len = param->alg_prm.result_size + iopad_alignment;
1360 if (param->opad_needed)
1361 kctx_len += param->alg_prm.result_size + iopad_alignment;
1362
1363 if (req_ctx->result)
1364 hash_size_in_response = digestsize;
1365 else
1366 hash_size_in_response = param->alg_prm.result_size;
1367 transhdr_len = HASH_TRANSHDR_SIZE(kctx_len);
1368 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
1369 if (!skb)
1370 return skb;
1371
1372 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
1373 chcr_req = __skb_put_zero(skb, transhdr_len);
1374
1375 chcr_req->sec_cpl.op_ivinsrtofst =
1376 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2, 0);
1377 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
1378
1379 chcr_req->sec_cpl.aadstart_cipherstop_hi =
1380 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
1381 chcr_req->sec_cpl.cipherstop_lo_authinsert =
1382 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
1383 chcr_req->sec_cpl.seqno_numivs =
1384 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
1385 param->opad_needed, 0);
1386
1387 chcr_req->sec_cpl.ivgen_hdrlen =
1388 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1389
1390 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1391 param->alg_prm.result_size);
1392
1393 if (param->opad_needed)
1394 memcpy(chcr_req->key_ctx.key +
1395 ((param->alg_prm.result_size <= 32) ? 32 :
1396 CHCR_HASH_MAX_DIGEST_SIZE),
1397 hmacctx->opad, param->alg_prm.result_size);
1398
1399 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
1400 param->alg_prm.mk_size, 0,
1401 param->opad_needed,
1402 ((kctx_len +
1403 sizeof(chcr_req->key_ctx)) >> 4));
1404 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
1405
1406 skb_set_transport_header(skb, transhdr_len);
1407 if (param->bfr_len != 0)
1408 write_buffer_to_skb(skb, &frags, req_ctx->reqbfr,
1409 param->bfr_len);
1410 if (param->sg_len != 0)
1411 write_sg_to_skb(skb, &frags, req->src, param->sg_len);
1412 atomic_inc(&adap->chcr_stats.digest_rqst);
1413 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len,
1414 hash_size_in_response, 0, DUMMY_BYTES, 0);
1415 req_ctx->skb = skb;
1416 skb_get(skb);
1417 return skb;
1418 }
1419
1420 static int chcr_ahash_update(struct ahash_request *req)
1421 {
1422 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1423 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1424 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1425 struct uld_ctx *u_ctx = NULL;
1426 struct sk_buff *skb;
1427 u8 remainder = 0, bs;
1428 unsigned int nbytes = req->nbytes;
1429 struct hash_wr_param params;
1430
1431 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1432
1433 u_ctx = ULD_CTX(ctx);
1434 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1435 ctx->tx_qidx))) {
1436 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1437 return -EBUSY;
1438 }
1439
1440 if (nbytes + req_ctx->reqlen >= bs) {
1441 remainder = (nbytes + req_ctx->reqlen) % bs;
1442 nbytes = nbytes + req_ctx->reqlen - remainder;
1443 } else {
1444 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
1445 + req_ctx->reqlen, nbytes, 0);
1446 req_ctx->reqlen += nbytes;
1447 return 0;
1448 }
1449
1450 params.opad_needed = 0;
1451 params.more = 1;
1452 params.last = 0;
1453 params.sg_len = nbytes - req_ctx->reqlen;
1454 params.bfr_len = req_ctx->reqlen;
1455 params.scmd1 = 0;
1456 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1457 req_ctx->result = 0;
1458 req_ctx->data_len += params.sg_len + params.bfr_len;
1459 skb = create_hash_wr(req, &params);
1460 if (!skb)
1461 return -ENOMEM;
1462
1463 if (remainder) {
1464 u8 *temp;
1465 /* Swap buffers */
1466 temp = req_ctx->reqbfr;
1467 req_ctx->reqbfr = req_ctx->skbfr;
1468 req_ctx->skbfr = temp;
1469 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
1470 req_ctx->reqbfr, remainder, req->nbytes -
1471 remainder);
1472 }
1473 req_ctx->reqlen = remainder;
1474 skb->dev = u_ctx->lldi.ports[0];
1475 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1476 chcr_send_wr(skb);
1477
1478 return -EINPROGRESS;
1479 }
1480
1481 static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1482 {
1483 memset(bfr_ptr, 0, bs);
1484 *bfr_ptr = 0x80;
1485 if (bs == 64)
1486 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
1487 else
1488 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
1489 }
1490
1491 static int chcr_ahash_final(struct ahash_request *req)
1492 {
1493 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1494 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1495 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1496 struct hash_wr_param params;
1497 struct sk_buff *skb;
1498 struct uld_ctx *u_ctx = NULL;
1499 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1500
1501 u_ctx = ULD_CTX(ctx);
1502 if (is_hmac(crypto_ahash_tfm(rtfm)))
1503 params.opad_needed = 1;
1504 else
1505 params.opad_needed = 0;
1506 params.sg_len = 0;
1507 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1508 req_ctx->result = 1;
1509 params.bfr_len = req_ctx->reqlen;
1510 req_ctx->data_len += params.bfr_len + params.sg_len;
1511 if (req_ctx->reqlen == 0) {
1512 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1513 params.last = 0;
1514 params.more = 1;
1515 params.scmd1 = 0;
1516 params.bfr_len = bs;
1517
1518 } else {
1519 params.scmd1 = req_ctx->data_len;
1520 params.last = 1;
1521 params.more = 0;
1522 }
1523 skb = create_hash_wr(req, &params);
1524 if (!skb)
1525 return -ENOMEM;
1526
1527 skb->dev = u_ctx->lldi.ports[0];
1528 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1529 chcr_send_wr(skb);
1530 return -EINPROGRESS;
1531 }
1532
1533 static int chcr_ahash_finup(struct ahash_request *req)
1534 {
1535 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1536 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1537 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1538 struct uld_ctx *u_ctx = NULL;
1539 struct sk_buff *skb;
1540 struct hash_wr_param params;
1541 u8 bs;
1542
1543 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1544 u_ctx = ULD_CTX(ctx);
1545
1546 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1547 ctx->tx_qidx))) {
1548 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1549 return -EBUSY;
1550 }
1551
1552 if (is_hmac(crypto_ahash_tfm(rtfm)))
1553 params.opad_needed = 1;
1554 else
1555 params.opad_needed = 0;
1556
1557 params.sg_len = req->nbytes;
1558 params.bfr_len = req_ctx->reqlen;
1559 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1560 req_ctx->data_len += params.bfr_len + params.sg_len;
1561 req_ctx->result = 1;
1562 if ((req_ctx->reqlen + req->nbytes) == 0) {
1563 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
1564 params.last = 0;
1565 params.more = 1;
1566 params.scmd1 = 0;
1567 params.bfr_len = bs;
1568 } else {
1569 params.scmd1 = req_ctx->data_len;
1570 params.last = 1;
1571 params.more = 0;
1572 }
1573
1574 skb = create_hash_wr(req, &params);
1575 if (!skb)
1576 return -ENOMEM;
1577
1578 skb->dev = u_ctx->lldi.ports[0];
1579 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1580 chcr_send_wr(skb);
1581
1582 return -EINPROGRESS;
1583 }
1584
1585 static int chcr_ahash_digest(struct ahash_request *req)
1586 {
1587 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1588 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
1589 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1590 struct uld_ctx *u_ctx = NULL;
1591 struct sk_buff *skb;
1592 struct hash_wr_param params;
1593 u8 bs;
1594
1595 rtfm->init(req);
1596 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1597
1598 u_ctx = ULD_CTX(ctx);
1599 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
1600 ctx->tx_qidx))) {
1601 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
1602 return -EBUSY;
1603 }
1604
1605 if (is_hmac(crypto_ahash_tfm(rtfm)))
1606 params.opad_needed = 1;
1607 else
1608 params.opad_needed = 0;
1609
1610 params.last = 0;
1611 params.more = 0;
1612 params.sg_len = req->nbytes;
1613 params.bfr_len = 0;
1614 params.scmd1 = 0;
1615 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1616 req_ctx->result = 1;
1617 req_ctx->data_len += params.bfr_len + params.sg_len;
1618
1619 if (req->nbytes == 0) {
1620 create_last_hash_block(req_ctx->reqbfr, bs, 0);
1621 params.more = 1;
1622 params.bfr_len = bs;
1623 }
1624
1625 skb = create_hash_wr(req, &params);
1626 if (!skb)
1627 return -ENOMEM;
1628
1629 skb->dev = u_ctx->lldi.ports[0];
1630 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
1631 chcr_send_wr(skb);
1632 return -EINPROGRESS;
1633 }
1634
1635 static int chcr_ahash_export(struct ahash_request *areq, void *out)
1636 {
1637 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1638 struct chcr_ahash_req_ctx *state = out;
1639
1640 state->reqlen = req_ctx->reqlen;
1641 state->data_len = req_ctx->data_len;
1642 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
1643 memcpy(state->partial_hash, req_ctx->partial_hash,
1644 CHCR_HASH_MAX_DIGEST_SIZE);
1645 return 0;
1646 }
1647
1648 static int chcr_ahash_import(struct ahash_request *areq, const void *in)
1649 {
1650 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1651 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
1652
1653 req_ctx->reqlen = state->reqlen;
1654 req_ctx->data_len = state->data_len;
1655 req_ctx->reqbfr = req_ctx->bfr1;
1656 req_ctx->skbfr = req_ctx->bfr2;
1657 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
1658 memcpy(req_ctx->partial_hash, state->partial_hash,
1659 CHCR_HASH_MAX_DIGEST_SIZE);
1660 return 0;
1661 }
1662
1663 static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
1664 unsigned int keylen)
1665 {
1666 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
1667 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1668 unsigned int digestsize = crypto_ahash_digestsize(tfm);
1669 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1670 unsigned int i, err = 0, updated_digestsize;
1671
1672 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
1673
1674 /* use the key to calculate the ipad and opad. ipad will sent with the
1675 * first request's data. opad will be sent with the final hash result
1676 * ipad in hmacctx->ipad and opad in hmacctx->opad location
1677 */
1678 shash->tfm = hmacctx->base_hash;
1679 shash->flags = crypto_shash_get_flags(hmacctx->base_hash);
1680 if (keylen > bs) {
1681 err = crypto_shash_digest(shash, key, keylen,
1682 hmacctx->ipad);
1683 if (err)
1684 goto out;
1685 keylen = digestsize;
1686 } else {
1687 memcpy(hmacctx->ipad, key, keylen);
1688 }
1689 memset(hmacctx->ipad + keylen, 0, bs - keylen);
1690 memcpy(hmacctx->opad, hmacctx->ipad, bs);
1691
1692 for (i = 0; i < bs / sizeof(int); i++) {
1693 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
1694 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
1695 }
1696
1697 updated_digestsize = digestsize;
1698 if (digestsize == SHA224_DIGEST_SIZE)
1699 updated_digestsize = SHA256_DIGEST_SIZE;
1700 else if (digestsize == SHA384_DIGEST_SIZE)
1701 updated_digestsize = SHA512_DIGEST_SIZE;
1702 err = chcr_compute_partial_hash(shash, hmacctx->ipad,
1703 hmacctx->ipad, digestsize);
1704 if (err)
1705 goto out;
1706 chcr_change_order(hmacctx->ipad, updated_digestsize);
1707
1708 err = chcr_compute_partial_hash(shash, hmacctx->opad,
1709 hmacctx->opad, digestsize);
1710 if (err)
1711 goto out;
1712 chcr_change_order(hmacctx->opad, updated_digestsize);
1713 out:
1714 return err;
1715 }
1716
1717 static int chcr_aes_xts_setkey(struct crypto_ablkcipher *cipher, const u8 *key,
1718 unsigned int key_len)
1719 {
1720 struct chcr_context *ctx = crypto_ablkcipher_ctx(cipher);
1721 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1722 unsigned short context_size = 0;
1723 int err;
1724
1725 err = chcr_cipher_fallback_setkey(cipher, key, key_len);
1726 if (err)
1727 goto badkey_err;
1728
1729 memcpy(ablkctx->key, key, key_len);
1730 ablkctx->enckey_len = key_len;
1731 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
1732 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
1733 ablkctx->key_ctx_hdr =
1734 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
1735 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
1736 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
1737 CHCR_KEYCTX_NO_KEY, 1,
1738 0, context_size);
1739 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
1740 return 0;
1741 badkey_err:
1742 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1743 ablkctx->enckey_len = 0;
1744
1745 return err;
1746 }
1747
1748 static int chcr_sha_init(struct ahash_request *areq)
1749 {
1750 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1751 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1752 int digestsize = crypto_ahash_digestsize(tfm);
1753
1754 req_ctx->data_len = 0;
1755 req_ctx->reqlen = 0;
1756 req_ctx->reqbfr = req_ctx->bfr1;
1757 req_ctx->skbfr = req_ctx->bfr2;
1758 req_ctx->skb = NULL;
1759 req_ctx->result = 0;
1760 copy_hash_init_values(req_ctx->partial_hash, digestsize);
1761 return 0;
1762 }
1763
1764 static int chcr_sha_cra_init(struct crypto_tfm *tfm)
1765 {
1766 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1767 sizeof(struct chcr_ahash_req_ctx));
1768 return chcr_device_init(crypto_tfm_ctx(tfm));
1769 }
1770
1771 static int chcr_hmac_init(struct ahash_request *areq)
1772 {
1773 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1774 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
1775 struct chcr_context *ctx = crypto_tfm_ctx(crypto_ahash_tfm(rtfm));
1776 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1777 unsigned int digestsize = crypto_ahash_digestsize(rtfm);
1778 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
1779
1780 chcr_sha_init(areq);
1781 req_ctx->data_len = bs;
1782 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1783 if (digestsize == SHA224_DIGEST_SIZE)
1784 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1785 SHA256_DIGEST_SIZE);
1786 else if (digestsize == SHA384_DIGEST_SIZE)
1787 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1788 SHA512_DIGEST_SIZE);
1789 else
1790 memcpy(req_ctx->partial_hash, hmacctx->ipad,
1791 digestsize);
1792 }
1793 return 0;
1794 }
1795
1796 static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
1797 {
1798 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1799 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1800 unsigned int digestsize =
1801 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
1802
1803 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
1804 sizeof(struct chcr_ahash_req_ctx));
1805 hmacctx->base_hash = chcr_alloc_shash(digestsize);
1806 if (IS_ERR(hmacctx->base_hash))
1807 return PTR_ERR(hmacctx->base_hash);
1808 return chcr_device_init(crypto_tfm_ctx(tfm));
1809 }
1810
1811 static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
1812 {
1813 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
1814 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
1815
1816 if (hmacctx->base_hash) {
1817 chcr_free_shash(hmacctx->base_hash);
1818 hmacctx->base_hash = NULL;
1819 }
1820 }
1821
1822 static int is_newsg(struct scatterlist *sgl, unsigned int *newents)
1823 {
1824 int nents = 0;
1825 int ret = 0;
1826
1827 while (sgl) {
1828 if (sgl->length > CHCR_SG_SIZE)
1829 ret = 1;
1830 nents += DIV_ROUND_UP(sgl->length, CHCR_SG_SIZE);
1831 sgl = sg_next(sgl);
1832 }
1833 *newents = nents;
1834 return ret;
1835 }
1836
1837 static inline void free_new_sg(struct scatterlist *sgl)
1838 {
1839 kfree(sgl);
1840 }
1841
1842 static struct scatterlist *alloc_new_sg(struct scatterlist *sgl,
1843 unsigned int nents)
1844 {
1845 struct scatterlist *newsg, *sg;
1846 int i, len, processed = 0;
1847 struct page *spage;
1848 int offset;
1849
1850 newsg = kmalloc_array(nents, sizeof(struct scatterlist), GFP_KERNEL);
1851 if (!newsg)
1852 return ERR_PTR(-ENOMEM);
1853 sg = newsg;
1854 sg_init_table(sg, nents);
1855 offset = sgl->offset;
1856 spage = sg_page(sgl);
1857 for (i = 0; i < nents; i++) {
1858 len = min_t(u32, sgl->length - processed, CHCR_SG_SIZE);
1859 sg_set_page(sg, spage, len, offset);
1860 processed += len;
1861 offset += len;
1862 if (offset >= PAGE_SIZE) {
1863 offset = offset % PAGE_SIZE;
1864 spage++;
1865 }
1866 if (processed == sgl->length) {
1867 processed = 0;
1868 sgl = sg_next(sgl);
1869 if (!sgl)
1870 break;
1871 spage = sg_page(sgl);
1872 offset = sgl->offset;
1873 }
1874 sg = sg_next(sg);
1875 }
1876 return newsg;
1877 }
1878
1879 static int chcr_copy_assoc(struct aead_request *req,
1880 struct chcr_aead_ctx *ctx)
1881 {
1882 SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null);
1883
1884 skcipher_request_set_tfm(skreq, ctx->null);
1885 skcipher_request_set_callback(skreq, aead_request_flags(req),
1886 NULL, NULL);
1887 skcipher_request_set_crypt(skreq, req->src, req->dst, req->assoclen,
1888 NULL);
1889
1890 return crypto_skcipher_encrypt(skreq);
1891 }
1892 static int chcr_aead_need_fallback(struct aead_request *req, int src_nent,
1893 int aadmax, int wrlen,
1894 unsigned short op_type)
1895 {
1896 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
1897
1898 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
1899 (req->assoclen > aadmax) ||
1900 (src_nent > MAX_SKB_FRAGS) ||
1901 (wrlen > MAX_WR_SIZE))
1902 return 1;
1903 return 0;
1904 }
1905
1906 static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
1907 {
1908 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1909 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1910 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1911 struct aead_request *subreq = aead_request_ctx(req);
1912
1913 aead_request_set_tfm(subreq, aeadctx->sw_cipher);
1914 aead_request_set_callback(subreq, req->base.flags,
1915 req->base.complete, req->base.data);
1916 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
1917 req->iv);
1918 aead_request_set_ad(subreq, req->assoclen);
1919 return op_type ? crypto_aead_decrypt(subreq) :
1920 crypto_aead_encrypt(subreq);
1921 }
1922
1923 static struct sk_buff *create_authenc_wr(struct aead_request *req,
1924 unsigned short qid,
1925 int size,
1926 unsigned short op_type)
1927 {
1928 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1929 struct chcr_context *ctx = crypto_aead_ctx(tfm);
1930 struct uld_ctx *u_ctx = ULD_CTX(ctx);
1931 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
1932 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
1933 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1934 struct sk_buff *skb = NULL;
1935 struct chcr_wr *chcr_req;
1936 struct cpl_rx_phys_dsgl *phys_cpl;
1937 struct phys_sge_parm sg_param;
1938 struct scatterlist *src;
1939 unsigned int frags = 0, transhdr_len;
1940 unsigned int ivsize = crypto_aead_ivsize(tfm), dst_size = 0;
1941 unsigned int kctx_len = 0, nents;
1942 unsigned short stop_offset = 0;
1943 unsigned int assoclen = req->assoclen;
1944 unsigned int authsize = crypto_aead_authsize(tfm);
1945 int error = -EINVAL, src_nent;
1946 int null = 0;
1947 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1948 GFP_ATOMIC;
1949 struct adapter *adap = padap(ctx->dev);
1950
1951 reqctx->newdstsg = NULL;
1952 dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
1953 authsize);
1954 if (aeadctx->enckey_len == 0 || (req->cryptlen <= 0))
1955 goto err;
1956
1957 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
1958 goto err;
1959 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
1960 if (src_nent < 0)
1961 goto err;
1962 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
1963
1964 if (req->src != req->dst) {
1965 error = chcr_copy_assoc(req, aeadctx);
1966 if (error)
1967 return ERR_PTR(error);
1968 }
1969 if (dst_size && is_newsg(req->dst, &nents)) {
1970 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
1971 if (IS_ERR(reqctx->newdstsg))
1972 return ERR_CAST(reqctx->newdstsg);
1973 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
1974 reqctx->newdstsg, req->assoclen);
1975 } else {
1976 if (req->src == req->dst)
1977 reqctx->dst = src;
1978 else
1979 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
1980 req->dst, req->assoclen);
1981 }
1982 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_NULL) {
1983 null = 1;
1984 assoclen = 0;
1985 }
1986 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
1987 (op_type ? -authsize : authsize));
1988 if (reqctx->dst_nents < 0) {
1989 pr_err("AUTHENC:Invalid Destination sg entries\n");
1990 error = -EINVAL;
1991 goto err;
1992 }
1993 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
1994 kctx_len = (ntohl(KEY_CONTEXT_CTX_LEN_V(aeadctx->key_ctx_hdr)) << 4)
1995 - sizeof(chcr_req->key_ctx);
1996 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
1997 if (chcr_aead_need_fallback(req, src_nent + MIN_AUTH_SG,
1998 T6_MAX_AAD_SIZE,
1999 transhdr_len + (sgl_len(src_nent + MIN_AUTH_SG) * 8),
2000 op_type)) {
2001 atomic_inc(&adap->chcr_stats.fallback);
2002 free_new_sg(reqctx->newdstsg);
2003 reqctx->newdstsg = NULL;
2004 return ERR_PTR(chcr_aead_fallback(req, op_type));
2005 }
2006 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
2007 if (!skb) {
2008 error = -ENOMEM;
2009 goto err;
2010 }
2011
2012 /* LLD is going to write the sge hdr. */
2013 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
2014
2015 /* Write WR */
2016 chcr_req = __skb_put_zero(skb, transhdr_len);
2017
2018 stop_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
2019
2020 /*
2021 * Input order is AAD,IV and Payload. where IV should be included as
2022 * the part of authdata. All other fields should be filled according
2023 * to the hardware spec
2024 */
2025 chcr_req->sec_cpl.op_ivinsrtofst =
2026 FILL_SEC_CPL_OP_IVINSR(ctx->dev->rx_channel_id, 2,
2027 (ivsize ? (assoclen + 1) : 0));
2028 chcr_req->sec_cpl.pldlen = htonl(assoclen + ivsize + req->cryptlen);
2029 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2030 assoclen ? 1 : 0, assoclen,
2031 assoclen + ivsize + 1,
2032 (stop_offset & 0x1F0) >> 4);
2033 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2034 stop_offset & 0xF,
2035 null ? 0 : assoclen + ivsize + 1,
2036 stop_offset, stop_offset);
2037 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2038 (op_type == CHCR_ENCRYPT_OP) ? 1 : 0,
2039 CHCR_SCMD_CIPHER_MODE_AES_CBC,
2040 actx->auth_mode, aeadctx->hmac_ctrl,
2041 ivsize >> 1);
2042 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2043 0, 1, dst_size);
2044
2045 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2046 if (op_type == CHCR_ENCRYPT_OP)
2047 memcpy(chcr_req->key_ctx.key, aeadctx->key,
2048 aeadctx->enckey_len);
2049 else
2050 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2051 aeadctx->enckey_len);
2052
2053 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) <<
2054 4), actx->h_iopad, kctx_len -
2055 (DIV_ROUND_UP(aeadctx->enckey_len, 16) << 4));
2056
2057 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2058 sg_param.nents = reqctx->dst_nents;
2059 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
2060 sg_param.qid = qid;
2061 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
2062 reqctx->dst, &sg_param);
2063 if (error)
2064 goto dstmap_fail;
2065
2066 skb_set_transport_header(skb, transhdr_len);
2067
2068 if (assoclen) {
2069 /* AAD buffer in */
2070 write_sg_to_skb(skb, &frags, req->src, assoclen);
2071
2072 }
2073 write_buffer_to_skb(skb, &frags, req->iv, ivsize);
2074 write_sg_to_skb(skb, &frags, src, req->cryptlen);
2075 atomic_inc(&adap->chcr_stats.cipher_rqst);
2076 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1,
2077 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
2078 reqctx->skb = skb;
2079 skb_get(skb);
2080
2081 return skb;
2082 dstmap_fail:
2083 /* ivmap_fail: */
2084 kfree_skb(skb);
2085 err:
2086 free_new_sg(reqctx->newdstsg);
2087 reqctx->newdstsg = NULL;
2088 return ERR_PTR(error);
2089 }
2090
2091 static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2092 {
2093 __be32 data;
2094
2095 memset(block, 0, csize);
2096 block += csize;
2097
2098 if (csize >= 4)
2099 csize = 4;
2100 else if (msglen > (unsigned int)(1 << (8 * csize)))
2101 return -EOVERFLOW;
2102
2103 data = cpu_to_be32(msglen);
2104 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2105
2106 return 0;
2107 }
2108
2109 static void generate_b0(struct aead_request *req,
2110 struct chcr_aead_ctx *aeadctx,
2111 unsigned short op_type)
2112 {
2113 unsigned int l, lp, m;
2114 int rc;
2115 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2116 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2117 u8 *b0 = reqctx->scratch_pad;
2118
2119 m = crypto_aead_authsize(aead);
2120
2121 memcpy(b0, reqctx->iv, 16);
2122
2123 lp = b0[0];
2124 l = lp + 1;
2125
2126 /* set m, bits 3-5 */
2127 *b0 |= (8 * ((m - 2) / 2));
2128
2129 /* set adata, bit 6, if associated data is used */
2130 if (req->assoclen)
2131 *b0 |= 64;
2132 rc = set_msg_len(b0 + 16 - l,
2133 (op_type == CHCR_DECRYPT_OP) ?
2134 req->cryptlen - m : req->cryptlen, l);
2135 }
2136
2137 static inline int crypto_ccm_check_iv(const u8 *iv)
2138 {
2139 /* 2 <= L <= 8, so 1 <= L' <= 7. */
2140 if (iv[0] < 1 || iv[0] > 7)
2141 return -EINVAL;
2142
2143 return 0;
2144 }
2145
2146 static int ccm_format_packet(struct aead_request *req,
2147 struct chcr_aead_ctx *aeadctx,
2148 unsigned int sub_type,
2149 unsigned short op_type)
2150 {
2151 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2152 int rc = 0;
2153
2154 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2155 reqctx->iv[0] = 3;
2156 memcpy(reqctx->iv + 1, &aeadctx->salt[0], 3);
2157 memcpy(reqctx->iv + 4, req->iv, 8);
2158 memset(reqctx->iv + 12, 0, 4);
2159 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2160 htons(req->assoclen - 8);
2161 } else {
2162 memcpy(reqctx->iv, req->iv, 16);
2163 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2164 htons(req->assoclen);
2165 }
2166 generate_b0(req, aeadctx, op_type);
2167 /* zero the ctr value */
2168 memset(reqctx->iv + 15 - reqctx->iv[0], 0, reqctx->iv[0] + 1);
2169 return rc;
2170 }
2171
2172 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2173 unsigned int dst_size,
2174 struct aead_request *req,
2175 unsigned short op_type,
2176 struct chcr_context *chcrctx)
2177 {
2178 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2179 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2180 unsigned int ivsize = AES_BLOCK_SIZE;
2181 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2182 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
2183 unsigned int c_id = chcrctx->dev->rx_channel_id;
2184 unsigned int ccm_xtra;
2185 unsigned char tag_offset = 0, auth_offset = 0;
2186 unsigned int assoclen;
2187
2188 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2189 assoclen = req->assoclen - 8;
2190 else
2191 assoclen = req->assoclen;
2192 ccm_xtra = CCM_B0_SIZE +
2193 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2194
2195 auth_offset = req->cryptlen ?
2196 (assoclen + ivsize + 1 + ccm_xtra) : 0;
2197 if (op_type == CHCR_DECRYPT_OP) {
2198 if (crypto_aead_authsize(tfm) != req->cryptlen)
2199 tag_offset = crypto_aead_authsize(tfm);
2200 else
2201 auth_offset = 0;
2202 }
2203
2204
2205 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(c_id,
2206 2, (ivsize ? (assoclen + 1) : 0) +
2207 ccm_xtra);
2208 sec_cpl->pldlen =
2209 htonl(assoclen + ivsize + req->cryptlen + ccm_xtra);
2210 /* For CCM there wil be b0 always. So AAD start will be 1 always */
2211 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2212 1, assoclen + ccm_xtra, assoclen
2213 + ivsize + 1 + ccm_xtra, 0);
2214
2215 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2216 auth_offset, tag_offset,
2217 (op_type == CHCR_ENCRYPT_OP) ? 0 :
2218 crypto_aead_authsize(tfm));
2219 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2220 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
2221 cipher_mode, mac_mode,
2222 aeadctx->hmac_ctrl, ivsize >> 1);
2223
2224 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2225 1, dst_size);
2226 }
2227
2228 int aead_ccm_validate_input(unsigned short op_type,
2229 struct aead_request *req,
2230 struct chcr_aead_ctx *aeadctx,
2231 unsigned int sub_type)
2232 {
2233 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2234 if (crypto_ccm_check_iv(req->iv)) {
2235 pr_err("CCM: IV check fails\n");
2236 return -EINVAL;
2237 }
2238 } else {
2239 if (req->assoclen != 16 && req->assoclen != 20) {
2240 pr_err("RFC4309: Invalid AAD length %d\n",
2241 req->assoclen);
2242 return -EINVAL;
2243 }
2244 }
2245 if (aeadctx->enckey_len == 0) {
2246 pr_err("CCM: Encryption key not set\n");
2247 return -EINVAL;
2248 }
2249 return 0;
2250 }
2251
2252 unsigned int fill_aead_req_fields(struct sk_buff *skb,
2253 struct aead_request *req,
2254 struct scatterlist *src,
2255 unsigned int ivsize,
2256 struct chcr_aead_ctx *aeadctx)
2257 {
2258 unsigned int frags = 0;
2259 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2260 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2261 /* b0 and aad length(if available) */
2262
2263 write_buffer_to_skb(skb, &frags, reqctx->scratch_pad, CCM_B0_SIZE +
2264 (req->assoclen ? CCM_AAD_FIELD_SIZE : 0));
2265 if (req->assoclen) {
2266 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2267 write_sg_to_skb(skb, &frags, req->src,
2268 req->assoclen - 8);
2269 else
2270 write_sg_to_skb(skb, &frags, req->src, req->assoclen);
2271 }
2272 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
2273 if (req->cryptlen)
2274 write_sg_to_skb(skb, &frags, src, req->cryptlen);
2275
2276 return frags;
2277 }
2278
2279 static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
2280 unsigned short qid,
2281 int size,
2282 unsigned short op_type)
2283 {
2284 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2285 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2286 struct uld_ctx *u_ctx = ULD_CTX(ctx);
2287 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2288 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2289 struct sk_buff *skb = NULL;
2290 struct chcr_wr *chcr_req;
2291 struct cpl_rx_phys_dsgl *phys_cpl;
2292 struct phys_sge_parm sg_param;
2293 struct scatterlist *src;
2294 unsigned int frags = 0, transhdr_len, ivsize = AES_BLOCK_SIZE;
2295 unsigned int dst_size = 0, kctx_len, nents;
2296 unsigned int sub_type;
2297 unsigned int authsize = crypto_aead_authsize(tfm);
2298 int error = -EINVAL, src_nent;
2299 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2300 GFP_ATOMIC;
2301 struct adapter *adap = padap(ctx->dev);
2302
2303 dst_size = req->assoclen + req->cryptlen + (op_type ? -authsize :
2304 authsize);
2305 reqctx->newdstsg = NULL;
2306 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
2307 goto err;
2308 src_nent = sg_nents_for_len(req->src, req->assoclen + req->cryptlen);
2309 if (src_nent < 0)
2310 goto err;
2311
2312 sub_type = get_aead_subtype(tfm);
2313 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, req->assoclen);
2314 if (req->src != req->dst) {
2315 error = chcr_copy_assoc(req, aeadctx);
2316 if (error) {
2317 pr_err("AAD copy to destination buffer fails\n");
2318 return ERR_PTR(error);
2319 }
2320 }
2321 if (dst_size && is_newsg(req->dst, &nents)) {
2322 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
2323 if (IS_ERR(reqctx->newdstsg))
2324 return ERR_CAST(reqctx->newdstsg);
2325 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2326 reqctx->newdstsg, req->assoclen);
2327 } else {
2328 if (req->src == req->dst)
2329 reqctx->dst = src;
2330 else
2331 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2332 req->dst, req->assoclen);
2333 }
2334 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
2335 (op_type ? -authsize : authsize));
2336 if (reqctx->dst_nents < 0) {
2337 pr_err("CCM:Invalid Destination sg entries\n");
2338 error = -EINVAL;
2339 goto err;
2340 }
2341 error = aead_ccm_validate_input(op_type, req, aeadctx, sub_type);
2342 if (error)
2343 goto err;
2344
2345 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
2346 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) * 2;
2347 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2348 if (chcr_aead_need_fallback(req, src_nent + MIN_CCM_SG,
2349 T6_MAX_AAD_SIZE - 18,
2350 transhdr_len + (sgl_len(src_nent + MIN_CCM_SG) * 8),
2351 op_type)) {
2352 atomic_inc(&adap->chcr_stats.fallback);
2353 free_new_sg(reqctx->newdstsg);
2354 reqctx->newdstsg = NULL;
2355 return ERR_PTR(chcr_aead_fallback(req, op_type));
2356 }
2357
2358 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
2359
2360 if (!skb) {
2361 error = -ENOMEM;
2362 goto err;
2363 }
2364
2365 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
2366
2367 chcr_req = __skb_put_zero(skb, transhdr_len);
2368
2369 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, op_type, ctx);
2370
2371 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2372 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
2373 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
2374 16), aeadctx->key, aeadctx->enckey_len);
2375
2376 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2377 error = ccm_format_packet(req, aeadctx, sub_type, op_type);
2378 if (error)
2379 goto dstmap_fail;
2380
2381 sg_param.nents = reqctx->dst_nents;
2382 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
2383 sg_param.qid = qid;
2384 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
2385 reqctx->dst, &sg_param);
2386 if (error)
2387 goto dstmap_fail;
2388
2389 skb_set_transport_header(skb, transhdr_len);
2390 frags = fill_aead_req_fields(skb, req, src, ivsize, aeadctx);
2391 atomic_inc(&adap->chcr_stats.aead_rqst);
2392 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, 0, 1,
2393 sizeof(struct cpl_rx_phys_dsgl) + dst_size, 0);
2394 reqctx->skb = skb;
2395 skb_get(skb);
2396 return skb;
2397 dstmap_fail:
2398 kfree_skb(skb);
2399 err:
2400 free_new_sg(reqctx->newdstsg);
2401 reqctx->newdstsg = NULL;
2402 return ERR_PTR(error);
2403 }
2404
2405 static struct sk_buff *create_gcm_wr(struct aead_request *req,
2406 unsigned short qid,
2407 int size,
2408 unsigned short op_type)
2409 {
2410 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2411 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2412 struct uld_ctx *u_ctx = ULD_CTX(ctx);
2413 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2414 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2415 struct sk_buff *skb = NULL;
2416 struct chcr_wr *chcr_req;
2417 struct cpl_rx_phys_dsgl *phys_cpl;
2418 struct phys_sge_parm sg_param;
2419 struct scatterlist *src;
2420 unsigned int frags = 0, transhdr_len;
2421 unsigned int ivsize = AES_BLOCK_SIZE;
2422 unsigned int dst_size = 0, kctx_len, nents, assoclen = req->assoclen;
2423 unsigned char tag_offset = 0;
2424 unsigned int authsize = crypto_aead_authsize(tfm);
2425 int error = -EINVAL, src_nent;
2426 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2427 GFP_ATOMIC;
2428 struct adapter *adap = padap(ctx->dev);
2429
2430 reqctx->newdstsg = NULL;
2431 dst_size = assoclen + req->cryptlen + (op_type ? -authsize :
2432 authsize);
2433 /* validate key size */
2434 if (aeadctx->enckey_len == 0)
2435 goto err;
2436
2437 if (op_type && req->cryptlen < crypto_aead_authsize(tfm))
2438 goto err;
2439 src_nent = sg_nents_for_len(req->src, assoclen + req->cryptlen);
2440 if (src_nent < 0)
2441 goto err;
2442
2443 src = scatterwalk_ffwd(reqctx->srcffwd, req->src, assoclen);
2444 if (req->src != req->dst) {
2445 error = chcr_copy_assoc(req, aeadctx);
2446 if (error)
2447 return ERR_PTR(error);
2448 }
2449
2450 if (dst_size && is_newsg(req->dst, &nents)) {
2451 reqctx->newdstsg = alloc_new_sg(req->dst, nents);
2452 if (IS_ERR(reqctx->newdstsg))
2453 return ERR_CAST(reqctx->newdstsg);
2454 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2455 reqctx->newdstsg, assoclen);
2456 } else {
2457 if (req->src == req->dst)
2458 reqctx->dst = src;
2459 else
2460 reqctx->dst = scatterwalk_ffwd(reqctx->dstffwd,
2461 req->dst, assoclen);
2462 }
2463
2464 reqctx->dst_nents = sg_nents_for_len(reqctx->dst, req->cryptlen +
2465 (op_type ? -authsize : authsize));
2466 if (reqctx->dst_nents < 0) {
2467 pr_err("GCM:Invalid Destination sg entries\n");
2468 error = -EINVAL;
2469 goto err;
2470 }
2471
2472
2473 dst_size = get_space_for_phys_dsgl(reqctx->dst_nents);
2474 kctx_len = ((DIV_ROUND_UP(aeadctx->enckey_len, 16)) << 4) +
2475 AEAD_H_SIZE;
2476 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
2477 if (chcr_aead_need_fallback(req, src_nent + MIN_GCM_SG,
2478 T6_MAX_AAD_SIZE,
2479 transhdr_len + (sgl_len(src_nent + MIN_GCM_SG) * 8),
2480 op_type)) {
2481 atomic_inc(&adap->chcr_stats.fallback);
2482 free_new_sg(reqctx->newdstsg);
2483 reqctx->newdstsg = NULL;
2484 return ERR_PTR(chcr_aead_fallback(req, op_type));
2485 }
2486 skb = alloc_skb((transhdr_len + sizeof(struct sge_opaque_hdr)), flags);
2487 if (!skb) {
2488 error = -ENOMEM;
2489 goto err;
2490 }
2491
2492 /* NIC driver is going to write the sge hdr. */
2493 skb_reserve(skb, sizeof(struct sge_opaque_hdr));
2494
2495 chcr_req = __skb_put_zero(skb, transhdr_len);
2496
2497 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
2498 assoclen = req->assoclen - 8;
2499
2500 tag_offset = (op_type == CHCR_ENCRYPT_OP) ? 0 : authsize;
2501 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
2502 ctx->dev->rx_channel_id, 2, (ivsize ?
2503 (assoclen + 1) : 0));
2504 chcr_req->sec_cpl.pldlen =
2505 htonl(assoclen + ivsize + req->cryptlen);
2506 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
2507 assoclen ? 1 : 0, assoclen,
2508 assoclen + ivsize + 1, 0);
2509 chcr_req->sec_cpl.cipherstop_lo_authinsert =
2510 FILL_SEC_CPL_AUTHINSERT(0, assoclen + ivsize + 1,
2511 tag_offset, tag_offset);
2512 chcr_req->sec_cpl.seqno_numivs =
2513 FILL_SEC_CPL_SCMD0_SEQNO(op_type, (op_type ==
2514 CHCR_ENCRYPT_OP) ? 1 : 0,
2515 CHCR_SCMD_CIPHER_MODE_AES_GCM,
2516 CHCR_SCMD_AUTH_MODE_GHASH,
2517 aeadctx->hmac_ctrl, ivsize >> 1);
2518 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2519 0, 1, dst_size);
2520 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
2521 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
2522 memcpy(chcr_req->key_ctx.key + (DIV_ROUND_UP(aeadctx->enckey_len, 16) *
2523 16), GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
2524
2525 /* prepare a 16 byte iv */
2526 /* S A L T | IV | 0x00000001 */
2527 if (get_aead_subtype(tfm) ==
2528 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
2529 memcpy(reqctx->iv, aeadctx->salt, 4);
2530 memcpy(reqctx->iv + 4, req->iv, 8);
2531 } else {
2532 memcpy(reqctx->iv, req->iv, 12);
2533 }
2534 *((unsigned int *)(reqctx->iv + 12)) = htonl(0x01);
2535
2536 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2537 sg_param.nents = reqctx->dst_nents;
2538 sg_param.obsize = req->cryptlen + (op_type ? -authsize : authsize);
2539 sg_param.qid = qid;
2540 error = map_writesg_phys_cpl(&u_ctx->lldi.pdev->dev, phys_cpl,
2541 reqctx->dst, &sg_param);
2542 if (error)
2543 goto dstmap_fail;
2544
2545 skb_set_transport_header(skb, transhdr_len);
2546 write_sg_to_skb(skb, &frags, req->src, assoclen);
2547 write_buffer_to_skb(skb, &frags, reqctx->iv, ivsize);
2548 write_sg_to_skb(skb, &frags, src, req->cryptlen);
2549 atomic_inc(&adap->chcr_stats.aead_rqst);
2550 create_wreq(ctx, chcr_req, &req->base, skb, kctx_len, size, 1,
2551 sizeof(struct cpl_rx_phys_dsgl) + dst_size,
2552 reqctx->verify);
2553 reqctx->skb = skb;
2554 skb_get(skb);
2555 return skb;
2556
2557 dstmap_fail:
2558 /* ivmap_fail: */
2559 kfree_skb(skb);
2560 err:
2561 free_new_sg(reqctx->newdstsg);
2562 reqctx->newdstsg = NULL;
2563 return ERR_PTR(error);
2564 }
2565
2566
2567
2568 static int chcr_aead_cra_init(struct crypto_aead *tfm)
2569 {
2570 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2571 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2572 struct aead_alg *alg = crypto_aead_alg(tfm);
2573
2574 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
2575 CRYPTO_ALG_NEED_FALLBACK |
2576 CRYPTO_ALG_ASYNC);
2577 if (IS_ERR(aeadctx->sw_cipher))
2578 return PTR_ERR(aeadctx->sw_cipher);
2579 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
2580 sizeof(struct aead_request) +
2581 crypto_aead_reqsize(aeadctx->sw_cipher)));
2582 aeadctx->null = crypto_get_default_null_skcipher();
2583 if (IS_ERR(aeadctx->null))
2584 return PTR_ERR(aeadctx->null);
2585 return chcr_device_init(ctx);
2586 }
2587
2588 static void chcr_aead_cra_exit(struct crypto_aead *tfm)
2589 {
2590 struct chcr_context *ctx = crypto_aead_ctx(tfm);
2591 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2592
2593 crypto_put_default_null_skcipher();
2594 crypto_free_aead(aeadctx->sw_cipher);
2595 }
2596
2597 static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
2598 unsigned int authsize)
2599 {
2600 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2601
2602 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
2603 aeadctx->mayverify = VERIFY_HW;
2604 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2605 }
2606 static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
2607 unsigned int authsize)
2608 {
2609 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2610 u32 maxauth = crypto_aead_maxauthsize(tfm);
2611
2612 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
2613 * true for sha1. authsize == 12 condition should be before
2614 * authsize == (maxauth >> 1)
2615 */
2616 if (authsize == ICV_4) {
2617 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2618 aeadctx->mayverify = VERIFY_HW;
2619 } else if (authsize == ICV_6) {
2620 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2621 aeadctx->mayverify = VERIFY_HW;
2622 } else if (authsize == ICV_10) {
2623 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2624 aeadctx->mayverify = VERIFY_HW;
2625 } else if (authsize == ICV_12) {
2626 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2627 aeadctx->mayverify = VERIFY_HW;
2628 } else if (authsize == ICV_14) {
2629 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2630 aeadctx->mayverify = VERIFY_HW;
2631 } else if (authsize == (maxauth >> 1)) {
2632 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2633 aeadctx->mayverify = VERIFY_HW;
2634 } else if (authsize == maxauth) {
2635 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2636 aeadctx->mayverify = VERIFY_HW;
2637 } else {
2638 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2639 aeadctx->mayverify = VERIFY_SW;
2640 }
2641 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2642 }
2643
2644
2645 static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
2646 {
2647 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2648
2649 switch (authsize) {
2650 case ICV_4:
2651 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2652 aeadctx->mayverify = VERIFY_HW;
2653 break;
2654 case ICV_8:
2655 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2656 aeadctx->mayverify = VERIFY_HW;
2657 break;
2658 case ICV_12:
2659 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2660 aeadctx->mayverify = VERIFY_HW;
2661 break;
2662 case ICV_14:
2663 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2664 aeadctx->mayverify = VERIFY_HW;
2665 break;
2666 case ICV_16:
2667 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2668 aeadctx->mayverify = VERIFY_HW;
2669 break;
2670 case ICV_13:
2671 case ICV_15:
2672 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2673 aeadctx->mayverify = VERIFY_SW;
2674 break;
2675 default:
2676
2677 crypto_tfm_set_flags((struct crypto_tfm *) tfm,
2678 CRYPTO_TFM_RES_BAD_KEY_LEN);
2679 return -EINVAL;
2680 }
2681 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2682 }
2683
2684 static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
2685 unsigned int authsize)
2686 {
2687 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2688
2689 switch (authsize) {
2690 case ICV_8:
2691 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2692 aeadctx->mayverify = VERIFY_HW;
2693 break;
2694 case ICV_12:
2695 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2696 aeadctx->mayverify = VERIFY_HW;
2697 break;
2698 case ICV_16:
2699 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2700 aeadctx->mayverify = VERIFY_HW;
2701 break;
2702 default:
2703 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
2704 CRYPTO_TFM_RES_BAD_KEY_LEN);
2705 return -EINVAL;
2706 }
2707 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2708 }
2709
2710 static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
2711 unsigned int authsize)
2712 {
2713 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
2714
2715 switch (authsize) {
2716 case ICV_4:
2717 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
2718 aeadctx->mayverify = VERIFY_HW;
2719 break;
2720 case ICV_6:
2721 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
2722 aeadctx->mayverify = VERIFY_HW;
2723 break;
2724 case ICV_8:
2725 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
2726 aeadctx->mayverify = VERIFY_HW;
2727 break;
2728 case ICV_10:
2729 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
2730 aeadctx->mayverify = VERIFY_HW;
2731 break;
2732 case ICV_12:
2733 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
2734 aeadctx->mayverify = VERIFY_HW;
2735 break;
2736 case ICV_14:
2737 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
2738 aeadctx->mayverify = VERIFY_HW;
2739 break;
2740 case ICV_16:
2741 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
2742 aeadctx->mayverify = VERIFY_HW;
2743 break;
2744 default:
2745 crypto_tfm_set_flags((struct crypto_tfm *)tfm,
2746 CRYPTO_TFM_RES_BAD_KEY_LEN);
2747 return -EINVAL;
2748 }
2749 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2750 }
2751
2752 static int chcr_ccm_common_setkey(struct crypto_aead *aead,
2753 const u8 *key,
2754 unsigned int keylen)
2755 {
2756 struct chcr_context *ctx = crypto_aead_ctx(aead);
2757 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2758 unsigned char ck_size, mk_size;
2759 int key_ctx_size = 0;
2760
2761 key_ctx_size = sizeof(struct _key_ctx) +
2762 ((DIV_ROUND_UP(keylen, 16)) << 4) * 2;
2763 if (keylen == AES_KEYSIZE_128) {
2764 mk_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2765 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2766 } else if (keylen == AES_KEYSIZE_192) {
2767 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2768 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
2769 } else if (keylen == AES_KEYSIZE_256) {
2770 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2771 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
2772 } else {
2773 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2774 CRYPTO_TFM_RES_BAD_KEY_LEN);
2775 aeadctx->enckey_len = 0;
2776 return -EINVAL;
2777 }
2778 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
2779 key_ctx_size >> 4);
2780 memcpy(aeadctx->key, key, keylen);
2781 aeadctx->enckey_len = keylen;
2782
2783 return 0;
2784 }
2785
2786 static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
2787 const u8 *key,
2788 unsigned int keylen)
2789 {
2790 struct chcr_context *ctx = crypto_aead_ctx(aead);
2791 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2792 int error;
2793
2794 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2795 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
2796 CRYPTO_TFM_REQ_MASK);
2797 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2798 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2799 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2800 CRYPTO_TFM_RES_MASK);
2801 if (error)
2802 return error;
2803 return chcr_ccm_common_setkey(aead, key, keylen);
2804 }
2805
2806 static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
2807 unsigned int keylen)
2808 {
2809 struct chcr_context *ctx = crypto_aead_ctx(aead);
2810 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2811 int error;
2812
2813 if (keylen < 3) {
2814 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2815 CRYPTO_TFM_RES_BAD_KEY_LEN);
2816 aeadctx->enckey_len = 0;
2817 return -EINVAL;
2818 }
2819 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2820 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
2821 CRYPTO_TFM_REQ_MASK);
2822 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2823 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2824 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2825 CRYPTO_TFM_RES_MASK);
2826 if (error)
2827 return error;
2828 keylen -= 3;
2829 memcpy(aeadctx->salt, key + keylen, 3);
2830 return chcr_ccm_common_setkey(aead, key, keylen);
2831 }
2832
2833 static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
2834 unsigned int keylen)
2835 {
2836 struct chcr_context *ctx = crypto_aead_ctx(aead);
2837 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2838 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
2839 struct crypto_cipher *cipher;
2840 unsigned int ck_size;
2841 int ret = 0, key_ctx_size = 0;
2842
2843 aeadctx->enckey_len = 0;
2844 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2845 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
2846 & CRYPTO_TFM_REQ_MASK);
2847 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2848 crypto_aead_clear_flags(aead, CRYPTO_TFM_RES_MASK);
2849 crypto_aead_set_flags(aead, crypto_aead_get_flags(aeadctx->sw_cipher) &
2850 CRYPTO_TFM_RES_MASK);
2851 if (ret)
2852 goto out;
2853
2854 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
2855 keylen > 3) {
2856 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
2857 memcpy(aeadctx->salt, key + keylen, 4);
2858 }
2859 if (keylen == AES_KEYSIZE_128) {
2860 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2861 } else if (keylen == AES_KEYSIZE_192) {
2862 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2863 } else if (keylen == AES_KEYSIZE_256) {
2864 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2865 } else {
2866 crypto_tfm_set_flags((struct crypto_tfm *)aead,
2867 CRYPTO_TFM_RES_BAD_KEY_LEN);
2868 pr_err("GCM: Invalid key length %d\n", keylen);
2869 ret = -EINVAL;
2870 goto out;
2871 }
2872
2873 memcpy(aeadctx->key, key, keylen);
2874 aeadctx->enckey_len = keylen;
2875 key_ctx_size = sizeof(struct _key_ctx) +
2876 ((DIV_ROUND_UP(keylen, 16)) << 4) +
2877 AEAD_H_SIZE;
2878 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
2879 CHCR_KEYCTX_MAC_KEY_SIZE_128,
2880 0, 0,
2881 key_ctx_size >> 4);
2882 /* Calculate the H = CIPH(K, 0 repeated 16 times).
2883 * It will go in key context
2884 */
2885 cipher = crypto_alloc_cipher("aes-generic", 0, 0);
2886 if (IS_ERR(cipher)) {
2887 aeadctx->enckey_len = 0;
2888 ret = -ENOMEM;
2889 goto out;
2890 }
2891
2892 ret = crypto_cipher_setkey(cipher, key, keylen);
2893 if (ret) {
2894 aeadctx->enckey_len = 0;
2895 goto out1;
2896 }
2897 memset(gctx->ghash_h, 0, AEAD_H_SIZE);
2898 crypto_cipher_encrypt_one(cipher, gctx->ghash_h, gctx->ghash_h);
2899
2900 out1:
2901 crypto_free_cipher(cipher);
2902 out:
2903 return ret;
2904 }
2905
2906 static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
2907 unsigned int keylen)
2908 {
2909 struct chcr_context *ctx = crypto_aead_ctx(authenc);
2910 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2911 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2912 /* it contains auth and cipher key both*/
2913 struct crypto_authenc_keys keys;
2914 unsigned int bs;
2915 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
2916 int err = 0, i, key_ctx_len = 0;
2917 unsigned char ck_size = 0;
2918 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
2919 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
2920 struct algo_param param;
2921 int align;
2922 u8 *o_ptr = NULL;
2923
2924 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
2925 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
2926 & CRYPTO_TFM_REQ_MASK);
2927 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
2928 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
2929 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
2930 & CRYPTO_TFM_RES_MASK);
2931 if (err)
2932 goto out;
2933
2934 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
2935 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
2936 goto out;
2937 }
2938
2939 if (get_alg_config(&param, max_authsize)) {
2940 pr_err("chcr : Unsupported digest size\n");
2941 goto out;
2942 }
2943 if (keys.enckeylen == AES_KEYSIZE_128) {
2944 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
2945 } else if (keys.enckeylen == AES_KEYSIZE_192) {
2946 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
2947 } else if (keys.enckeylen == AES_KEYSIZE_256) {
2948 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
2949 } else {
2950 pr_err("chcr : Unsupported cipher key\n");
2951 goto out;
2952 }
2953
2954 /* Copy only encryption key. We use authkey to generate h(ipad) and
2955 * h(opad) so authkey is not needed again. authkeylen size have the
2956 * size of the hash digest size.
2957 */
2958 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
2959 aeadctx->enckey_len = keys.enckeylen;
2960 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
2961 aeadctx->enckey_len << 3);
2962
2963 base_hash = chcr_alloc_shash(max_authsize);
2964 if (IS_ERR(base_hash)) {
2965 pr_err("chcr : Base driver cannot be loaded\n");
2966 aeadctx->enckey_len = 0;
2967 return -EINVAL;
2968 }
2969 {
2970 SHASH_DESC_ON_STACK(shash, base_hash);
2971 shash->tfm = base_hash;
2972 shash->flags = crypto_shash_get_flags(base_hash);
2973 bs = crypto_shash_blocksize(base_hash);
2974 align = KEYCTX_ALIGN_PAD(max_authsize);
2975 o_ptr = actx->h_iopad + param.result_size + align;
2976
2977 if (keys.authkeylen > bs) {
2978 err = crypto_shash_digest(shash, keys.authkey,
2979 keys.authkeylen,
2980 o_ptr);
2981 if (err) {
2982 pr_err("chcr : Base driver cannot be loaded\n");
2983 goto out;
2984 }
2985 keys.authkeylen = max_authsize;
2986 } else
2987 memcpy(o_ptr, keys.authkey, keys.authkeylen);
2988
2989 /* Compute the ipad-digest*/
2990 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
2991 memcpy(pad, o_ptr, keys.authkeylen);
2992 for (i = 0; i < bs >> 2; i++)
2993 *((unsigned int *)pad + i) ^= IPAD_DATA;
2994
2995 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
2996 max_authsize))
2997 goto out;
2998 /* Compute the opad-digest */
2999 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3000 memcpy(pad, o_ptr, keys.authkeylen);
3001 for (i = 0; i < bs >> 2; i++)
3002 *((unsigned int *)pad + i) ^= OPAD_DATA;
3003
3004 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
3005 goto out;
3006
3007 /* convert the ipad and opad digest to network order */
3008 chcr_change_order(actx->h_iopad, param.result_size);
3009 chcr_change_order(o_ptr, param.result_size);
3010 key_ctx_len = sizeof(struct _key_ctx) +
3011 ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4) +
3012 (param.result_size + align) * 2;
3013 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
3014 0, 1, key_ctx_len >> 4);
3015 actx->auth_mode = param.auth_mode;
3016 chcr_free_shash(base_hash);
3017
3018 return 0;
3019 }
3020 out:
3021 aeadctx->enckey_len = 0;
3022 if (!IS_ERR(base_hash))
3023 chcr_free_shash(base_hash);
3024 return -EINVAL;
3025 }
3026
3027 static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3028 const u8 *key, unsigned int keylen)
3029 {
3030 struct chcr_context *ctx = crypto_aead_ctx(authenc);
3031 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
3032 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3033 struct crypto_authenc_keys keys;
3034 int err;
3035 /* it contains auth and cipher key both*/
3036 int key_ctx_len = 0;
3037 unsigned char ck_size = 0;
3038
3039 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3040 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3041 & CRYPTO_TFM_REQ_MASK);
3042 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
3043 crypto_aead_clear_flags(authenc, CRYPTO_TFM_RES_MASK);
3044 crypto_aead_set_flags(authenc, crypto_aead_get_flags(aeadctx->sw_cipher)
3045 & CRYPTO_TFM_RES_MASK);
3046 if (err)
3047 goto out;
3048
3049 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) {
3050 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
3051 goto out;
3052 }
3053 if (keys.enckeylen == AES_KEYSIZE_128) {
3054 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3055 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3056 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3057 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3058 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3059 } else {
3060 pr_err("chcr : Unsupported cipher key\n");
3061 goto out;
3062 }
3063 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3064 aeadctx->enckey_len = keys.enckeylen;
3065 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3066 aeadctx->enckey_len << 3);
3067 key_ctx_len = sizeof(struct _key_ctx)
3068 + ((DIV_ROUND_UP(keys.enckeylen, 16)) << 4);
3069
3070 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3071 0, key_ctx_len >> 4);
3072 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
3073 return 0;
3074 out:
3075 aeadctx->enckey_len = 0;
3076 return -EINVAL;
3077 }
3078 static int chcr_aead_encrypt(struct aead_request *req)
3079 {
3080 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3081 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3082
3083 reqctx->verify = VERIFY_HW;
3084
3085 switch (get_aead_subtype(tfm)) {
3086 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
3087 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
3088 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3089 create_authenc_wr);
3090 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3091 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3092 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3093 create_aead_ccm_wr);
3094 default:
3095 return chcr_aead_op(req, CHCR_ENCRYPT_OP, 0,
3096 create_gcm_wr);
3097 }
3098 }
3099
3100 static int chcr_aead_decrypt(struct aead_request *req)
3101 {
3102 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3103 struct chcr_aead_ctx *aeadctx = AEAD_CTX(crypto_aead_ctx(tfm));
3104 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3105 int size;
3106
3107 if (aeadctx->mayverify == VERIFY_SW) {
3108 size = crypto_aead_maxauthsize(tfm);
3109 reqctx->verify = VERIFY_SW;
3110 } else {
3111 size = 0;
3112 reqctx->verify = VERIFY_HW;
3113 }
3114
3115 switch (get_aead_subtype(tfm)) {
3116 case CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC:
3117 case CRYPTO_ALG_SUB_TYPE_AEAD_NULL:
3118 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3119 create_authenc_wr);
3120 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3121 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
3122 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3123 create_aead_ccm_wr);
3124 default:
3125 return chcr_aead_op(req, CHCR_DECRYPT_OP, size,
3126 create_gcm_wr);
3127 }
3128 }
3129
3130 static int chcr_aead_op(struct aead_request *req,
3131 unsigned short op_type,
3132 int size,
3133 create_wr_t create_wr_fn)
3134 {
3135 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3136 struct chcr_context *ctx = crypto_aead_ctx(tfm);
3137 struct uld_ctx *u_ctx;
3138 struct sk_buff *skb;
3139
3140 if (!ctx->dev) {
3141 pr_err("chcr : %s : No crypto device.\n", __func__);
3142 return -ENXIO;
3143 }
3144 u_ctx = ULD_CTX(ctx);
3145 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
3146 ctx->tx_qidx)) {
3147 if (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))
3148 return -EBUSY;
3149 }
3150
3151 /* Form a WR from req */
3152 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[ctx->rx_qidx], size,
3153 op_type);
3154
3155 if (IS_ERR(skb) || !skb)
3156 return PTR_ERR(skb);
3157
3158 skb->dev = u_ctx->lldi.ports[0];
3159 set_wr_txq(skb, CPL_PRIORITY_DATA, ctx->tx_qidx);
3160 chcr_send_wr(skb);
3161 return -EINPROGRESS;
3162 }
3163 static struct chcr_alg_template driver_algs[] = {
3164 /* AES-CBC */
3165 {
3166 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
3167 .is_registered = 0,
3168 .alg.crypto = {
3169 .cra_name = "cbc(aes)",
3170 .cra_driver_name = "cbc-aes-chcr",
3171 .cra_blocksize = AES_BLOCK_SIZE,
3172 .cra_init = chcr_cra_init,
3173 .cra_exit = chcr_cra_exit,
3174 .cra_u.ablkcipher = {
3175 .min_keysize = AES_MIN_KEY_SIZE,
3176 .max_keysize = AES_MAX_KEY_SIZE,
3177 .ivsize = AES_BLOCK_SIZE,
3178 .setkey = chcr_aes_cbc_setkey,
3179 .encrypt = chcr_aes_encrypt,
3180 .decrypt = chcr_aes_decrypt,
3181 }
3182 }
3183 },
3184 {
3185 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
3186 .is_registered = 0,
3187 .alg.crypto = {
3188 .cra_name = "xts(aes)",
3189 .cra_driver_name = "xts-aes-chcr",
3190 .cra_blocksize = AES_BLOCK_SIZE,
3191 .cra_init = chcr_cra_init,
3192 .cra_exit = NULL,
3193 .cra_u .ablkcipher = {
3194 .min_keysize = 2 * AES_MIN_KEY_SIZE,
3195 .max_keysize = 2 * AES_MAX_KEY_SIZE,
3196 .ivsize = AES_BLOCK_SIZE,
3197 .setkey = chcr_aes_xts_setkey,
3198 .encrypt = chcr_aes_encrypt,
3199 .decrypt = chcr_aes_decrypt,
3200 }
3201 }
3202 },
3203 {
3204 .type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
3205 .is_registered = 0,
3206 .alg.crypto = {
3207 .cra_name = "ctr(aes)",
3208 .cra_driver_name = "ctr-aes-chcr",
3209 .cra_blocksize = 1,
3210 .cra_init = chcr_cra_init,
3211 .cra_exit = chcr_cra_exit,
3212 .cra_u.ablkcipher = {
3213 .min_keysize = AES_MIN_KEY_SIZE,
3214 .max_keysize = AES_MAX_KEY_SIZE,
3215 .ivsize = AES_BLOCK_SIZE,
3216 .setkey = chcr_aes_ctr_setkey,
3217 .encrypt = chcr_aes_encrypt,
3218 .decrypt = chcr_aes_decrypt,
3219 }
3220 }
3221 },
3222 {
3223 .type = CRYPTO_ALG_TYPE_ABLKCIPHER |
3224 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3225 .is_registered = 0,
3226 .alg.crypto = {
3227 .cra_name = "rfc3686(ctr(aes))",
3228 .cra_driver_name = "rfc3686-ctr-aes-chcr",
3229 .cra_blocksize = 1,
3230 .cra_init = chcr_rfc3686_init,
3231 .cra_exit = chcr_cra_exit,
3232 .cra_u.ablkcipher = {
3233 .min_keysize = AES_MIN_KEY_SIZE +
3234 CTR_RFC3686_NONCE_SIZE,
3235 .max_keysize = AES_MAX_KEY_SIZE +
3236 CTR_RFC3686_NONCE_SIZE,
3237 .ivsize = CTR_RFC3686_IV_SIZE,
3238 .setkey = chcr_aes_rfc3686_setkey,
3239 .encrypt = chcr_aes_encrypt,
3240 .decrypt = chcr_aes_decrypt,
3241 .geniv = "seqiv",
3242 }
3243 }
3244 },
3245 /* SHA */
3246 {
3247 .type = CRYPTO_ALG_TYPE_AHASH,
3248 .is_registered = 0,
3249 .alg.hash = {
3250 .halg.digestsize = SHA1_DIGEST_SIZE,
3251 .halg.base = {
3252 .cra_name = "sha1",
3253 .cra_driver_name = "sha1-chcr",
3254 .cra_blocksize = SHA1_BLOCK_SIZE,
3255 }
3256 }
3257 },
3258 {
3259 .type = CRYPTO_ALG_TYPE_AHASH,
3260 .is_registered = 0,
3261 .alg.hash = {
3262 .halg.digestsize = SHA256_DIGEST_SIZE,
3263 .halg.base = {
3264 .cra_name = "sha256",
3265 .cra_driver_name = "sha256-chcr",
3266 .cra_blocksize = SHA256_BLOCK_SIZE,
3267 }
3268 }
3269 },
3270 {
3271 .type = CRYPTO_ALG_TYPE_AHASH,
3272 .is_registered = 0,
3273 .alg.hash = {
3274 .halg.digestsize = SHA224_DIGEST_SIZE,
3275 .halg.base = {
3276 .cra_name = "sha224",
3277 .cra_driver_name = "sha224-chcr",
3278 .cra_blocksize = SHA224_BLOCK_SIZE,
3279 }
3280 }
3281 },
3282 {
3283 .type = CRYPTO_ALG_TYPE_AHASH,
3284 .is_registered = 0,
3285 .alg.hash = {
3286 .halg.digestsize = SHA384_DIGEST_SIZE,
3287 .halg.base = {
3288 .cra_name = "sha384",
3289 .cra_driver_name = "sha384-chcr",
3290 .cra_blocksize = SHA384_BLOCK_SIZE,
3291 }
3292 }
3293 },
3294 {
3295 .type = CRYPTO_ALG_TYPE_AHASH,
3296 .is_registered = 0,
3297 .alg.hash = {
3298 .halg.digestsize = SHA512_DIGEST_SIZE,
3299 .halg.base = {
3300 .cra_name = "sha512",
3301 .cra_driver_name = "sha512-chcr",
3302 .cra_blocksize = SHA512_BLOCK_SIZE,
3303 }
3304 }
3305 },
3306 /* HMAC */
3307 {
3308 .type = CRYPTO_ALG_TYPE_HMAC,
3309 .is_registered = 0,
3310 .alg.hash = {
3311 .halg.digestsize = SHA1_DIGEST_SIZE,
3312 .halg.base = {
3313 .cra_name = "hmac(sha1)",
3314 .cra_driver_name = "hmac-sha1-chcr",
3315 .cra_blocksize = SHA1_BLOCK_SIZE,
3316 }
3317 }
3318 },
3319 {
3320 .type = CRYPTO_ALG_TYPE_HMAC,
3321 .is_registered = 0,
3322 .alg.hash = {
3323 .halg.digestsize = SHA224_DIGEST_SIZE,
3324 .halg.base = {
3325 .cra_name = "hmac(sha224)",
3326 .cra_driver_name = "hmac-sha224-chcr",
3327 .cra_blocksize = SHA224_BLOCK_SIZE,
3328 }
3329 }
3330 },
3331 {
3332 .type = CRYPTO_ALG_TYPE_HMAC,
3333 .is_registered = 0,
3334 .alg.hash = {
3335 .halg.digestsize = SHA256_DIGEST_SIZE,
3336 .halg.base = {
3337 .cra_name = "hmac(sha256)",
3338 .cra_driver_name = "hmac-sha256-chcr",
3339 .cra_blocksize = SHA256_BLOCK_SIZE,
3340 }
3341 }
3342 },
3343 {
3344 .type = CRYPTO_ALG_TYPE_HMAC,
3345 .is_registered = 0,
3346 .alg.hash = {
3347 .halg.digestsize = SHA384_DIGEST_SIZE,
3348 .halg.base = {
3349 .cra_name = "hmac(sha384)",
3350 .cra_driver_name = "hmac-sha384-chcr",
3351 .cra_blocksize = SHA384_BLOCK_SIZE,
3352 }
3353 }
3354 },
3355 {
3356 .type = CRYPTO_ALG_TYPE_HMAC,
3357 .is_registered = 0,
3358 .alg.hash = {
3359 .halg.digestsize = SHA512_DIGEST_SIZE,
3360 .halg.base = {
3361 .cra_name = "hmac(sha512)",
3362 .cra_driver_name = "hmac-sha512-chcr",
3363 .cra_blocksize = SHA512_BLOCK_SIZE,
3364 }
3365 }
3366 },
3367 /* Add AEAD Algorithms */
3368 {
3369 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
3370 .is_registered = 0,
3371 .alg.aead = {
3372 .base = {
3373 .cra_name = "gcm(aes)",
3374 .cra_driver_name = "gcm-aes-chcr",
3375 .cra_blocksize = 1,
3376 .cra_priority = CHCR_AEAD_PRIORITY,
3377 .cra_ctxsize = sizeof(struct chcr_context) +
3378 sizeof(struct chcr_aead_ctx) +
3379 sizeof(struct chcr_gcm_ctx),
3380 },
3381 .ivsize = 12,
3382 .maxauthsize = GHASH_DIGEST_SIZE,
3383 .setkey = chcr_gcm_setkey,
3384 .setauthsize = chcr_gcm_setauthsize,
3385 }
3386 },
3387 {
3388 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
3389 .is_registered = 0,
3390 .alg.aead = {
3391 .base = {
3392 .cra_name = "rfc4106(gcm(aes))",
3393 .cra_driver_name = "rfc4106-gcm-aes-chcr",
3394 .cra_blocksize = 1,
3395 .cra_priority = CHCR_AEAD_PRIORITY + 1,
3396 .cra_ctxsize = sizeof(struct chcr_context) +
3397 sizeof(struct chcr_aead_ctx) +
3398 sizeof(struct chcr_gcm_ctx),
3399
3400 },
3401 .ivsize = 8,
3402 .maxauthsize = GHASH_DIGEST_SIZE,
3403 .setkey = chcr_gcm_setkey,
3404 .setauthsize = chcr_4106_4309_setauthsize,
3405 }
3406 },
3407 {
3408 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
3409 .is_registered = 0,
3410 .alg.aead = {
3411 .base = {
3412 .cra_name = "ccm(aes)",
3413 .cra_driver_name = "ccm-aes-chcr",
3414 .cra_blocksize = 1,
3415 .cra_priority = CHCR_AEAD_PRIORITY,
3416 .cra_ctxsize = sizeof(struct chcr_context) +
3417 sizeof(struct chcr_aead_ctx),
3418
3419 },
3420 .ivsize = AES_BLOCK_SIZE,
3421 .maxauthsize = GHASH_DIGEST_SIZE,
3422 .setkey = chcr_aead_ccm_setkey,
3423 .setauthsize = chcr_ccm_setauthsize,
3424 }
3425 },
3426 {
3427 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
3428 .is_registered = 0,
3429 .alg.aead = {
3430 .base = {
3431 .cra_name = "rfc4309(ccm(aes))",
3432 .cra_driver_name = "rfc4309-ccm-aes-chcr",
3433 .cra_blocksize = 1,
3434 .cra_priority = CHCR_AEAD_PRIORITY + 1,
3435 .cra_ctxsize = sizeof(struct chcr_context) +
3436 sizeof(struct chcr_aead_ctx),
3437
3438 },
3439 .ivsize = 8,
3440 .maxauthsize = GHASH_DIGEST_SIZE,
3441 .setkey = chcr_aead_rfc4309_setkey,
3442 .setauthsize = chcr_4106_4309_setauthsize,
3443 }
3444 },
3445 {
3446 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3447 .is_registered = 0,
3448 .alg.aead = {
3449 .base = {
3450 .cra_name = "authenc(hmac(sha1),cbc(aes))",
3451 .cra_driver_name =
3452 "authenc-hmac-sha1-cbc-aes-chcr",
3453 .cra_blocksize = AES_BLOCK_SIZE,
3454 .cra_priority = CHCR_AEAD_PRIORITY,
3455 .cra_ctxsize = sizeof(struct chcr_context) +
3456 sizeof(struct chcr_aead_ctx) +
3457 sizeof(struct chcr_authenc_ctx),
3458
3459 },
3460 .ivsize = AES_BLOCK_SIZE,
3461 .maxauthsize = SHA1_DIGEST_SIZE,
3462 .setkey = chcr_authenc_setkey,
3463 .setauthsize = chcr_authenc_setauthsize,
3464 }
3465 },
3466 {
3467 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3468 .is_registered = 0,
3469 .alg.aead = {
3470 .base = {
3471
3472 .cra_name = "authenc(hmac(sha256),cbc(aes))",
3473 .cra_driver_name =
3474 "authenc-hmac-sha256-cbc-aes-chcr",
3475 .cra_blocksize = AES_BLOCK_SIZE,
3476 .cra_priority = CHCR_AEAD_PRIORITY,
3477 .cra_ctxsize = sizeof(struct chcr_context) +
3478 sizeof(struct chcr_aead_ctx) +
3479 sizeof(struct chcr_authenc_ctx),
3480
3481 },
3482 .ivsize = AES_BLOCK_SIZE,
3483 .maxauthsize = SHA256_DIGEST_SIZE,
3484 .setkey = chcr_authenc_setkey,
3485 .setauthsize = chcr_authenc_setauthsize,
3486 }
3487 },
3488 {
3489 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3490 .is_registered = 0,
3491 .alg.aead = {
3492 .base = {
3493 .cra_name = "authenc(hmac(sha224),cbc(aes))",
3494 .cra_driver_name =
3495 "authenc-hmac-sha224-cbc-aes-chcr",
3496 .cra_blocksize = AES_BLOCK_SIZE,
3497 .cra_priority = CHCR_AEAD_PRIORITY,
3498 .cra_ctxsize = sizeof(struct chcr_context) +
3499 sizeof(struct chcr_aead_ctx) +
3500 sizeof(struct chcr_authenc_ctx),
3501 },
3502 .ivsize = AES_BLOCK_SIZE,
3503 .maxauthsize = SHA224_DIGEST_SIZE,
3504 .setkey = chcr_authenc_setkey,
3505 .setauthsize = chcr_authenc_setauthsize,
3506 }
3507 },
3508 {
3509 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3510 .is_registered = 0,
3511 .alg.aead = {
3512 .base = {
3513 .cra_name = "authenc(hmac(sha384),cbc(aes))",
3514 .cra_driver_name =
3515 "authenc-hmac-sha384-cbc-aes-chcr",
3516 .cra_blocksize = AES_BLOCK_SIZE,
3517 .cra_priority = CHCR_AEAD_PRIORITY,
3518 .cra_ctxsize = sizeof(struct chcr_context) +
3519 sizeof(struct chcr_aead_ctx) +
3520 sizeof(struct chcr_authenc_ctx),
3521
3522 },
3523 .ivsize = AES_BLOCK_SIZE,
3524 .maxauthsize = SHA384_DIGEST_SIZE,
3525 .setkey = chcr_authenc_setkey,
3526 .setauthsize = chcr_authenc_setauthsize,
3527 }
3528 },
3529 {
3530 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_AUTHENC,
3531 .is_registered = 0,
3532 .alg.aead = {
3533 .base = {
3534 .cra_name = "authenc(hmac(sha512),cbc(aes))",
3535 .cra_driver_name =
3536 "authenc-hmac-sha512-cbc-aes-chcr",
3537 .cra_blocksize = AES_BLOCK_SIZE,
3538 .cra_priority = CHCR_AEAD_PRIORITY,
3539 .cra_ctxsize = sizeof(struct chcr_context) +
3540 sizeof(struct chcr_aead_ctx) +
3541 sizeof(struct chcr_authenc_ctx),
3542
3543 },
3544 .ivsize = AES_BLOCK_SIZE,
3545 .maxauthsize = SHA512_DIGEST_SIZE,
3546 .setkey = chcr_authenc_setkey,
3547 .setauthsize = chcr_authenc_setauthsize,
3548 }
3549 },
3550 {
3551 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_NULL,
3552 .is_registered = 0,
3553 .alg.aead = {
3554 .base = {
3555 .cra_name = "authenc(digest_null,cbc(aes))",
3556 .cra_driver_name =
3557 "authenc-digest_null-cbc-aes-chcr",
3558 .cra_blocksize = AES_BLOCK_SIZE,
3559 .cra_priority = CHCR_AEAD_PRIORITY,
3560 .cra_ctxsize = sizeof(struct chcr_context) +
3561 sizeof(struct chcr_aead_ctx) +
3562 sizeof(struct chcr_authenc_ctx),
3563
3564 },
3565 .ivsize = AES_BLOCK_SIZE,
3566 .maxauthsize = 0,
3567 .setkey = chcr_aead_digest_null_setkey,
3568 .setauthsize = chcr_authenc_null_setauthsize,
3569 }
3570 },
3571 };
3572
3573 /*
3574 * chcr_unregister_alg - Deregister crypto algorithms with
3575 * kernel framework.
3576 */
3577 static int chcr_unregister_alg(void)
3578 {
3579 int i;
3580
3581 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3582 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
3583 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3584 if (driver_algs[i].is_registered)
3585 crypto_unregister_alg(
3586 &driver_algs[i].alg.crypto);
3587 break;
3588 case CRYPTO_ALG_TYPE_AEAD:
3589 if (driver_algs[i].is_registered)
3590 crypto_unregister_aead(
3591 &driver_algs[i].alg.aead);
3592 break;
3593 case CRYPTO_ALG_TYPE_AHASH:
3594 if (driver_algs[i].is_registered)
3595 crypto_unregister_ahash(
3596 &driver_algs[i].alg.hash);
3597 break;
3598 }
3599 driver_algs[i].is_registered = 0;
3600 }
3601 return 0;
3602 }
3603
3604 #define SZ_AHASH_CTX sizeof(struct chcr_context)
3605 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
3606 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
3607 #define AHASH_CRA_FLAGS (CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC)
3608
3609 /*
3610 * chcr_register_alg - Register crypto algorithms with kernel framework.
3611 */
3612 static int chcr_register_alg(void)
3613 {
3614 struct crypto_alg ai;
3615 struct ahash_alg *a_hash;
3616 int err = 0, i;
3617 char *name = NULL;
3618
3619 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3620 if (driver_algs[i].is_registered)
3621 continue;
3622 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
3623 case CRYPTO_ALG_TYPE_ABLKCIPHER:
3624 driver_algs[i].alg.crypto.cra_priority =
3625 CHCR_CRA_PRIORITY;
3626 driver_algs[i].alg.crypto.cra_module = THIS_MODULE;
3627 driver_algs[i].alg.crypto.cra_flags =
3628 CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC |
3629 CRYPTO_ALG_NEED_FALLBACK;
3630 driver_algs[i].alg.crypto.cra_ctxsize =
3631 sizeof(struct chcr_context) +
3632 sizeof(struct ablk_ctx);
3633 driver_algs[i].alg.crypto.cra_alignmask = 0;
3634 driver_algs[i].alg.crypto.cra_type =
3635 &crypto_ablkcipher_type;
3636 err = crypto_register_alg(&driver_algs[i].alg.crypto);
3637 name = driver_algs[i].alg.crypto.cra_driver_name;
3638 break;
3639 case CRYPTO_ALG_TYPE_AEAD:
3640 driver_algs[i].alg.aead.base.cra_flags =
3641 CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC |
3642 CRYPTO_ALG_NEED_FALLBACK;
3643 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
3644 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
3645 driver_algs[i].alg.aead.init = chcr_aead_cra_init;
3646 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
3647 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
3648 err = crypto_register_aead(&driver_algs[i].alg.aead);
3649 name = driver_algs[i].alg.aead.base.cra_driver_name;
3650 break;
3651 case CRYPTO_ALG_TYPE_AHASH:
3652 a_hash = &driver_algs[i].alg.hash;
3653 a_hash->update = chcr_ahash_update;
3654 a_hash->final = chcr_ahash_final;
3655 a_hash->finup = chcr_ahash_finup;
3656 a_hash->digest = chcr_ahash_digest;
3657 a_hash->export = chcr_ahash_export;
3658 a_hash->import = chcr_ahash_import;
3659 a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
3660 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
3661 a_hash->halg.base.cra_module = THIS_MODULE;
3662 a_hash->halg.base.cra_flags = AHASH_CRA_FLAGS;
3663 a_hash->halg.base.cra_alignmask = 0;
3664 a_hash->halg.base.cra_exit = NULL;
3665 a_hash->halg.base.cra_type = &crypto_ahash_type;
3666
3667 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
3668 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
3669 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
3670 a_hash->init = chcr_hmac_init;
3671 a_hash->setkey = chcr_ahash_setkey;
3672 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
3673 } else {
3674 a_hash->init = chcr_sha_init;
3675 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
3676 a_hash->halg.base.cra_init = chcr_sha_cra_init;
3677 }
3678 err = crypto_register_ahash(&driver_algs[i].alg.hash);
3679 ai = driver_algs[i].alg.hash.halg.base;
3680 name = ai.cra_driver_name;
3681 break;
3682 }
3683 if (err) {
3684 pr_err("chcr : %s : Algorithm registration failed\n",
3685 name);
3686 goto register_err;
3687 } else {
3688 driver_algs[i].is_registered = 1;
3689 }
3690 }
3691 return 0;
3692
3693 register_err:
3694 chcr_unregister_alg();
3695 return err;
3696 }
3697
3698 /*
3699 * start_crypto - Register the crypto algorithms.
3700 * This should called once when the first device comesup. After this
3701 * kernel will start calling driver APIs for crypto operations.
3702 */
3703 int start_crypto(void)
3704 {
3705 return chcr_register_alg();
3706 }
3707
3708 /*
3709 * stop_crypto - Deregister all the crypto algorithms with kernel.
3710 * This should be called once when the last device goes down. After this
3711 * kernel will not call the driver API for crypto operations.
3712 */
3713 int stop_crypto(void)
3714 {
3715 chcr_unregister_alg();
3716 return 0;
3717 }
Ubuntu Artful kernel mirror