]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blame - drivers/crypto/chelsio/chcr_algo.c
projects / mirror_ubuntu-hirsute-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Crypto/chcr: fix gcm-aes and rfc4106-gcm failed tests
[mirror_ubuntu-hirsute-kernel.git] / drivers / crypto / chelsio / chcr_algo.c
CommitLineData
324429d7
HS		 1/*
2 * This file is part of the Chelsio T6 Crypto driver for Linux.
3 *
4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 *
34 * Written and Maintained by:
35 * Manoj Malviya (manojmalviya@chelsio.com)
36 * Atul Gupta (atul.gupta@chelsio.com)
37 * Jitendra Lulla (jlulla@chelsio.com)
38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
39 * Harsh Jain (harsh@chelsio.com)
40 */
41
42#define pr_fmt(fmt) "chcr:" fmt
43
44#include <linux/kernel.h>
45#include <linux/module.h>
46#include <linux/crypto.h>
47#include <linux/cryptohash.h>
48#include <linux/skbuff.h>
49#include <linux/rtnetlink.h>
50#include <linux/highmem.h>
51#include <linux/scatterlist.h>
52
53#include <crypto/aes.h>
54#include <crypto/algapi.h>
55#include <crypto/hash.h>
8f6acb7f 56#include <crypto/gcm.h>
324429d7 57#include <crypto/sha.h>
2debd332 58#include <crypto/authenc.h>
b8fd1f41
HJ		 59#include <crypto/ctr.h>
60#include <crypto/gf128mul.h>
2debd332
HJ		 61#include <crypto/internal/aead.h>
62#include <crypto/null.h>
63#include <crypto/internal/skcipher.h>
64#include <crypto/aead.h>
65#include <crypto/scatterwalk.h>
324429d7
HS		 66#include <crypto/internal/hash.h>
67
68#include "t4fw_api.h"
69#include "t4_msg.h"
70#include "chcr_core.h"
71#include "chcr_algo.h"
72#include "chcr_crypto.h"
73
2f47d580
HJ		 74#define IV AES_BLOCK_SIZE
75
8579e076
CIK		 76static unsigned int sgl_ent_len[] = {
77 0, 0, 16, 24, 40, 48, 64, 72, 88,
78 96, 112, 120, 136, 144, 160, 168, 184,
79 192, 208, 216, 232, 240, 256, 264, 280,
80 288, 304, 312, 328, 336, 352, 360, 376
81};
6dad4e8a 82
8579e076
CIK		 83static unsigned int dsgl_ent_len[] = {
84 0, 32, 32, 48, 48, 64, 64, 80, 80,
85 112, 112, 128, 128, 144, 144, 160, 160,
86 192, 192, 208, 208, 224, 224, 240, 240,
87 272, 272, 288, 288, 304, 304, 320, 320
88};
6dad4e8a
AG		 89
90static u32 round_constant[11] = {
91 0x01000000, 0x02000000, 0x04000000, 0x08000000,
92 0x10000000, 0x20000000, 0x40000000, 0x80000000,
93 0x1B000000, 0x36000000, 0x6C000000
94};
95
7cea6d3e 96static int chcr_handle_cipher_resp(struct skcipher_request *req,
6dad4e8a
AG		 97 unsigned char *input, int err);
98
2debd332
HJ		 99static inline struct chcr_aead_ctx *AEAD_CTX(struct chcr_context *ctx)
100{
101 return ctx->crypto_ctx->aeadctx;
102}
103
324429d7
HS		 104static inline struct ablk_ctx *ABLK_CTX(struct chcr_context *ctx)
105{
106 return ctx->crypto_ctx->ablkctx;
107}
108
109static inline struct hmac_ctx *HMAC_CTX(struct chcr_context *ctx)
110{
111 return ctx->crypto_ctx->hmacctx;
112}
113
2debd332
HJ		 114static inline struct chcr_gcm_ctx *GCM_CTX(struct chcr_aead_ctx *gctx)
115{
116 return gctx->ctx->gcm;
117}
118
119static inline struct chcr_authenc_ctx *AUTHENC_CTX(struct chcr_aead_ctx *gctx)
120{
121 return gctx->ctx->authenc;
122}
123
324429d7
HS		 124static inline struct uld_ctx *ULD_CTX(struct chcr_context *ctx)
125{
fef4912b 126 return container_of(ctx->dev, struct uld_ctx, dev);
324429d7
HS		 127}
128
129static inline int is_ofld_imm(const struct sk_buff *skb)
130{
2f47d580 131 return (skb->len <= SGE_MAX_WR_LEN);
324429d7
HS		 132}
133
5110e655
HJ		 134static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx *reqctx)
135{
136 memset(&reqctx->hctx_wr, 0, sizeof(struct chcr_hctx_per_wr));
137}
138
2f47d580
HJ		 139static int sg_nents_xlen(struct scatterlist *sg, unsigned int reqlen,
140 unsigned int entlen,
141 unsigned int skip)
2956f36c
HJ		 142{
143 int nents = 0;
144 unsigned int less;
2f47d580 145 unsigned int skip_len = 0;
2956f36c 146
2f47d580
HJ		 147 while (sg && skip) {
148 if (sg_dma_len(sg) <= skip) {
149 skip -= sg_dma_len(sg);
150 skip_len = 0;
151 sg = sg_next(sg);
152 } else {
153 skip_len = skip;
154 skip = 0;
155 }
2956f36c
HJ		 156 }
157
2f47d580
HJ		 158 while (sg && reqlen) {
159 less = min(reqlen, sg_dma_len(sg) - skip_len);
160 nents += DIV_ROUND_UP(less, entlen);
161 reqlen -= less;
162 skip_len = 0;
163 sg = sg_next(sg);
164 }
2956f36c
HJ		 165 return nents;
166}
167
6dad4e8a 168static inline int get_aead_subtype(struct crypto_aead *aead)
2f47d580 169{
6dad4e8a
AG		 170 struct aead_alg *alg = crypto_aead_alg(aead);
171 struct chcr_alg_template *chcr_crypto_alg =
172 container_of(alg, struct chcr_alg_template, alg.aead);
173 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
2f47d580 174}
2f47d580 175
6dad4e8a 176void chcr_verify_tag(struct aead_request *req, u8 *input, int *err)
2debd332
HJ		 177{
178 u8 temp[SHA512_DIGEST_SIZE];
179 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
180 int authsize = crypto_aead_authsize(tfm);
181 struct cpl_fw6_pld *fw6_pld;
182 int cmp = 0;
183
184 fw6_pld = (struct cpl_fw6_pld *)input;
185 if ((get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) ||
186 (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM)) {
d600fc8a 187 cmp = crypto_memneq(&fw6_pld->data[2], (fw6_pld + 1), authsize);
2debd332
HJ		 188 } else {
189
190 sg_pcopy_to_buffer(req->src, sg_nents(req->src), temp,
191 authsize, req->assoclen +
192 req->cryptlen - authsize);
d600fc8a 193 cmp = crypto_memneq(temp, (fw6_pld + 1), authsize);
2debd332
HJ		 194 }
195 if (cmp)
196 *err = -EBADMSG;
197 else
198 *err = 0;
199}
200
fef4912b
HJ		 201static int chcr_inc_wrcount(struct chcr_dev *dev)
202{
fef4912b 203 if (dev->state == CHCR_DETACH)
33ddc108
AG		 204 return 1;
205 atomic_inc(&dev->inflight);
206 return 0;
fef4912b
HJ		 207}
208
209static inline void chcr_dec_wrcount(struct chcr_dev *dev)
210{
211 atomic_dec(&dev->inflight);
212}
213
f31ba0f9 214static inline int chcr_handle_aead_resp(struct aead_request *req,
6dad4e8a
AG		 215 unsigned char *input,
216 int err)
217{
218 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
fef4912b
HJ		 219 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
220 struct chcr_dev *dev = a_ctx(tfm)->dev;
6dad4e8a 221
4262c98a 222 chcr_aead_common_exit(req);
6dad4e8a
AG		 223 if (reqctx->verify == VERIFY_SW) {
224 chcr_verify_tag(req, input, &err);
225 reqctx->verify = VERIFY_HW;
226 }
fef4912b 227 chcr_dec_wrcount(dev);
6dad4e8a 228 req->base.complete(&req->base, err);
f31ba0f9
HJ		 229
230 return err;
6dad4e8a
AG		 231}
232
2f47d580 233static void get_aes_decrypt_key(unsigned char *dec_key,
39f91a34
HJ		 234 const unsigned char *key,
235 unsigned int keylength)
236{
237 u32 temp;
238 u32 w_ring[MAX_NK];
239 int i, j, k;
240 u8 nr, nk;
241
242 switch (keylength) {
243 case AES_KEYLENGTH_128BIT:
244 nk = KEYLENGTH_4BYTES;
245 nr = NUMBER_OF_ROUNDS_10;
246 break;
247 case AES_KEYLENGTH_192BIT:
248 nk = KEYLENGTH_6BYTES;
249 nr = NUMBER_OF_ROUNDS_12;
250 break;
251 case AES_KEYLENGTH_256BIT:
252 nk = KEYLENGTH_8BYTES;
253 nr = NUMBER_OF_ROUNDS_14;
254 break;
255 default:
256 return;
257 }
258 for (i = 0; i < nk; i++)
259 w_ring[i] = be32_to_cpu(*(u32 *)&key[4 * i]);
260
261 i = 0;
262 temp = w_ring[nk - 1];
263 while (i + nk < (nr + 1) * 4) {
264 if (!(i % nk)) {
265 /* RotWord(temp) */
266 temp = (temp << 8) | (temp >> 24);
267 temp = aes_ks_subword(temp);
268 temp ^= round_constant[i / nk];
269 } else if (nk == 8 && (i % 4 == 0)) {
270 temp = aes_ks_subword(temp);
271 }
272 w_ring[i % nk] ^= temp;
273 temp = w_ring[i % nk];
274 i++;
275 }
276 i--;
277 for (k = 0, j = i % nk; k < nk; k++) {
278 *((u32 *)dec_key + k) = htonl(w_ring[j]);
279 j--;
280 if (j < 0)
281 j += nk;
282 }
283}
284
e7922729 285static struct crypto_shash *chcr_alloc_shash(unsigned int ds)
324429d7 286{
ec1bca94 287 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
324429d7
HS		 288
289 switch (ds) {
290 case SHA1_DIGEST_SIZE:
e7922729 291 base_hash = crypto_alloc_shash("sha1", 0, 0);
324429d7
HS		 292 break;
293 case SHA224_DIGEST_SIZE:
e7922729 294 base_hash = crypto_alloc_shash("sha224", 0, 0);
324429d7
HS		 295 break;
296 case SHA256_DIGEST_SIZE:
e7922729 297 base_hash = crypto_alloc_shash("sha256", 0, 0);
324429d7
HS		 298 break;
299 case SHA384_DIGEST_SIZE:
e7922729 300 base_hash = crypto_alloc_shash("sha384", 0, 0);
324429d7
HS		 301 break;
302 case SHA512_DIGEST_SIZE:
e7922729 303 base_hash = crypto_alloc_shash("sha512", 0, 0);
324429d7
HS		 304 break;
305 }
324429d7 306
e7922729 307 return base_hash;
324429d7
HS		 308}
309
310static int chcr_compute_partial_hash(struct shash_desc *desc,
311 char *iopad, char *result_hash,
312 int digest_size)
313{
314 struct sha1_state sha1_st;
315 struct sha256_state sha256_st;
316 struct sha512_state sha512_st;
317 int error;
318
319 if (digest_size == SHA1_DIGEST_SIZE) {
320 error = crypto_shash_init(desc) ?:
321 crypto_shash_update(desc, iopad, SHA1_BLOCK_SIZE) ?:
322 crypto_shash_export(desc, (void *)&sha1_st);
323 memcpy(result_hash, sha1_st.state, SHA1_DIGEST_SIZE);
324 } else if (digest_size == SHA224_DIGEST_SIZE) {
325 error = crypto_shash_init(desc) ?:
326 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
327 crypto_shash_export(desc, (void *)&sha256_st);
328 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
329
330 } else if (digest_size == SHA256_DIGEST_SIZE) {
331 error = crypto_shash_init(desc) ?:
332 crypto_shash_update(desc, iopad, SHA256_BLOCK_SIZE) ?:
333 crypto_shash_export(desc, (void *)&sha256_st);
334 memcpy(result_hash, sha256_st.state, SHA256_DIGEST_SIZE);
335
336 } else if (digest_size == SHA384_DIGEST_SIZE) {
337 error = crypto_shash_init(desc) ?:
338 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
339 crypto_shash_export(desc, (void *)&sha512_st);
340 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
341
342 } else if (digest_size == SHA512_DIGEST_SIZE) {
343 error = crypto_shash_init(desc) ?:
344 crypto_shash_update(desc, iopad, SHA512_BLOCK_SIZE) ?:
345 crypto_shash_export(desc, (void *)&sha512_st);
346 memcpy(result_hash, sha512_st.state, SHA512_DIGEST_SIZE);
347 } else {
348 error = -EINVAL;
349 pr_err("Unknown digest size %d\n", digest_size);
350 }
351 return error;
352}
353
354static void chcr_change_order(char *buf, int ds)
355{
356 int i;
357
358 if (ds == SHA512_DIGEST_SIZE) {
359 for (i = 0; i < (ds / sizeof(u64)); i++)
360 *((__be64 *)buf + i) =
361 cpu_to_be64(*((u64 *)buf + i));
362 } else {
363 for (i = 0; i < (ds / sizeof(u32)); i++)
364 *((__be32 *)buf + i) =
365 cpu_to_be32(*((u32 *)buf + i));
366 }
367}
368
369static inline int is_hmac(struct crypto_tfm *tfm)
370{
371 struct crypto_alg *alg = tfm->__crt_alg;
372 struct chcr_alg_template *chcr_crypto_alg =
373 container_of(__crypto_ahash_alg(alg), struct chcr_alg_template,
374 alg.hash);
5c86a8ff 375 if (chcr_crypto_alg->type == CRYPTO_ALG_TYPE_HMAC)
324429d7
HS		 376 return 1;
377 return 0;
378}
379
2f47d580
HJ		 380static inline void dsgl_walk_init(struct dsgl_walk *walk,
381 struct cpl_rx_phys_dsgl *dsgl)
324429d7 382{
2f47d580
HJ		 383 walk->dsgl = dsgl;
384 walk->nents = 0;
385 walk->to = (struct phys_sge_pairs *)(dsgl + 1);
386}
387
add92a81
HJ		 388static inline void dsgl_walk_end(struct dsgl_walk *walk, unsigned short qid,
389 int pci_chan_id)
2f47d580
HJ		 390{
391 struct cpl_rx_phys_dsgl *phys_cpl;
392
393 phys_cpl = walk->dsgl;
324429d7
HS		 394
395 phys_cpl->op_to_tid = htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL)
396 | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
2f47d580
HJ		 397 phys_cpl->pcirlxorder_to_noofsgentr =
398 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
399 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
400 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
401 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
402 CPL_RX_PHYS_DSGL_DCAID_V(0) |
403 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk->nents));
404 phys_cpl->rss_hdr_int.opcode = CPL_RX_PHYS_ADDR;
405 phys_cpl->rss_hdr_int.qid = htons(qid);
406 phys_cpl->rss_hdr_int.hash_val = 0;
add92a81 407 phys_cpl->rss_hdr_int.channel = pci_chan_id;
2f47d580
HJ		 408}
409
410static inline void dsgl_walk_add_page(struct dsgl_walk *walk,
411 size_t size,
c4f6d44d 412 dma_addr_t addr)
2f47d580
HJ		 413{
414 int j;
415
416 if (!size)
417 return;
418 j = walk->nents;
419 walk->to->len[j % 8] = htons(size);
c4f6d44d 420 walk->to->addr[j % 8] = cpu_to_be64(addr);
2f47d580
HJ		 421 j++;
422 if ((j % 8) == 0)
423 walk->to++;
424 walk->nents = j;
425}
426
427static void dsgl_walk_add_sg(struct dsgl_walk *walk,
428 struct scatterlist *sg,
429 unsigned int slen,
430 unsigned int skip)
431{
432 int skip_len = 0;
433 unsigned int left_size = slen, len = 0;
434 unsigned int j = walk->nents;
435 int offset, ent_len;
436
437 if (!slen)
438 return;
439 while (sg && skip) {
440 if (sg_dma_len(sg) <= skip) {
441 skip -= sg_dma_len(sg);
442 skip_len = 0;
443 sg = sg_next(sg);
444 } else {
445 skip_len = skip;
446 skip = 0;
447 }
448 }
449
2956f36c 450 while (left_size && sg) {
2f47d580 451 len = min_t(u32, left_size, sg_dma_len(sg) - skip_len);
2956f36c
HJ		 452 offset = 0;
453 while (len) {
2f47d580
HJ		 454 ent_len = min_t(u32, len, CHCR_DST_SG_SIZE);
455 walk->to->len[j % 8] = htons(ent_len);
456 walk->to->addr[j % 8] = cpu_to_be64(sg_dma_address(sg) +
457 offset + skip_len);
2956f36c
HJ		 458 offset += ent_len;
459 len -= ent_len;
460 j++;
461 if ((j % 8) == 0)
2f47d580 462 walk->to++;
2956f36c 463 }
2f47d580
HJ		 464 walk->last_sg = sg;
465 walk->last_sg_len = min_t(u32, left_size, sg_dma_len(sg) -
466 skip_len) + skip_len;
467 left_size -= min_t(u32, left_size, sg_dma_len(sg) - skip_len);
468 skip_len = 0;
2956f36c
HJ		 469 sg = sg_next(sg);
470 }
2f47d580
HJ		 471 walk->nents = j;
472}
473
474static inline void ulptx_walk_init(struct ulptx_walk *walk,
475 struct ulptx_sgl *ulp)
476{
477 walk->sgl = ulp;
478 walk->nents = 0;
479 walk->pair_idx = 0;
480 walk->pair = ulp->sge;
481 walk->last_sg = NULL;
482 walk->last_sg_len = 0;
483}
484
485static inline void ulptx_walk_end(struct ulptx_walk *walk)
486{
487 walk->sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
488 ULPTX_NSGE_V(walk->nents));
489}
2956f36c 490
2f47d580
HJ		 491
492static inline void ulptx_walk_add_page(struct ulptx_walk *walk,
493 size_t size,
c4f6d44d 494 dma_addr_t addr)
2f47d580
HJ		 495{
496 if (!size)
497 return;
498
499 if (walk->nents == 0) {
500 walk->sgl->len0 = cpu_to_be32(size);
c4f6d44d 501 walk->sgl->addr0 = cpu_to_be64(addr);
2f47d580 502 } else {
c4f6d44d 503 walk->pair->addr[walk->pair_idx] = cpu_to_be64(addr);
2f47d580
HJ		 504 walk->pair->len[walk->pair_idx] = cpu_to_be32(size);
505 walk->pair_idx = !walk->pair_idx;
506 if (!walk->pair_idx)
507 walk->pair++;
508 }
509 walk->nents++;
324429d7
HS		 510}
511
2f47d580 512static void ulptx_walk_add_sg(struct ulptx_walk *walk,
adf1ca61 513 struct scatterlist *sg,
2f47d580
HJ		 514 unsigned int len,
515 unsigned int skip)
324429d7 516{
2f47d580
HJ		 517 int small;
518 int skip_len = 0;
519 unsigned int sgmin;
324429d7 520
2f47d580
HJ		 521 if (!len)
522 return;
2f47d580
HJ		 523 while (sg && skip) {
524 if (sg_dma_len(sg) <= skip) {
525 skip -= sg_dma_len(sg);
526 skip_len = 0;
527 sg = sg_next(sg);
528 } else {
529 skip_len = skip;
530 skip = 0;
531 }
532 }
8daa32b9
HJ		 533 WARN(!sg, "SG should not be null here\n");
534 if (sg && (walk->nents == 0)) {
2f47d580
HJ		 535 small = min_t(unsigned int, sg_dma_len(sg) - skip_len, len);
536 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
537 walk->sgl->len0 = cpu_to_be32(sgmin);
538 walk->sgl->addr0 = cpu_to_be64(sg_dma_address(sg) + skip_len);
539 walk->nents++;
540 len -= sgmin;
541 walk->last_sg = sg;
542 walk->last_sg_len = sgmin + skip_len;
543 skip_len += sgmin;
544 if (sg_dma_len(sg) == skip_len) {
545 sg = sg_next(sg);
546 skip_len = 0;
547 }
548 }
549
550 while (sg && len) {
551 small = min(sg_dma_len(sg) - skip_len, len);
552 sgmin = min_t(unsigned int, small, CHCR_SRC_SG_SIZE);
553 walk->pair->len[walk->pair_idx] = cpu_to_be32(sgmin);
554 walk->pair->addr[walk->pair_idx] =
555 cpu_to_be64(sg_dma_address(sg) + skip_len);
556 walk->pair_idx = !walk->pair_idx;
557 walk->nents++;
558 if (!walk->pair_idx)
559 walk->pair++;
560 len -= sgmin;
561 skip_len += sgmin;
562 walk->last_sg = sg;
563 walk->last_sg_len = skip_len;
564 if (sg_dma_len(sg) == skip_len) {
565 sg = sg_next(sg);
566 skip_len = 0;
567 }
324429d7 568 }
324429d7
HS		 569}
570
7cea6d3e 571static inline int get_cryptoalg_subtype(struct crypto_skcipher *tfm)
324429d7 572{
7cea6d3e 573 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
324429d7 574 struct chcr_alg_template *chcr_crypto_alg =
7cea6d3e 575 container_of(alg, struct chcr_alg_template, alg.skcipher);
324429d7
HS		 576
577 return chcr_crypto_alg->type & CRYPTO_ALG_SUB_TYPE_MASK;
578}
579
b8fd1f41
HJ		 580static int cxgb4_is_crypto_q_full(struct net_device *dev, unsigned int idx)
581{
582 struct adapter *adap = netdev2adap(dev);
583 struct sge_uld_txq_info *txq_info =
584 adap->sge.uld_txq_info[CXGB4_TX_CRYPTO];
585 struct sge_uld_txq *txq;
586 int ret = 0;
587
588 local_bh_disable();
589 txq = &txq_info->uldtxq[idx];
590 spin_lock(&txq->sendq.lock);
591 if (txq->full)
592 ret = -1;
593 spin_unlock(&txq->sendq.lock);
594 local_bh_enable();
595 return ret;
596}
597
324429d7
HS		 598static int generate_copy_rrkey(struct ablk_ctx *ablkctx,
599 struct _key_ctx *key_ctx)
600{
601 if (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) {
cc1b156d 602 memcpy(key_ctx->key, ablkctx->rrkey, ablkctx->enckey_len);
324429d7
HS		 603 } else {
604 memcpy(key_ctx->key,
605 ablkctx->key + (ablkctx->enckey_len >> 1),
606 ablkctx->enckey_len >> 1);
cc1b156d
HJ		 607 memcpy(key_ctx->key + (ablkctx->enckey_len >> 1),
608 ablkctx->rrkey, ablkctx->enckey_len >> 1);
324429d7
HS		 609 }
610 return 0;
611}
5110e655
HJ		 612
613static int chcr_hash_ent_in_wr(struct scatterlist *src,
614 unsigned int minsg,
615 unsigned int space,
616 unsigned int srcskip)
617{
618 int srclen = 0;
619 int srcsg = minsg;
620 int soffset = 0, sless;
621
622 if (sg_dma_len(src) == srcskip) {
623 src = sg_next(src);
624 srcskip = 0;
625 }
626 while (src && space > (sgl_ent_len[srcsg + 1])) {
627 sless = min_t(unsigned int, sg_dma_len(src) - soffset - srcskip,
628 CHCR_SRC_SG_SIZE);
629 srclen += sless;
630 soffset += sless;
631 srcsg++;
632 if (sg_dma_len(src) == (soffset + srcskip)) {
633 src = sg_next(src);
634 soffset = 0;
635 srcskip = 0;
636 }
637 }
638 return srclen;
639}
640
b8fd1f41
HJ		 641static int chcr_sg_ent_in_wr(struct scatterlist *src,
642 struct scatterlist *dst,
643 unsigned int minsg,
2f47d580
HJ		 644 unsigned int space,
645 unsigned int srcskip,
646 unsigned int dstskip)
b8fd1f41
HJ		 647{
648 int srclen = 0, dstlen = 0;
2f47d580 649 int srcsg = minsg, dstsg = minsg;
1d693cf6 650 int offset = 0, soffset = 0, less, sless = 0;
b8fd1f41 651
2f47d580
HJ		 652 if (sg_dma_len(src) == srcskip) {
653 src = sg_next(src);
654 srcskip = 0;
655 }
2f47d580
HJ		 656 if (sg_dma_len(dst) == dstskip) {
657 dst = sg_next(dst);
658 dstskip = 0;
659 }
660
661 while (src && dst &&
b8fd1f41 662 space > (sgl_ent_len[srcsg + 1] + dsgl_ent_len[dstsg])) {
1d693cf6
HJ		 663 sless = min_t(unsigned int, sg_dma_len(src) - srcskip - soffset,
664 CHCR_SRC_SG_SIZE);
665 srclen += sless;
b8fd1f41 666 srcsg++;
2956f36c 667 offset = 0;
b8fd1f41
HJ		 668 while (dst && ((dstsg + 1) <= MAX_DSGL_ENT) &&
669 space > (sgl_ent_len[srcsg] + dsgl_ent_len[dstsg + 1])) {
670 if (srclen <= dstlen)
671 break;
2f47d580 672 less = min_t(unsigned int, sg_dma_len(dst) - offset -
db6deea4 673 dstskip, CHCR_DST_SG_SIZE);
2956f36c
HJ		 674 dstlen += less;
675 offset += less;
1d693cf6 676 if ((offset + dstskip) == sg_dma_len(dst)) {
2956f36c
HJ		 677 dst = sg_next(dst);
678 offset = 0;
679 }
b8fd1f41 680 dstsg++;
2f47d580 681 dstskip = 0;
b8fd1f41 682 }
1d693cf6
HJ		 683 soffset += sless;
684 if ((soffset + srcskip) == sg_dma_len(src)) {
685 src = sg_next(src);
686 srcskip = 0;
687 soffset = 0;
688 }
689
b8fd1f41 690 }
b8fd1f41
HJ		 691 return min(srclen, dstlen);
692}
693
28874f26 694static int chcr_cipher_fallback(struct crypto_sync_skcipher *cipher,
b8fd1f41
HJ		 695 u32 flags,
696 struct scatterlist *src,
697 struct scatterlist *dst,
698 unsigned int nbytes,
699 u8 *iv,
700 unsigned short op_type)
701{
702 int err;
703
28874f26 704 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, cipher);
6faa0f57 705
28874f26 706 skcipher_request_set_sync_tfm(subreq, cipher);
b8fd1f41
HJ		 707 skcipher_request_set_callback(subreq, flags, NULL, NULL);
708 skcipher_request_set_crypt(subreq, src, dst,
709 nbytes, iv);
710
711 err = op_type ? crypto_skcipher_decrypt(subreq) :
712 crypto_skcipher_encrypt(subreq);
713 skcipher_request_zero(subreq);
714
715 return err;
324429d7 716
b8fd1f41 717}
567be3a5
AS		 718
719static inline int get_qidxs(struct crypto_async_request *req,
720 unsigned int *txqidx, unsigned int *rxqidx)
721{
722 struct crypto_tfm *tfm = req->tfm;
723 int ret = 0;
724
725 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
726 case CRYPTO_ALG_TYPE_AEAD:
727 {
728 struct aead_request *aead_req =
729 container_of(req, struct aead_request, base);
730 struct chcr_aead_reqctx *reqctx = aead_request_ctx(aead_req);
731 *txqidx = reqctx->txqidx;
732 *rxqidx = reqctx->rxqidx;
733 break;
734 }
735 case CRYPTO_ALG_TYPE_SKCIPHER:
736 {
737 struct skcipher_request *sk_req =
738 container_of(req, struct skcipher_request, base);
739 struct chcr_skcipher_req_ctx *reqctx =
740 skcipher_request_ctx(sk_req);
741 *txqidx = reqctx->txqidx;
742 *rxqidx = reqctx->rxqidx;
743 break;
744 }
745 case CRYPTO_ALG_TYPE_AHASH:
746 {
747 struct ahash_request *ahash_req =
748 container_of(req, struct ahash_request, base);
749 struct chcr_ahash_req_ctx *reqctx =
750 ahash_request_ctx(ahash_req);
751 *txqidx = reqctx->txqidx;
752 *rxqidx = reqctx->rxqidx;
753 break;
754 }
755 default:
756 ret = -EINVAL;
757 /* should never get here */
758 BUG();
759 break;
760 }
761 return ret;
762}
763
324429d7 764static inline void create_wreq(struct chcr_context *ctx,
358961d1 765 struct chcr_wr *chcr_req,
2f47d580
HJ		 766 struct crypto_async_request *req,
767 unsigned int imm,
570265bf 768 int hash_sz,
2f47d580 769 unsigned int len16,
2512a624
HJ		 770 unsigned int sc_len,
771 unsigned int lcb)
324429d7
HS		 772{
773 struct uld_ctx *u_ctx = ULD_CTX(ctx);
567be3a5
AS		 774 unsigned int tx_channel_id, rx_channel_id;
775 unsigned int txqidx = 0, rxqidx = 0;
776 unsigned int qid, fid;
777
778 get_qidxs(req, &txqidx, &rxqidx);
779 qid = u_ctx->lldi.rxq_ids[rxqidx];
780 fid = u_ctx->lldi.rxq_ids[0];
781 tx_channel_id = txqidx / ctx->txq_perchan;
782 rx_channel_id = rxqidx / ctx->rxq_perchan;
324429d7 783
324429d7 784
570265bf 785 chcr_req->wreq.op_to_cctx_size = FILL_WR_OP_CCTX_SIZE;
358961d1 786 chcr_req->wreq.pld_size_hash_size =
570265bf 787 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz));
358961d1 788 chcr_req->wreq.len16_pkd =
2f47d580 789 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16, 16)));
358961d1 790 chcr_req->wreq.cookie = cpu_to_be64((uintptr_t)req);
567be3a5
AS		 791 chcr_req->wreq.rx_chid_to_rx_q_id = FILL_WR_RX_Q_ID(rx_channel_id, qid,
792 !!lcb, txqidx);
324429d7 793
567be3a5 794 chcr_req->ulptx.cmd_dest = FILL_ULPTX_CMD_DEST(tx_channel_id, fid);
2f47d580 795 chcr_req->ulptx.len = htonl((DIV_ROUND_UP(len16, 16) -
567be3a5 796 ((sizeof(chcr_req->wreq)) >> 4)));
2f47d580 797 chcr_req->sc_imm.cmd_more = FILL_CMD_MORE(!imm);
358961d1 798 chcr_req->sc_imm.len = cpu_to_be32(sizeof(struct cpl_tx_sec_pdu) +
2f47d580 799 sizeof(chcr_req->key_ctx) + sc_len);
324429d7
HS		 800}
801
802/**
803 * create_cipher_wr - form the WR for cipher operations
804 * @req: cipher req.
805 * @ctx: crypto driver context of the request.
806 * @qid: ingress qid where response of this WR should be received.
807 * @op_type: encryption or decryption
808 */
b8fd1f41 809static struct sk_buff *create_cipher_wr(struct cipher_wr_param *wrparam)
324429d7 810{
7cea6d3e 811 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req);
567be3a5
AS		 812 struct chcr_context *ctx = c_ctx(tfm);
813 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
324429d7 814 struct sk_buff *skb = NULL;
358961d1 815 struct chcr_wr *chcr_req;
324429d7 816 struct cpl_rx_phys_dsgl *phys_cpl;
2f47d580 817 struct ulptx_sgl *ulptx;
7cea6d3e
AB		 818 struct chcr_skcipher_req_ctx *reqctx =
819 skcipher_request_ctx(wrparam->req);
2f47d580 820 unsigned int temp = 0, transhdr_len, dst_size;
b8fd1f41 821 int error;
2956f36c 822 int nents;
2f47d580 823 unsigned int kctx_len;
b8fd1f41
HJ		 824 gfp_t flags = wrparam->req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
825 GFP_KERNEL : GFP_ATOMIC;
567be3a5
AS		 826 struct adapter *adap = padap(ctx->dev);
827 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
324429d7 828
2f47d580
HJ		 829 nents = sg_nents_xlen(reqctx->dstsg, wrparam->bytes, CHCR_DST_SG_SIZE,
830 reqctx->dst_ofst);
335bcc4a 831 dst_size = get_space_for_phys_dsgl(nents);
125d01ca 832 kctx_len = roundup(ablkctx->enckey_len, 16);
2f47d580
HJ		 833 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
834 nents = sg_nents_xlen(reqctx->srcsg, wrparam->bytes,
835 CHCR_SRC_SG_SIZE, reqctx->src_ofst);
335bcc4a
HJ		 836 temp = reqctx->imm ? roundup(wrparam->bytes, 16) :
837 (sgl_len(nents) * 8);
2f47d580 838 transhdr_len += temp;
125d01ca 839 transhdr_len = roundup(transhdr_len, 16);
2f47d580 840 skb = alloc_skb(SGE_MAX_WR_LEN, flags);
b8fd1f41
HJ		 841 if (!skb) {
842 error = -ENOMEM;
843 goto err;
844 }
de77b966 845 chcr_req = __skb_put_zero(skb, transhdr_len);
358961d1 846 chcr_req->sec_cpl.op_ivinsrtofst =
567be3a5 847 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
358961d1 848
2f47d580 849 chcr_req->sec_cpl.pldlen = htonl(IV + wrparam->bytes);
358961d1 850 chcr_req->sec_cpl.aadstart_cipherstop_hi =
2f47d580 851 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV + 1, 0);
358961d1
HJ		 852
853 chcr_req->sec_cpl.cipherstop_lo_authinsert =
854 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
b8fd1f41 855 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, 0,
324429d7 856 ablkctx->ciph_mode,
2f47d580 857 0, 0, IV >> 1);
358961d1 858 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
335bcc4a 859 0, 1, dst_size);
324429d7 860
358961d1 861 chcr_req->key_ctx.ctx_hdr = ablkctx->key_ctx_hdr;
b8fd1f41 862 if ((reqctx->op == CHCR_DECRYPT_OP) &&
7cea6d3e 863 (!(get_cryptoalg_subtype(tfm) ==
b8fd1f41 864 CRYPTO_ALG_SUB_TYPE_CTR)) &&
7cea6d3e 865 (!(get_cryptoalg_subtype(tfm) ==
b8fd1f41 866 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686))) {
358961d1 867 generate_copy_rrkey(ablkctx, &chcr_req->key_ctx);
324429d7 868 } else {
b8fd1f41
HJ		 869 if ((ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC) ||
870 (ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CTR)) {
358961d1
HJ		 871 memcpy(chcr_req->key_ctx.key, ablkctx->key,
872 ablkctx->enckey_len);
324429d7 873 } else {
358961d1 874 memcpy(chcr_req->key_ctx.key, ablkctx->key +
324429d7
HS		 875 (ablkctx->enckey_len >> 1),
876 ablkctx->enckey_len >> 1);
358961d1 877 memcpy(chcr_req->key_ctx.key +
324429d7
HS		 878 (ablkctx->enckey_len >> 1),
879 ablkctx->key,
880 ablkctx->enckey_len >> 1);
881 }
882 }
358961d1 883 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2f47d580
HJ		 884 ulptx = (struct ulptx_sgl *)((u8 *)(phys_cpl + 1) + dst_size);
885 chcr_add_cipher_src_ent(wrparam->req, ulptx, wrparam);
886 chcr_add_cipher_dst_ent(wrparam->req, phys_cpl, wrparam, wrparam->qid);
324429d7 887
ee0863ba 888 atomic_inc(&adap->chcr_stats.cipher_rqst);
335bcc4a
HJ		 889 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + kctx_len + IV
890 + (reqctx->imm ? (wrparam->bytes) : 0);
2f47d580
HJ		 891 create_wreq(c_ctx(tfm), chcr_req, &(wrparam->req->base), reqctx->imm, 0,
892 transhdr_len, temp,
2512a624 893 ablkctx->ciph_mode == CHCR_SCMD_CIPHER_MODE_AES_CBC);
5c86a8ff 894 reqctx->skb = skb;
5fb78dba
HJ		 895
896 if (reqctx->op && (ablkctx->ciph_mode ==
897 CHCR_SCMD_CIPHER_MODE_AES_CBC))
898 sg_pcopy_to_buffer(wrparam->req->src,
7cea6d3e 899 sg_nents(wrparam->req->src), wrparam->req->iv, 16,
5fb78dba
HJ		 900 reqctx->processed + wrparam->bytes - AES_BLOCK_SIZE);
901
324429d7 902 return skb;
b8fd1f41
HJ		 903err:
904 return ERR_PTR(error);
905}
906
907static inline int chcr_keyctx_ck_size(unsigned int keylen)
908{
909 int ck_size = 0;
910
911 if (keylen == AES_KEYSIZE_128)
912 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
913 else if (keylen == AES_KEYSIZE_192)
914 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
915 else if (keylen == AES_KEYSIZE_256)
916 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
917 else
918 ck_size = 0;
919
920 return ck_size;
921}
7cea6d3e 922static int chcr_cipher_fallback_setkey(struct crypto_skcipher *cipher,
b8fd1f41
HJ		 923 const u8 *key,
924 unsigned int keylen)
925{
2f47d580 926 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
b8fd1f41 927
28874f26
KC		 928 crypto_sync_skcipher_clear_flags(ablkctx->sw_cipher,
929 CRYPTO_TFM_REQ_MASK);
930 crypto_sync_skcipher_set_flags(ablkctx->sw_cipher,
931 cipher->base.crt_flags & CRYPTO_TFM_REQ_MASK);
af5034e8 932 return crypto_sync_skcipher_setkey(ablkctx->sw_cipher, key, keylen);
324429d7
HS		 933}
934
7cea6d3e 935static int chcr_aes_cbc_setkey(struct crypto_skcipher *cipher,
b8fd1f41 936 const u8 *key,
324429d7
HS		 937 unsigned int keylen)
938{
2f47d580 939 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
324429d7
HS		 940 unsigned int ck_size, context_size;
941 u16 alignment = 0;
b8fd1f41 942 int err;
324429d7 943
b8fd1f41
HJ		 944 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
945 if (err)
324429d7 946 goto badkey_err;
b8fd1f41
HJ		 947
948 ck_size = chcr_keyctx_ck_size(keylen);
949 alignment = ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192 ? 8 : 0;
cc1b156d
HJ		 950 memcpy(ablkctx->key, key, keylen);
951 ablkctx->enckey_len = keylen;
952 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, keylen << 3);
324429d7
HS		 953 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
954 keylen + alignment) >> 4;
955
956 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
957 0, 0, context_size);
958 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CBC;
959 return 0;
960badkey_err:
324429d7 961 ablkctx->enckey_len = 0;
b8fd1f41
HJ		 962
963 return err;
324429d7
HS		 964}
965
7cea6d3e 966static int chcr_aes_ctr_setkey(struct crypto_skcipher *cipher,
b8fd1f41
HJ		 967 const u8 *key,
968 unsigned int keylen)
324429d7 969{
2f47d580 970 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
b8fd1f41
HJ		 971 unsigned int ck_size, context_size;
972 u16 alignment = 0;
973 int err;
974
975 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
976 if (err)
977 goto badkey_err;
978 ck_size = chcr_keyctx_ck_size(keylen);
979 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
980 memcpy(ablkctx->key, key, keylen);
981 ablkctx->enckey_len = keylen;
982 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
983 keylen + alignment) >> 4;
984
985 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
986 0, 0, context_size);
987 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
988
989 return 0;
990badkey_err:
b8fd1f41
HJ		 991 ablkctx->enckey_len = 0;
992
993 return err;
994}
995
7cea6d3e 996static int chcr_aes_rfc3686_setkey(struct crypto_skcipher *cipher,
b8fd1f41
HJ		 997 const u8 *key,
998 unsigned int keylen)
999{
2f47d580 1000 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
b8fd1f41
HJ		 1001 unsigned int ck_size, context_size;
1002 u16 alignment = 0;
1003 int err;
1004
1005 if (keylen < CTR_RFC3686_NONCE_SIZE)
1006 return -EINVAL;
1007 memcpy(ablkctx->nonce, key + (keylen - CTR_RFC3686_NONCE_SIZE),
1008 CTR_RFC3686_NONCE_SIZE);
1009
1010 keylen -= CTR_RFC3686_NONCE_SIZE;
1011 err = chcr_cipher_fallback_setkey(cipher, key, keylen);
1012 if (err)
1013 goto badkey_err;
1014
1015 ck_size = chcr_keyctx_ck_size(keylen);
1016 alignment = (ck_size == CHCR_KEYCTX_CIPHER_KEY_SIZE_192) ? 8 : 0;
1017 memcpy(ablkctx->key, key, keylen);
1018 ablkctx->enckey_len = keylen;
1019 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD +
1020 keylen + alignment) >> 4;
1021
1022 ablkctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY,
1023 0, 0, context_size);
1024 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_CTR;
1025
1026 return 0;
1027badkey_err:
b8fd1f41
HJ		 1028 ablkctx->enckey_len = 0;
1029
1030 return err;
1031}
1032static void ctr_add_iv(u8 *dstiv, u8 *srciv, u32 add)
1033{
1034 unsigned int size = AES_BLOCK_SIZE;
1035 __be32 *b = (__be32 *)(dstiv + size);
1036 u32 c, prev;
1037
1038 memcpy(dstiv, srciv, AES_BLOCK_SIZE);
1039 for (; size >= 4; size -= 4) {
1040 prev = be32_to_cpu(*--b);
1041 c = prev + add;
1042 *b = cpu_to_be32(c);
1043 if (prev < c)
1044 break;
1045 add = 1;
1046 }
1047
1048}
1049
1050static unsigned int adjust_ctr_overflow(u8 *iv, u32 bytes)
1051{
1052 __be32 *b = (__be32 *)(iv + AES_BLOCK_SIZE);
1053 u64 c;
1054 u32 temp = be32_to_cpu(*--b);
1055
1056 temp = ~temp;
1057 c = (u64)temp + 1; // No of block can processed withou overflow
1058 if ((bytes / AES_BLOCK_SIZE) > c)
1059 bytes = c * AES_BLOCK_SIZE;
1060 return bytes;
1061}
1062
7cea6d3e 1063static int chcr_update_tweak(struct skcipher_request *req, u8 *iv,
209897d5 1064 u32 isfinal)
b8fd1f41 1065{
7cea6d3e 1066 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
2f47d580 1067 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
7cea6d3e 1068 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
571c47ab 1069 struct crypto_aes_ctx aes;
b8fd1f41
HJ		 1070 int ret, i;
1071 u8 *key;
1072 unsigned int keylen;
de1a00ac
HJ		 1073 int round = reqctx->last_req_len / AES_BLOCK_SIZE;
1074 int round8 = round / 8;
b8fd1f41 1075
de1a00ac 1076 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
b8fd1f41 1077
b8fd1f41
HJ		 1078 keylen = ablkctx->enckey_len / 2;
1079 key = ablkctx->key + keylen;
571c47ab 1080 ret = aes_expandkey(&aes, key, keylen);
b8fd1f41 1081 if (ret)
571c47ab
AB		 1082 return ret;
1083 aes_encrypt(&aes, iv, iv);
de1a00ac
HJ		 1084 for (i = 0; i < round8; i++)
1085 gf128mul_x8_ble((le128 *)iv, (le128 *)iv);
1086
1087 for (i = 0; i < (round % 8); i++)
b8fd1f41
HJ		 1088 gf128mul_x_ble((le128 *)iv, (le128 *)iv);
1089
209897d5 1090 if (!isfinal)
571c47ab
AB		 1091 aes_decrypt(&aes, iv, iv);
1092
1093 memzero_explicit(&aes, sizeof(aes));
1094 return 0;
b8fd1f41
HJ		 1095}
1096
7cea6d3e 1097static int chcr_update_cipher_iv(struct skcipher_request *req,
b8fd1f41
HJ		 1098 struct cpl_fw6_pld *fw6_pld, u8 *iv)
1099{
7cea6d3e
AB		 1100 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1101 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1102 int subtype = get_cryptoalg_subtype(tfm);
ab677ff4 1103 int ret = 0;
324429d7 1104
b8fd1f41 1105 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
7cea6d3e 1106 ctr_add_iv(iv, req->iv, (reqctx->processed /
b8fd1f41
HJ		 1107 AES_BLOCK_SIZE));
1108 else if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_RFC3686)
1109 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1110 CTR_RFC3686_IV_SIZE) = cpu_to_be32((reqctx->processed /
1111 AES_BLOCK_SIZE) + 1);
1112 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS)
209897d5 1113 ret = chcr_update_tweak(req, iv, 0);
b8fd1f41
HJ		 1114 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
1115 if (reqctx->op)
5fb78dba 1116 /*Updated before sending last WR*/
7cea6d3e 1117 memcpy(iv, req->iv, AES_BLOCK_SIZE);
b8fd1f41
HJ		 1118 else
1119 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1120 }
1121
324429d7 1122 return ret;
b8fd1f41 1123
324429d7
HS		 1124}
1125
b8fd1f41
HJ		 1126/* We need separate function for final iv because in rfc3686 Initial counter
1127 * starts from 1 and buffer size of iv is 8 byte only which remains constant
1128 * for subsequent update requests
1129 */
1130
7cea6d3e 1131static int chcr_final_cipher_iv(struct skcipher_request *req,
b8fd1f41
HJ		 1132 struct cpl_fw6_pld *fw6_pld, u8 *iv)
1133{
7cea6d3e
AB		 1134 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1135 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
1136 int subtype = get_cryptoalg_subtype(tfm);
b8fd1f41
HJ		 1137 int ret = 0;
1138
1139 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR)
7cea6d3e 1140 ctr_add_iv(iv, req->iv, DIV_ROUND_UP(reqctx->processed,
0a4491d3 1141 AES_BLOCK_SIZE));
bed44d0c
AS		 1142 else if (subtype == CRYPTO_ALG_SUB_TYPE_XTS) {
1143 if (!reqctx->partial_req)
1144 memcpy(iv, reqctx->iv, AES_BLOCK_SIZE);
1145 else
1146 ret = chcr_update_tweak(req, iv, 1);
1147 }
b8fd1f41 1148 else if (subtype == CRYPTO_ALG_SUB_TYPE_CBC) {
5fb78dba
HJ		 1149 /*Already updated for Decrypt*/
1150 if (!reqctx->op)
b8fd1f41
HJ		 1151 memcpy(iv, &fw6_pld->data[2], AES_BLOCK_SIZE);
1152
1153 }
1154 return ret;
1155
1156}
1157
7cea6d3e 1158static int chcr_handle_cipher_resp(struct skcipher_request *req,
b8fd1f41 1159 unsigned char *input, int err)
324429d7 1160{
7cea6d3e 1161 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1c502e2e 1162 struct chcr_context *ctx = c_ctx(tfm);
2f47d580
HJ		 1163 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
1164 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
324429d7 1165 struct sk_buff *skb;
b8fd1f41 1166 struct cpl_fw6_pld *fw6_pld = (struct cpl_fw6_pld *)input;
7cea6d3e 1167 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
567be3a5 1168 struct cipher_wr_param wrparam;
fef4912b 1169 struct chcr_dev *dev = c_ctx(tfm)->dev;
b8fd1f41
HJ		 1170 int bytes;
1171
b8fd1f41 1172 if (err)
2f47d580 1173 goto unmap;
7cea6d3e 1174 if (req->cryptlen == reqctx->processed) {
2f47d580
HJ		 1175 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1176 req);
7cea6d3e 1177 err = chcr_final_cipher_iv(req, fw6_pld, req->iv);
b8fd1f41
HJ		 1178 goto complete;
1179 }
1180
2f47d580 1181 if (!reqctx->imm) {
335bcc4a 1182 bytes = chcr_sg_ent_in_wr(reqctx->srcsg, reqctx->dstsg, 0,
5110e655 1183 CIP_SPACE_LEFT(ablkctx->enckey_len),
2f47d580 1184 reqctx->src_ofst, reqctx->dst_ofst);
7cea6d3e
AB		 1185 if ((bytes + reqctx->processed) >= req->cryptlen)
1186 bytes = req->cryptlen - reqctx->processed;
db6deea4 1187 else
125d01ca 1188 bytes = rounddown(bytes, 16);
2f47d580
HJ		 1189 } else {
1190 /*CTR mode counter overfloa*/
7cea6d3e 1191 bytes = req->cryptlen - reqctx->processed;
2f47d580 1192 }
b8fd1f41
HJ		 1193 err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv);
1194 if (err)
2f47d580 1195 goto unmap;
b8fd1f41
HJ		 1196
1197 if (unlikely(bytes == 0)) {
2f47d580
HJ		 1198 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1199 req);
b8fd1f41
HJ		 1200 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1201 req->base.flags,
2f47d580
HJ		 1202 req->src,
1203 req->dst,
7cea6d3e
AB		 1204 req->cryptlen,
1205 req->iv,
b8fd1f41
HJ		 1206 reqctx->op);
1207 goto complete;
1208 }
1209
7cea6d3e 1210 if (get_cryptoalg_subtype(tfm) ==
b8fd1f41
HJ		 1211 CRYPTO_ALG_SUB_TYPE_CTR)
1212 bytes = adjust_ctr_overflow(reqctx->iv, bytes);
567be3a5 1213 wrparam.qid = u_ctx->lldi.rxq_ids[reqctx->rxqidx];
b8fd1f41
HJ		 1214 wrparam.req = req;
1215 wrparam.bytes = bytes;
1216 skb = create_cipher_wr(&wrparam);
1217 if (IS_ERR(skb)) {
1218 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__);
1219 err = PTR_ERR(skb);
2f47d580 1220 goto unmap;
b8fd1f41
HJ		 1221 }
1222 skb->dev = u_ctx->lldi.ports[0];
567be3a5 1223 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
b8fd1f41 1224 chcr_send_wr(skb);
2f47d580
HJ		 1225 reqctx->last_req_len = bytes;
1226 reqctx->processed += bytes;
1c502e2e
AS		 1227 if (get_cryptoalg_subtype(tfm) ==
1228 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1229 CRYPTO_TFM_REQ_MAY_SLEEP ) {
1230 complete(&ctx->cbc_aes_aio_done);
1231 }
b8fd1f41 1232 return 0;
2f47d580
HJ		 1233unmap:
1234 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
b8fd1f41 1235complete:
1c502e2e
AS		 1236 if (get_cryptoalg_subtype(tfm) ==
1237 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1238 CRYPTO_TFM_REQ_MAY_SLEEP ) {
1239 complete(&ctx->cbc_aes_aio_done);
1240 }
fef4912b 1241 chcr_dec_wrcount(dev);
b8fd1f41
HJ		 1242 req->base.complete(&req->base, err);
1243 return err;
1244}
1245
7cea6d3e 1246static int process_cipher(struct skcipher_request *req,
b8fd1f41
HJ		 1247 unsigned short qid,
1248 struct sk_buff **skb,
1249 unsigned short op_type)
1250{
7cea6d3e
AB		 1251 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
1252 unsigned int ivsize = crypto_skcipher_ivsize(tfm);
1253 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2f47d580 1254 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(tfm));
b8fd1f41 1255 struct cipher_wr_param wrparam;
2956f36c 1256 int bytes, err = -EINVAL;
b8fd1f41 1257
b8fd1f41 1258 reqctx->processed = 0;
bed44d0c 1259 reqctx->partial_req = 0;
7cea6d3e 1260 if (!req->iv)
b8fd1f41
HJ		 1261 goto error;
1262 if ((ablkctx->enckey_len == 0) || (ivsize > AES_BLOCK_SIZE) ||
7cea6d3e
AB		 1263 (req->cryptlen == 0) ||
1264 (req->cryptlen % crypto_skcipher_blocksize(tfm))) {
b8fd1f41 1265 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
7cea6d3e 1266 ablkctx->enckey_len, req->cryptlen, ivsize);
b8fd1f41
HJ		 1267 goto error;
1268 }
fef4912b
HJ		 1269
1270 err = chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
1271 if (err)
1272 goto error;
7cea6d3e 1273 if (req->cryptlen < (SGE_MAX_WR_LEN - (sizeof(struct chcr_wr) +
2f47d580
HJ		 1274 AES_MIN_KEY_SIZE +
1275 sizeof(struct cpl_rx_phys_dsgl) +
1276 /*Min dsgl size*/
1277 32))) {
1278 /* Can be sent as Imm*/
1279 unsigned int dnents = 0, transhdr_len, phys_dsgl, kctx_len;
1280
7cea6d3e 1281 dnents = sg_nents_xlen(req->dst, req->cryptlen,
2f47d580 1282 CHCR_DST_SG_SIZE, 0);
2f47d580 1283 phys_dsgl = get_space_for_phys_dsgl(dnents);
125d01ca 1284 kctx_len = roundup(ablkctx->enckey_len, 16);
2f47d580 1285 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, phys_dsgl);
7cea6d3e 1286 reqctx->imm = (transhdr_len + IV + req->cryptlen) <=
2f47d580 1287 SGE_MAX_WR_LEN;
7cea6d3e 1288 bytes = IV + req->cryptlen;
2f47d580
HJ		 1289
1290 } else {
1291 reqctx->imm = 0;
1292 }
1293
1294 if (!reqctx->imm) {
335bcc4a 1295 bytes = chcr_sg_ent_in_wr(req->src, req->dst, 0,
5110e655 1296 CIP_SPACE_LEFT(ablkctx->enckey_len),
2f47d580 1297 0, 0);
7cea6d3e
AB		 1298 if ((bytes + reqctx->processed) >= req->cryptlen)
1299 bytes = req->cryptlen - reqctx->processed;
db6deea4 1300 else
125d01ca 1301 bytes = rounddown(bytes, 16);
2f47d580 1302 } else {
7cea6d3e 1303 bytes = req->cryptlen;
2f47d580 1304 }
7cea6d3e 1305 if (get_cryptoalg_subtype(tfm) ==
db6deea4 1306 CRYPTO_ALG_SUB_TYPE_CTR) {
7cea6d3e 1307 bytes = adjust_ctr_overflow(req->iv, bytes);
b8fd1f41 1308 }
7cea6d3e 1309 if (get_cryptoalg_subtype(tfm) ==
b8fd1f41
HJ		 1310 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686) {
1311 memcpy(reqctx->iv, ablkctx->nonce, CTR_RFC3686_NONCE_SIZE);
7cea6d3e 1312 memcpy(reqctx->iv + CTR_RFC3686_NONCE_SIZE, req->iv,
b8fd1f41
HJ		 1313 CTR_RFC3686_IV_SIZE);
1314
1315 /* initialize counter portion of counter block */
1316 *(__be32 *)(reqctx->iv + CTR_RFC3686_NONCE_SIZE +
1317 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
1318
1319 } else {
1320
7cea6d3e 1321 memcpy(reqctx->iv, req->iv, IV);
b8fd1f41
HJ		 1322 }
1323 if (unlikely(bytes == 0)) {
2f47d580
HJ		 1324 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev,
1325 req);
b8fd1f41
HJ		 1326 err = chcr_cipher_fallback(ablkctx->sw_cipher,
1327 req->base.flags,
1328 req->src,
1329 req->dst,
7cea6d3e 1330 req->cryptlen,
7ffb9118 1331 reqctx->iv,
b8fd1f41
HJ		 1332 op_type);
1333 goto error;
1334 }
b8fd1f41 1335 reqctx->op = op_type;
2f47d580
HJ		 1336 reqctx->srcsg = req->src;
1337 reqctx->dstsg = req->dst;
1338 reqctx->src_ofst = 0;
1339 reqctx->dst_ofst = 0;
b8fd1f41
HJ		 1340 wrparam.qid = qid;
1341 wrparam.req = req;
1342 wrparam.bytes = bytes;
1343 *skb = create_cipher_wr(&wrparam);
1344 if (IS_ERR(*skb)) {
1345 err = PTR_ERR(*skb);
2f47d580 1346 goto unmap;
b8fd1f41 1347 }
2f47d580
HJ		 1348 reqctx->processed = bytes;
1349 reqctx->last_req_len = bytes;
bed44d0c 1350 reqctx->partial_req = !!(req->cryptlen - reqctx->processed);
b8fd1f41
HJ		 1351
1352 return 0;
2f47d580
HJ		 1353unmap:
1354 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm))->lldi.pdev->dev, req);
b8fd1f41
HJ		 1355error:
1356 return err;
1357}
1358
7cea6d3e 1359static int chcr_aes_encrypt(struct skcipher_request *req)
b8fd1f41 1360{
7cea6d3e 1361 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
bed44d0c 1362 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
fef4912b 1363 struct chcr_dev *dev = c_ctx(tfm)->dev;
b8fd1f41 1364 struct sk_buff *skb = NULL;
567be3a5 1365 int err;
2f47d580 1366 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
567be3a5
AS		 1367 struct chcr_context *ctx = c_ctx(tfm);
1368 unsigned int cpu;
1369
1370 cpu = get_cpu();
1371 reqctx->txqidx = cpu % ctx->ntxq;
1372 reqctx->rxqidx = cpu % ctx->nrxq;
1373 put_cpu();
324429d7 1374
fef4912b
HJ		 1375 err = chcr_inc_wrcount(dev);
1376 if (err)
1377 return -ENXIO;
324429d7 1378 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
567be3a5
AS		 1379 reqctx->txqidx) &&
1380 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
fef4912b
HJ		 1381 err = -ENOSPC;
1382 goto error;
324429d7
HS		 1383 }
1384
567be3a5 1385 err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx],
2f47d580 1386 &skb, CHCR_ENCRYPT_OP);
b8fd1f41
HJ		 1387 if (err || !skb)
1388 return err;
324429d7 1389 skb->dev = u_ctx->lldi.ports[0];
567be3a5 1390 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
324429d7 1391 chcr_send_wr(skb);
1c502e2e
AS		 1392 if (get_cryptoalg_subtype(tfm) ==
1393 CRYPTO_ALG_SUB_TYPE_CBC && req->base.flags ==
1394 CRYPTO_TFM_REQ_MAY_SLEEP ) {
bed44d0c 1395 reqctx->partial_req = 1;
1c502e2e
AS		 1396 wait_for_completion(&ctx->cbc_aes_aio_done);
1397 }
567be3a5 1398 return -EINPROGRESS;
fef4912b
HJ		 1399error:
1400 chcr_dec_wrcount(dev);
1401 return err;
324429d7
HS		 1402}
1403
7cea6d3e 1404static int chcr_aes_decrypt(struct skcipher_request *req)
324429d7 1405{
7cea6d3e 1406 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
567be3a5 1407 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2f47d580 1408 struct uld_ctx *u_ctx = ULD_CTX(c_ctx(tfm));
fef4912b 1409 struct chcr_dev *dev = c_ctx(tfm)->dev;
b8fd1f41 1410 struct sk_buff *skb = NULL;
567be3a5
AS		 1411 int err;
1412 struct chcr_context *ctx = c_ctx(tfm);
1413 unsigned int cpu;
1414
1415 cpu = get_cpu();
1416 reqctx->txqidx = cpu % ctx->ntxq;
1417 reqctx->rxqidx = cpu % ctx->nrxq;
1418 put_cpu();
324429d7 1419
fef4912b
HJ		 1420 err = chcr_inc_wrcount(dev);
1421 if (err)
1422 return -ENXIO;
1423
324429d7 1424 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
567be3a5
AS		 1425 reqctx->txqidx) &&
1426 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))))
6faa0f57 1427 return -ENOSPC;
567be3a5 1428 err = process_cipher(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx],
fc6176a2 1429 &skb, CHCR_DECRYPT_OP);
b8fd1f41
HJ		 1430 if (err || !skb)
1431 return err;
324429d7 1432 skb->dev = u_ctx->lldi.ports[0];
567be3a5 1433 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
324429d7 1434 chcr_send_wr(skb);
567be3a5 1435 return -EINPROGRESS;
324429d7 1436}
324429d7
HS		 1437static int chcr_device_init(struct chcr_context *ctx)
1438{
14c19b17 1439 struct uld_ctx *u_ctx = NULL;
567be3a5
AS		 1440 int txq_perchan, ntxq;
1441 int err = 0, rxq_perchan;
324429d7 1442
324429d7 1443 if (!ctx->dev) {
14c19b17
HJ		 1444 u_ctx = assign_chcr_device();
1445 if (!u_ctx) {
324429d7
HS		 1446 pr_err("chcr device assignment fails\n");
1447 goto out;
1448 }
fef4912b 1449 ctx->dev = &u_ctx->dev;
a1c6fd43 1450 ntxq = u_ctx->lldi.ntxq;
324429d7 1451 rxq_perchan = u_ctx->lldi.nrxq / u_ctx->lldi.nchan;
72a56ca9 1452 txq_perchan = ntxq / u_ctx->lldi.nchan;
567be3a5
AS		 1453 ctx->ntxq = ntxq;
1454 ctx->nrxq = u_ctx->lldi.nrxq;
1455 ctx->rxq_perchan = rxq_perchan;
1456 ctx->txq_perchan = txq_perchan;
324429d7
HS		 1457 }
1458out:
1459 return err;
1460}
1461
7cea6d3e 1462static int chcr_init_tfm(struct crypto_skcipher *tfm)
324429d7 1463{
7cea6d3e
AB		 1464 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1465 struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
b8fd1f41
HJ		 1466 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1467
7cea6d3e 1468 ablkctx->sw_cipher = crypto_alloc_sync_skcipher(alg->base.cra_name, 0,
28874f26 1469 CRYPTO_ALG_NEED_FALLBACK);
b8fd1f41 1470 if (IS_ERR(ablkctx->sw_cipher)) {
7cea6d3e 1471 pr_err("failed to allocate fallback for %s\n", alg->base.cra_name);
b8fd1f41
HJ		 1472 return PTR_ERR(ablkctx->sw_cipher);
1473 }
1c502e2e 1474 init_completion(&ctx->cbc_aes_aio_done);
7cea6d3e
AB		 1475 crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx));
1476
1477 return chcr_device_init(ctx);
324429d7
HS		 1478}
1479
7cea6d3e 1480static int chcr_rfc3686_init(struct crypto_skcipher *tfm)
b8fd1f41 1481{
7cea6d3e
AB		 1482 struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
1483 struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
b8fd1f41
HJ		 1484 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1485
1486 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1487 * cannot be used as fallback in chcr_handle_cipher_response
1488 */
28874f26
KC		 1489 ablkctx->sw_cipher = crypto_alloc_sync_skcipher("ctr(aes)", 0,
1490 CRYPTO_ALG_NEED_FALLBACK);
b8fd1f41 1491 if (IS_ERR(ablkctx->sw_cipher)) {
7cea6d3e 1492 pr_err("failed to allocate fallback for %s\n", alg->base.cra_name);
b8fd1f41
HJ		 1493 return PTR_ERR(ablkctx->sw_cipher);
1494 }
7cea6d3e
AB		 1495 crypto_skcipher_set_reqsize(tfm, sizeof(struct chcr_skcipher_req_ctx));
1496 return chcr_device_init(ctx);
b8fd1f41
HJ		 1497}
1498
1499
7cea6d3e 1500static void chcr_exit_tfm(struct crypto_skcipher *tfm)
b8fd1f41 1501{
7cea6d3e 1502 struct chcr_context *ctx = crypto_skcipher_ctx(tfm);
b8fd1f41
HJ		 1503 struct ablk_ctx *ablkctx = ABLK_CTX(ctx);
1504
28874f26 1505 crypto_free_sync_skcipher(ablkctx->sw_cipher);
b8fd1f41
HJ		 1506}
1507
324429d7
HS		 1508static int get_alg_config(struct algo_param *params,
1509 unsigned int auth_size)
1510{
1511 switch (auth_size) {
1512 case SHA1_DIGEST_SIZE:
1513 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_160;
1514 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA1;
1515 params->result_size = SHA1_DIGEST_SIZE;
1516 break;
1517 case SHA224_DIGEST_SIZE:
1518 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1519 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA224;
1520 params->result_size = SHA256_DIGEST_SIZE;
1521 break;
1522 case SHA256_DIGEST_SIZE:
1523 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
1524 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA256;
1525 params->result_size = SHA256_DIGEST_SIZE;
1526 break;
1527 case SHA384_DIGEST_SIZE:
1528 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1529 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_384;
1530 params->result_size = SHA512_DIGEST_SIZE;
1531 break;
1532 case SHA512_DIGEST_SIZE:
1533 params->mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_512;
1534 params->auth_mode = CHCR_SCMD_AUTH_MODE_SHA512_512;
1535 params->result_size = SHA512_DIGEST_SIZE;
1536 break;
1537 default:
1538 pr_err("chcr : ERROR, unsupported digest size\n");
1539 return -EINVAL;
1540 }
1541 return 0;
1542}
1543
e7922729 1544static inline void chcr_free_shash(struct crypto_shash *base_hash)
324429d7 1545{
e7922729 1546 crypto_free_shash(base_hash);
324429d7
HS		 1547}
1548
1549/**
358961d1 1550 * create_hash_wr - Create hash work request
324429d7
HS		 1551 * @req - Cipher req base
1552 */
358961d1 1553static struct sk_buff *create_hash_wr(struct ahash_request *req,
2debd332 1554 struct hash_wr_param *param)
324429d7
HS		 1555{
1556 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1557 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
567be3a5
AS		 1558 struct chcr_context *ctx = h_ctx(tfm);
1559 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
324429d7 1560 struct sk_buff *skb = NULL;
567be3a5 1561 struct uld_ctx *u_ctx = ULD_CTX(ctx);
358961d1 1562 struct chcr_wr *chcr_req;
2f47d580 1563 struct ulptx_sgl *ulptx;
5110e655
HJ		 1564 unsigned int nents = 0, transhdr_len;
1565 unsigned int temp = 0;
358961d1
HJ		 1566 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1567 GFP_ATOMIC;
2f47d580
HJ		 1568 struct adapter *adap = padap(h_ctx(tfm)->dev);
1569 int error = 0;
567be3a5 1570 unsigned int rx_channel_id = req_ctx->rxqidx / ctx->rxq_perchan;
324429d7 1571
5110e655
HJ		 1572 transhdr_len = HASH_TRANSHDR_SIZE(param->kctx_len);
1573 req_ctx->hctx_wr.imm = (transhdr_len + param->bfr_len +
1574 param->sg_len) <= SGE_MAX_WR_LEN;
1575 nents = sg_nents_xlen(req_ctx->hctx_wr.srcsg, param->sg_len,
1576 CHCR_SRC_SG_SIZE, req_ctx->hctx_wr.src_ofst);
2f47d580 1577 nents += param->bfr_len ? 1 : 0;
5110e655
HJ		 1578 transhdr_len += req_ctx->hctx_wr.imm ? roundup(param->bfr_len +
1579 param->sg_len, 16) : (sgl_len(nents) * 8);
125d01ca 1580 transhdr_len = roundup(transhdr_len, 16);
2f47d580 1581
5110e655 1582 skb = alloc_skb(transhdr_len, flags);
324429d7 1583 if (!skb)
2f47d580 1584 return ERR_PTR(-ENOMEM);
de77b966 1585 chcr_req = __skb_put_zero(skb, transhdr_len);
324429d7 1586
358961d1 1587 chcr_req->sec_cpl.op_ivinsrtofst =
567be3a5
AS		 1588 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 0);
1589
358961d1 1590 chcr_req->sec_cpl.pldlen = htonl(param->bfr_len + param->sg_len);
324429d7 1591
358961d1 1592 chcr_req->sec_cpl.aadstart_cipherstop_hi =
324429d7 1593 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
358961d1 1594 chcr_req->sec_cpl.cipherstop_lo_authinsert =
324429d7 1595 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
358961d1 1596 chcr_req->sec_cpl.seqno_numivs =
324429d7 1597 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param->alg_prm.auth_mode,
358961d1 1598 param->opad_needed, 0);
324429d7 1599
358961d1 1600 chcr_req->sec_cpl.ivgen_hdrlen =
324429d7
HS		 1601 FILL_SEC_CPL_IVGEN_HDRLEN(param->last, param->more, 0, 1, 0, 0);
1602
358961d1
HJ		 1603 memcpy(chcr_req->key_ctx.key, req_ctx->partial_hash,
1604 param->alg_prm.result_size);
324429d7
HS		 1605
1606 if (param->opad_needed)
358961d1
HJ		 1607 memcpy(chcr_req->key_ctx.key +
1608 ((param->alg_prm.result_size <= 32) ? 32 :
1609 CHCR_HASH_MAX_DIGEST_SIZE),
324429d7
HS		 1610 hmacctx->opad, param->alg_prm.result_size);
1611
358961d1 1612 chcr_req->key_ctx.ctx_hdr = FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY,
324429d7
HS		 1613 param->alg_prm.mk_size, 0,
1614 param->opad_needed,
5110e655 1615 ((param->kctx_len +
358961d1
HJ		 1616 sizeof(chcr_req->key_ctx)) >> 4));
1617 chcr_req->sec_cpl.scmd1 = cpu_to_be64((u64)param->scmd1);
5110e655 1618 ulptx = (struct ulptx_sgl *)((u8 *)(chcr_req + 1) + param->kctx_len +
2f47d580
HJ		 1619 DUMMY_BYTES);
1620 if (param->bfr_len != 0) {
5110e655
HJ		 1621 req_ctx->hctx_wr.dma_addr =
1622 dma_map_single(&u_ctx->lldi.pdev->dev, req_ctx->reqbfr,
1623 param->bfr_len, DMA_TO_DEVICE);
2f47d580 1624 if (dma_mapping_error(&u_ctx->lldi.pdev->dev,
5110e655 1625 req_ctx->hctx_wr. dma_addr)) {
2f47d580
HJ		 1626 error = -ENOMEM;
1627 goto err;
1628 }
5110e655 1629 req_ctx->hctx_wr.dma_len = param->bfr_len;
2f47d580 1630 } else {
5110e655 1631 req_ctx->hctx_wr.dma_addr = 0;
2f47d580
HJ		 1632 }
1633 chcr_add_hash_src_ent(req, ulptx, param);
1634 /* Request upto max wr size */
5110e655
HJ		 1635 temp = param->kctx_len + DUMMY_BYTES + (req_ctx->hctx_wr.imm ?
1636 (param->sg_len + param->bfr_len) : 0);
ee0863ba 1637 atomic_inc(&adap->chcr_stats.digest_rqst);
5110e655
HJ		 1638 create_wreq(h_ctx(tfm), chcr_req, &req->base, req_ctx->hctx_wr.imm,
1639 param->hash_size, transhdr_len,
2f47d580 1640 temp, 0);
5110e655 1641 req_ctx->hctx_wr.skb = skb;
324429d7 1642 return skb;
2f47d580
HJ		 1643err:
1644 kfree_skb(skb);
1645 return ERR_PTR(error);
324429d7
HS		 1646}
1647
1648static int chcr_ahash_update(struct ahash_request *req)
1649{
1650 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1651 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
567be3a5
AS		 1652 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1653 struct chcr_context *ctx = h_ctx(rtfm);
fef4912b 1654 struct chcr_dev *dev = h_ctx(rtfm)->dev;
324429d7
HS		 1655 struct sk_buff *skb;
1656 u8 remainder = 0, bs;
1657 unsigned int nbytes = req->nbytes;
1658 struct hash_wr_param params;
567be3a5
AS		 1659 int error;
1660 unsigned int cpu;
1661
1662 cpu = get_cpu();
1663 req_ctx->txqidx = cpu % ctx->ntxq;
1664 req_ctx->rxqidx = cpu % ctx->nrxq;
1665 put_cpu();
324429d7
HS		 1666
1667 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
324429d7 1668
44fce12a
HJ		 1669 if (nbytes + req_ctx->reqlen >= bs) {
1670 remainder = (nbytes + req_ctx->reqlen) % bs;
1671 nbytes = nbytes + req_ctx->reqlen - remainder;
324429d7 1672 } else {
44fce12a
HJ		 1673 sg_pcopy_to_buffer(req->src, sg_nents(req->src), req_ctx->reqbfr
1674 + req_ctx->reqlen, nbytes, 0);
1675 req_ctx->reqlen += nbytes;
324429d7
HS		 1676 return 0;
1677 }
fef4912b
HJ		 1678 error = chcr_inc_wrcount(dev);
1679 if (error)
1680 return -ENXIO;
1681 /* Detach state for CHCR means lldi or padap is freed. Increasing
1682 * inflight count for dev guarantees that lldi and padap is valid
1683 */
1684 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
567be3a5
AS		 1685 req_ctx->txqidx) &&
1686 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
fef4912b
HJ		 1687 error = -ENOSPC;
1688 goto err;
fef4912b
HJ		 1689 }
1690
5110e655 1691 chcr_init_hctx_per_wr(req_ctx);
2f47d580 1692 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
fef4912b
HJ		 1693 if (error) {
1694 error = -ENOMEM;
1695 goto err;
1696 }
5110e655
HJ		 1697 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1698 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1699 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1700 HASH_SPACE_LEFT(params.kctx_len), 0);
1701 if (params.sg_len > req->nbytes)
1702 params.sg_len = req->nbytes;
1703 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs) -
1704 req_ctx->reqlen;
324429d7
HS		 1705 params.opad_needed = 0;
1706 params.more = 1;
1707 params.last = 0;
44fce12a 1708 params.bfr_len = req_ctx->reqlen;
324429d7 1709 params.scmd1 = 0;
5110e655
HJ		 1710 req_ctx->hctx_wr.srcsg = req->src;
1711
1712 params.hash_size = params.alg_prm.result_size;
324429d7 1713 req_ctx->data_len += params.sg_len + params.bfr_len;
358961d1 1714 skb = create_hash_wr(req, &params);
2f47d580
HJ		 1715 if (IS_ERR(skb)) {
1716 error = PTR_ERR(skb);
1717 goto unmap;
1718 }
324429d7 1719
5110e655 1720 req_ctx->hctx_wr.processed += params.sg_len;
44fce12a 1721 if (remainder) {
44fce12a 1722 /* Swap buffers */
abfa2b37 1723 swap(req_ctx->reqbfr, req_ctx->skbfr);
324429d7 1724 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
44fce12a 1725 req_ctx->reqbfr, remainder, req->nbytes -
324429d7 1726 remainder);
44fce12a
HJ		 1727 }
1728 req_ctx->reqlen = remainder;
324429d7 1729 skb->dev = u_ctx->lldi.ports[0];
567be3a5 1730 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
324429d7 1731 chcr_send_wr(skb);
567be3a5 1732 return -EINPROGRESS;
2f47d580
HJ		 1733unmap:
1734 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
fef4912b
HJ		 1735err:
1736 chcr_dec_wrcount(dev);
2f47d580 1737 return error;
324429d7
HS		 1738}
1739
1740static void create_last_hash_block(char *bfr_ptr, unsigned int bs, u64 scmd1)
1741{
1742 memset(bfr_ptr, 0, bs);
1743 *bfr_ptr = 0x80;
1744 if (bs == 64)
1745 *(__be64 *)(bfr_ptr + 56) = cpu_to_be64(scmd1 << 3);
1746 else
1747 *(__be64 *)(bfr_ptr + 120) = cpu_to_be64(scmd1 << 3);
1748}
1749
1750static int chcr_ahash_final(struct ahash_request *req)
1751{
1752 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1753 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
fef4912b 1754 struct chcr_dev *dev = h_ctx(rtfm)->dev;
324429d7
HS		 1755 struct hash_wr_param params;
1756 struct sk_buff *skb;
567be3a5
AS		 1757 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1758 struct chcr_context *ctx = h_ctx(rtfm);
324429d7 1759 u8 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
fef4912b 1760 int error = -EINVAL;
567be3a5
AS		 1761 unsigned int cpu;
1762
1763 cpu = get_cpu();
1764 req_ctx->txqidx = cpu % ctx->ntxq;
1765 req_ctx->rxqidx = cpu % ctx->nrxq;
1766 put_cpu();
fef4912b
HJ		 1767
1768 error = chcr_inc_wrcount(dev);
1769 if (error)
1770 return -ENXIO;
324429d7 1771
5110e655 1772 chcr_init_hctx_per_wr(req_ctx);
324429d7
HS		 1773 if (is_hmac(crypto_ahash_tfm(rtfm)))
1774 params.opad_needed = 1;
1775 else
1776 params.opad_needed = 0;
1777 params.sg_len = 0;
5110e655 1778 req_ctx->hctx_wr.isfinal = 1;
324429d7 1779 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
5110e655
HJ		 1780 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1781 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1782 params.opad_needed = 1;
1783 params.kctx_len *= 2;
1784 } else {
1785 params.opad_needed = 0;
1786 }
1787
1788 req_ctx->hctx_wr.result = 1;
44fce12a 1789 params.bfr_len = req_ctx->reqlen;
324429d7 1790 req_ctx->data_len += params.bfr_len + params.sg_len;
5110e655 1791 req_ctx->hctx_wr.srcsg = req->src;
44fce12a
HJ		 1792 if (req_ctx->reqlen == 0) {
1793 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
324429d7
HS		 1794 params.last = 0;
1795 params.more = 1;
1796 params.scmd1 = 0;
1797 params.bfr_len = bs;
1798
1799 } else {
1800 params.scmd1 = req_ctx->data_len;
1801 params.last = 1;
1802 params.more = 0;
1803 }
5110e655 1804 params.hash_size = crypto_ahash_digestsize(rtfm);
358961d1 1805 skb = create_hash_wr(req, &params);
fef4912b
HJ		 1806 if (IS_ERR(skb)) {
1807 error = PTR_ERR(skb);
1808 goto err;
1809 }
5110e655 1810 req_ctx->reqlen = 0;
324429d7 1811 skb->dev = u_ctx->lldi.ports[0];
567be3a5 1812 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
324429d7
HS		 1813 chcr_send_wr(skb);
1814 return -EINPROGRESS;
fef4912b
HJ		 1815err:
1816 chcr_dec_wrcount(dev);
1817 return error;
324429d7
HS		 1818}
1819
1820static int chcr_ahash_finup(struct ahash_request *req)
1821{
1822 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1823 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
fef4912b 1824 struct chcr_dev *dev = h_ctx(rtfm)->dev;
567be3a5
AS		 1825 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1826 struct chcr_context *ctx = h_ctx(rtfm);
324429d7
HS		 1827 struct sk_buff *skb;
1828 struct hash_wr_param params;
1829 u8 bs;
567be3a5
AS		 1830 int error;
1831 unsigned int cpu;
1832
1833 cpu = get_cpu();
1834 req_ctx->txqidx = cpu % ctx->ntxq;
1835 req_ctx->rxqidx = cpu % ctx->nrxq;
1836 put_cpu();
324429d7
HS		 1837
1838 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
fef4912b
HJ		 1839 error = chcr_inc_wrcount(dev);
1840 if (error)
1841 return -ENXIO;
324429d7
HS		 1842
1843 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
567be3a5
AS		 1844 req_ctx->txqidx) &&
1845 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
fef4912b
HJ		 1846 error = -ENOSPC;
1847 goto err;
324429d7 1848 }
5110e655
HJ		 1849 chcr_init_hctx_per_wr(req_ctx);
1850 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
fef4912b
HJ		 1851 if (error) {
1852 error = -ENOMEM;
1853 goto err;
1854 }
324429d7 1855
5110e655
HJ		 1856 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
1857 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1858 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1859 params.kctx_len *= 2;
324429d7 1860 params.opad_needed = 1;
5110e655 1861 } else {
324429d7 1862 params.opad_needed = 0;
5110e655 1863 }
324429d7 1864
5110e655
HJ		 1865 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1866 HASH_SPACE_LEFT(params.kctx_len), 0);
1867 if (params.sg_len < req->nbytes) {
1868 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1869 params.kctx_len /= 2;
1870 params.opad_needed = 0;
1871 }
1872 params.last = 0;
1873 params.more = 1;
1874 params.sg_len = rounddown(params.sg_len + req_ctx->reqlen, bs)
1875 - req_ctx->reqlen;
1876 params.hash_size = params.alg_prm.result_size;
1877 params.scmd1 = 0;
1878 } else {
1879 params.last = 1;
1880 params.more = 0;
1881 params.sg_len = req->nbytes;
1882 params.hash_size = crypto_ahash_digestsize(rtfm);
1883 params.scmd1 = req_ctx->data_len + req_ctx->reqlen +
1884 params.sg_len;
1885 }
44fce12a 1886 params.bfr_len = req_ctx->reqlen;
324429d7 1887 req_ctx->data_len += params.bfr_len + params.sg_len;
5110e655
HJ		 1888 req_ctx->hctx_wr.result = 1;
1889 req_ctx->hctx_wr.srcsg = req->src;
44fce12a
HJ		 1890 if ((req_ctx->reqlen + req->nbytes) == 0) {
1891 create_last_hash_block(req_ctx->reqbfr, bs, req_ctx->data_len);
324429d7
HS		 1892 params.last = 0;
1893 params.more = 1;
1894 params.scmd1 = 0;
1895 params.bfr_len = bs;
324429d7 1896 }
358961d1 1897 skb = create_hash_wr(req, &params);
2f47d580
HJ		 1898 if (IS_ERR(skb)) {
1899 error = PTR_ERR(skb);
1900 goto unmap;
1901 }
5110e655
HJ		 1902 req_ctx->reqlen = 0;
1903 req_ctx->hctx_wr.processed += params.sg_len;
324429d7 1904 skb->dev = u_ctx->lldi.ports[0];
567be3a5 1905 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
324429d7 1906 chcr_send_wr(skb);
567be3a5 1907 return -EINPROGRESS;
2f47d580
HJ		 1908unmap:
1909 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
fef4912b
HJ		 1910err:
1911 chcr_dec_wrcount(dev);
2f47d580 1912 return error;
324429d7
HS		 1913}
1914
1915static int chcr_ahash_digest(struct ahash_request *req)
1916{
1917 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
1918 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
fef4912b 1919 struct chcr_dev *dev = h_ctx(rtfm)->dev;
567be3a5
AS		 1920 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(rtfm));
1921 struct chcr_context *ctx = h_ctx(rtfm);
324429d7
HS		 1922 struct sk_buff *skb;
1923 struct hash_wr_param params;
1924 u8 bs;
567be3a5
AS		 1925 int error;
1926 unsigned int cpu;
1927
1928 cpu = get_cpu();
1929 req_ctx->txqidx = cpu % ctx->ntxq;
1930 req_ctx->rxqidx = cpu % ctx->nrxq;
1931 put_cpu();
324429d7
HS		 1932
1933 rtfm->init(req);
1934 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
fef4912b
HJ		 1935 error = chcr_inc_wrcount(dev);
1936 if (error)
1937 return -ENXIO;
324429d7 1938
324429d7 1939 if (unlikely(cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
567be3a5
AS		 1940 req_ctx->txqidx) &&
1941 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
fef4912b
HJ		 1942 error = -ENOSPC;
1943 goto err;
324429d7
HS		 1944 }
1945
5110e655 1946 chcr_init_hctx_per_wr(req_ctx);
2f47d580 1947 error = chcr_hash_dma_map(&u_ctx->lldi.pdev->dev, req);
fef4912b
HJ		 1948 if (error) {
1949 error = -ENOMEM;
1950 goto err;
1951 }
324429d7 1952
324429d7 1953 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
5110e655
HJ		 1954 params.kctx_len = roundup(params.alg_prm.result_size, 16);
1955 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1956 params.kctx_len *= 2;
1957 params.opad_needed = 1;
1958 } else {
1959 params.opad_needed = 0;
1960 }
1961 params.sg_len = chcr_hash_ent_in_wr(req->src, !!req_ctx->reqlen,
1962 HASH_SPACE_LEFT(params.kctx_len), 0);
1963 if (params.sg_len < req->nbytes) {
1964 if (is_hmac(crypto_ahash_tfm(rtfm))) {
1965 params.kctx_len /= 2;
1966 params.opad_needed = 0;
1967 }
1968 params.last = 0;
1969 params.more = 1;
1970 params.scmd1 = 0;
1971 params.sg_len = rounddown(params.sg_len, bs);
1972 params.hash_size = params.alg_prm.result_size;
1973 } else {
1974 params.sg_len = req->nbytes;
1975 params.hash_size = crypto_ahash_digestsize(rtfm);
1976 params.last = 1;
1977 params.more = 0;
1978 params.scmd1 = req->nbytes + req_ctx->data_len;
1979
1980 }
1981 params.bfr_len = 0;
1982 req_ctx->hctx_wr.result = 1;
1983 req_ctx->hctx_wr.srcsg = req->src;
324429d7
HS		 1984 req_ctx->data_len += params.bfr_len + params.sg_len;
1985
44fce12a
HJ		 1986 if (req->nbytes == 0) {
1987 create_last_hash_block(req_ctx->reqbfr, bs, 0);
324429d7
HS		 1988 params.more = 1;
1989 params.bfr_len = bs;
1990 }
1991
358961d1 1992 skb = create_hash_wr(req, &params);
2f47d580
HJ		 1993 if (IS_ERR(skb)) {
1994 error = PTR_ERR(skb);
1995 goto unmap;
1996 }
5110e655 1997 req_ctx->hctx_wr.processed += params.sg_len;
324429d7 1998 skb->dev = u_ctx->lldi.ports[0];
567be3a5 1999 set_wr_txq(skb, CPL_PRIORITY_DATA, req_ctx->txqidx);
324429d7 2000 chcr_send_wr(skb);
567be3a5 2001 return -EINPROGRESS;
2f47d580
HJ		 2002unmap:
2003 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
fef4912b
HJ		 2004err:
2005 chcr_dec_wrcount(dev);
2f47d580 2006 return error;
324429d7
HS		 2007}
2008
6f76672b
HJ		 2009static int chcr_ahash_continue(struct ahash_request *req)
2010{
2011 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2012 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
2013 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(req);
567be3a5
AS		 2014 struct chcr_context *ctx = h_ctx(rtfm);
2015 struct uld_ctx *u_ctx = ULD_CTX(ctx);
6f76672b
HJ		 2016 struct sk_buff *skb;
2017 struct hash_wr_param params;
2018 u8 bs;
2019 int error;
567be3a5
AS		 2020 unsigned int cpu;
2021
2022 cpu = get_cpu();
2023 reqctx->txqidx = cpu % ctx->ntxq;
2024 reqctx->rxqidx = cpu % ctx->nrxq;
2025 put_cpu();
6f76672b
HJ		 2026
2027 bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
6f76672b
HJ		 2028 get_alg_config(&params.alg_prm, crypto_ahash_digestsize(rtfm));
2029 params.kctx_len = roundup(params.alg_prm.result_size, 16);
2030 if (is_hmac(crypto_ahash_tfm(rtfm))) {
2031 params.kctx_len *= 2;
2032 params.opad_needed = 1;
2033 } else {
2034 params.opad_needed = 0;
2035 }
2036 params.sg_len = chcr_hash_ent_in_wr(hctx_wr->srcsg, 0,
2037 HASH_SPACE_LEFT(params.kctx_len),
2038 hctx_wr->src_ofst);
2039 if ((params.sg_len + hctx_wr->processed) > req->nbytes)
2040 params.sg_len = req->nbytes - hctx_wr->processed;
2041 if (!hctx_wr->result ||
2042 ((params.sg_len + hctx_wr->processed) < req->nbytes)) {
2043 if (is_hmac(crypto_ahash_tfm(rtfm))) {
2044 params.kctx_len /= 2;
2045 params.opad_needed = 0;
2046 }
2047 params.last = 0;
2048 params.more = 1;
2049 params.sg_len = rounddown(params.sg_len, bs);
2050 params.hash_size = params.alg_prm.result_size;
2051 params.scmd1 = 0;
2052 } else {
2053 params.last = 1;
2054 params.more = 0;
2055 params.hash_size = crypto_ahash_digestsize(rtfm);
2056 params.scmd1 = reqctx->data_len + params.sg_len;
2057 }
2058 params.bfr_len = 0;
2059 reqctx->data_len += params.sg_len;
2060 skb = create_hash_wr(req, &params);
2061 if (IS_ERR(skb)) {
2062 error = PTR_ERR(skb);
2063 goto err;
2064 }
2065 hctx_wr->processed += params.sg_len;
2066 skb->dev = u_ctx->lldi.ports[0];
567be3a5 2067 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
6f76672b
HJ		 2068 chcr_send_wr(skb);
2069 return 0;
2070err:
2071 return error;
2072}
2073
2074static inline void chcr_handle_ahash_resp(struct ahash_request *req,
2075 unsigned char *input,
2076 int err)
2077{
2078 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2079 struct chcr_hctx_per_wr *hctx_wr = &reqctx->hctx_wr;
2080 int digestsize, updated_digestsize;
2081 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
2082 struct uld_ctx *u_ctx = ULD_CTX(h_ctx(tfm));
fef4912b 2083 struct chcr_dev *dev = h_ctx(tfm)->dev;
6f76672b
HJ		 2084
2085 if (input == NULL)
2086 goto out;
2087 digestsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(req));
2088 updated_digestsize = digestsize;
2089 if (digestsize == SHA224_DIGEST_SIZE)
2090 updated_digestsize = SHA256_DIGEST_SIZE;
2091 else if (digestsize == SHA384_DIGEST_SIZE)
2092 updated_digestsize = SHA512_DIGEST_SIZE;
2093
2094 if (hctx_wr->dma_addr) {
2095 dma_unmap_single(&u_ctx->lldi.pdev->dev, hctx_wr->dma_addr,
2096 hctx_wr->dma_len, DMA_TO_DEVICE);
2097 hctx_wr->dma_addr = 0;
2098 }
2099 if (hctx_wr->isfinal || ((hctx_wr->processed + reqctx->reqlen) ==
2100 req->nbytes)) {
2101 if (hctx_wr->result == 1) {
2102 hctx_wr->result = 0;
2103 memcpy(req->result, input + sizeof(struct cpl_fw6_pld),
2104 digestsize);
2105 } else {
2106 memcpy(reqctx->partial_hash,
2107 input + sizeof(struct cpl_fw6_pld),
2108 updated_digestsize);
2109
2110 }
2111 goto unmap;
2112 }
2113 memcpy(reqctx->partial_hash, input + sizeof(struct cpl_fw6_pld),
2114 updated_digestsize);
2115
2116 err = chcr_ahash_continue(req);
2117 if (err)
2118 goto unmap;
2119 return;
2120unmap:
2121 if (hctx_wr->is_sg_map)
2122 chcr_hash_dma_unmap(&u_ctx->lldi.pdev->dev, req);
2123
2124
2125out:
fef4912b 2126 chcr_dec_wrcount(dev);
6f76672b
HJ		 2127 req->base.complete(&req->base, err);
2128}
2129
2130/*
2131 * chcr_handle_resp - Unmap the DMA buffers associated with the request
2132 * @req: crypto request
2133 */
2134int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input,
2135 int err)
2136{
2137 struct crypto_tfm *tfm = req->tfm;
2138 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2139 struct adapter *adap = padap(ctx->dev);
2140
2141 switch (tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK) {
2142 case CRYPTO_ALG_TYPE_AEAD:
f31ba0f9 2143 err = chcr_handle_aead_resp(aead_request_cast(req), input, err);
6f76672b
HJ		 2144 break;
2145
7cea6d3e
AB		 2146 case CRYPTO_ALG_TYPE_SKCIPHER:
2147 chcr_handle_cipher_resp(skcipher_request_cast(req),
6f76672b
HJ		 2148 input, err);
2149 break;
6f76672b
HJ		 2150 case CRYPTO_ALG_TYPE_AHASH:
2151 chcr_handle_ahash_resp(ahash_request_cast(req), input, err);
2152 }
2153 atomic_inc(&adap->chcr_stats.complete);
2154 return err;
2155}
324429d7
HS		 2156static int chcr_ahash_export(struct ahash_request *areq, void *out)
2157{
2158 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2159 struct chcr_ahash_req_ctx *state = out;
2160
44fce12a 2161 state->reqlen = req_ctx->reqlen;
324429d7 2162 state->data_len = req_ctx->data_len;
44fce12a 2163 memcpy(state->bfr1, req_ctx->reqbfr, req_ctx->reqlen);
324429d7
HS		 2164 memcpy(state->partial_hash, req_ctx->partial_hash,
2165 CHCR_HASH_MAX_DIGEST_SIZE);
5110e655 2166 chcr_init_hctx_per_wr(state);
fc6176a2 2167 return 0;
324429d7
HS		 2168}
2169
2170static int chcr_ahash_import(struct ahash_request *areq, const void *in)
2171{
2172 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2173 struct chcr_ahash_req_ctx *state = (struct chcr_ahash_req_ctx *)in;
2174
44fce12a 2175 req_ctx->reqlen = state->reqlen;
324429d7 2176 req_ctx->data_len = state->data_len;
44fce12a
HJ		 2177 req_ctx->reqbfr = req_ctx->bfr1;
2178 req_ctx->skbfr = req_ctx->bfr2;
2179 memcpy(req_ctx->bfr1, state->bfr1, CHCR_HASH_MAX_BLOCK_SIZE_128);
324429d7
HS		 2180 memcpy(req_ctx->partial_hash, state->partial_hash,
2181 CHCR_HASH_MAX_DIGEST_SIZE);
5110e655 2182 chcr_init_hctx_per_wr(req_ctx);
324429d7
HS		 2183 return 0;
2184}
2185
2186static int chcr_ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2187 unsigned int keylen)
2188{
2f47d580 2189 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(tfm));
324429d7
HS		 2190 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2191 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2192 unsigned int i, err = 0, updated_digestsize;
2193
e7922729
HJ		 2194 SHASH_DESC_ON_STACK(shash, hmacctx->base_hash);
2195
2196 /* use the key to calculate the ipad and opad. ipad will sent with the
324429d7
HS		 2197 * first request's data. opad will be sent with the final hash result
2198 * ipad in hmacctx->ipad and opad in hmacctx->opad location
2199 */
e7922729 2200 shash->tfm = hmacctx->base_hash;
324429d7 2201 if (keylen > bs) {
e7922729 2202 err = crypto_shash_digest(shash, key, keylen,
324429d7
HS		 2203 hmacctx->ipad);
2204 if (err)
2205 goto out;
2206 keylen = digestsize;
2207 } else {
2208 memcpy(hmacctx->ipad, key, keylen);
2209 }
2210 memset(hmacctx->ipad + keylen, 0, bs - keylen);
2211 memcpy(hmacctx->opad, hmacctx->ipad, bs);
2212
2213 for (i = 0; i < bs / sizeof(int); i++) {
2214 *((unsigned int *)(&hmacctx->ipad) + i) ^= IPAD_DATA;
2215 *((unsigned int *)(&hmacctx->opad) + i) ^= OPAD_DATA;
2216 }
2217
2218 updated_digestsize = digestsize;
2219 if (digestsize == SHA224_DIGEST_SIZE)
2220 updated_digestsize = SHA256_DIGEST_SIZE;
2221 else if (digestsize == SHA384_DIGEST_SIZE)
2222 updated_digestsize = SHA512_DIGEST_SIZE;
e7922729 2223 err = chcr_compute_partial_hash(shash, hmacctx->ipad,
324429d7
HS		 2224 hmacctx->ipad, digestsize);
2225 if (err)
2226 goto out;
2227 chcr_change_order(hmacctx->ipad, updated_digestsize);
2228
e7922729 2229 err = chcr_compute_partial_hash(shash, hmacctx->opad,
324429d7
HS		 2230 hmacctx->opad, digestsize);
2231 if (err)
2232 goto out;
2233 chcr_change_order(hmacctx->opad, updated_digestsize);
2234out:
2235 return err;
2236}
2237
7cea6d3e 2238static int chcr_aes_xts_setkey(struct crypto_skcipher *cipher, const u8 *key,
324429d7
HS		 2239 unsigned int key_len)
2240{
2f47d580 2241 struct ablk_ctx *ablkctx = ABLK_CTX(c_ctx(cipher));
324429d7 2242 unsigned short context_size = 0;
b8fd1f41 2243 int err;
324429d7 2244
b8fd1f41
HJ		 2245 err = chcr_cipher_fallback_setkey(cipher, key, key_len);
2246 if (err)
2247 goto badkey_err;
cc1b156d
HJ		 2248
2249 memcpy(ablkctx->key, key, key_len);
2250 ablkctx->enckey_len = key_len;
2251 get_aes_decrypt_key(ablkctx->rrkey, ablkctx->key, key_len << 2);
2252 context_size = (KEY_CONTEXT_HDR_SALT_AND_PAD + key_len) >> 4;
2253 ablkctx->key_ctx_hdr =
2254 FILL_KEY_CTX_HDR((key_len == AES_KEYSIZE_256) ?
2255 CHCR_KEYCTX_CIPHER_KEY_SIZE_128 :
2256 CHCR_KEYCTX_CIPHER_KEY_SIZE_256,
2257 CHCR_KEYCTX_NO_KEY, 1,
2258 0, context_size);
2259 ablkctx->ciph_mode = CHCR_SCMD_CIPHER_MODE_AES_XTS;
2260 return 0;
b8fd1f41 2261badkey_err:
b8fd1f41
HJ		 2262 ablkctx->enckey_len = 0;
2263
2264 return err;
324429d7
HS		 2265}
2266
2267static int chcr_sha_init(struct ahash_request *areq)
2268{
2269 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2270 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2271 int digestsize = crypto_ahash_digestsize(tfm);
2272
2273 req_ctx->data_len = 0;
44fce12a
HJ		 2274 req_ctx->reqlen = 0;
2275 req_ctx->reqbfr = req_ctx->bfr1;
2276 req_ctx->skbfr = req_ctx->bfr2;
324429d7 2277 copy_hash_init_values(req_ctx->partial_hash, digestsize);
5110e655 2278
324429d7
HS		 2279 return 0;
2280}
2281
2282static int chcr_sha_cra_init(struct crypto_tfm *tfm)
2283{
2284 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2285 sizeof(struct chcr_ahash_req_ctx));
2286 return chcr_device_init(crypto_tfm_ctx(tfm));
2287}
2288
2289static int chcr_hmac_init(struct ahash_request *areq)
2290{
2291 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2292 struct crypto_ahash *rtfm = crypto_ahash_reqtfm(areq);
2f47d580 2293 struct hmac_ctx *hmacctx = HMAC_CTX(h_ctx(rtfm));
324429d7
HS		 2294 unsigned int digestsize = crypto_ahash_digestsize(rtfm);
2295 unsigned int bs = crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm));
2296
2297 chcr_sha_init(areq);
2298 req_ctx->data_len = bs;
2299 if (is_hmac(crypto_ahash_tfm(rtfm))) {
2300 if (digestsize == SHA224_DIGEST_SIZE)
2301 memcpy(req_ctx->partial_hash, hmacctx->ipad,
2302 SHA256_DIGEST_SIZE);
2303 else if (digestsize == SHA384_DIGEST_SIZE)
2304 memcpy(req_ctx->partial_hash, hmacctx->ipad,
2305 SHA512_DIGEST_SIZE);
2306 else
2307 memcpy(req_ctx->partial_hash, hmacctx->ipad,
2308 digestsize);
2309 }
2310 return 0;
2311}
2312
2313static int chcr_hmac_cra_init(struct crypto_tfm *tfm)
2314{
2315 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2316 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
2317 unsigned int digestsize =
2318 crypto_ahash_digestsize(__crypto_ahash_cast(tfm));
2319
2320 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
2321 sizeof(struct chcr_ahash_req_ctx));
e7922729
HJ		 2322 hmacctx->base_hash = chcr_alloc_shash(digestsize);
2323 if (IS_ERR(hmacctx->base_hash))
2324 return PTR_ERR(hmacctx->base_hash);
324429d7
HS		 2325 return chcr_device_init(crypto_tfm_ctx(tfm));
2326}
2327
324429d7
HS		 2328static void chcr_hmac_cra_exit(struct crypto_tfm *tfm)
2329{
2330 struct chcr_context *ctx = crypto_tfm_ctx(tfm);
2331 struct hmac_ctx *hmacctx = HMAC_CTX(ctx);
2332
e7922729
HJ		 2333 if (hmacctx->base_hash) {
2334 chcr_free_shash(hmacctx->base_hash);
2335 hmacctx->base_hash = NULL;
324429d7
HS		 2336 }
2337}
2338
4262c98a
HJ		 2339inline void chcr_aead_common_exit(struct aead_request *req)
2340{
2341 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2342 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2343 struct uld_ctx *u_ctx = ULD_CTX(a_ctx(tfm));
2344
2345 chcr_aead_dma_unmap(&u_ctx->lldi.pdev->dev, req, reqctx->op);
2346}
2347
2348static int chcr_aead_common_init(struct aead_request *req)
2debd332 2349{
2f47d580
HJ		 2350 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2351 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2352 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2f47d580 2353 unsigned int authsize = crypto_aead_authsize(tfm);
4262c98a 2354 int error = -EINVAL;
2debd332 2355
2f47d580
HJ		 2356 /* validate key size */
2357 if (aeadctx->enckey_len == 0)
2358 goto err;
4262c98a 2359 if (reqctx->op && req->cryptlen < authsize)
2f47d580 2360 goto err;
4262c98a
HJ		 2361 if (reqctx->b0_len)
2362 reqctx->scratch_pad = reqctx->iv + IV;
2363 else
2364 reqctx->scratch_pad = NULL;
2365
2f47d580 2366 error = chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm))->lldi.pdev->dev, req,
4262c98a 2367 reqctx->op);
2f47d580
HJ		 2368 if (error) {
2369 error = -ENOMEM;
2370 goto err;
2371 }
1f479e4c 2372
2f47d580
HJ		 2373 return 0;
2374err:
2375 return error;
2debd332 2376}
2f47d580
HJ		 2377
2378static int chcr_aead_need_fallback(struct aead_request *req, int dst_nents,
0e93708d
HJ		 2379 int aadmax, int wrlen,
2380 unsigned short op_type)
2381{
2382 unsigned int authsize = crypto_aead_authsize(crypto_aead_reqtfm(req));
2383
2384 if (((req->cryptlen - (op_type ? authsize : 0)) == 0) ||
2f47d580 2385 dst_nents > MAX_DSGL_ENT ||
0e93708d 2386 (req->assoclen > aadmax) ||
2f47d580 2387 (wrlen > SGE_MAX_WR_LEN))
0e93708d
HJ		 2388 return 1;
2389 return 0;
2390}
2debd332 2391
0e93708d
HJ		 2392static int chcr_aead_fallback(struct aead_request *req, unsigned short op_type)
2393{
2394 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2f47d580 2395 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
0e93708d
HJ		 2396 struct aead_request *subreq = aead_request_ctx(req);
2397
2398 aead_request_set_tfm(subreq, aeadctx->sw_cipher);
2399 aead_request_set_callback(subreq, req->base.flags,
2400 req->base.complete, req->base.data);
4262c98a 2401 aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen,
0e93708d 2402 req->iv);
fc6176a2 2403 aead_request_set_ad(subreq, req->assoclen);
0e93708d
HJ		 2404 return op_type ? crypto_aead_decrypt(subreq) :
2405 crypto_aead_encrypt(subreq);
2406}
2debd332
HJ		 2407
2408static struct sk_buff *create_authenc_wr(struct aead_request *req,
2409 unsigned short qid,
4262c98a 2410 int size)
2debd332
HJ		 2411{
2412 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
567be3a5
AS		 2413 struct chcr_context *ctx = a_ctx(tfm);
2414 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2debd332
HJ		 2415 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
2416 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2417 struct sk_buff *skb = NULL;
2418 struct chcr_wr *chcr_req;
2419 struct cpl_rx_phys_dsgl *phys_cpl;
2f47d580
HJ		 2420 struct ulptx_sgl *ulptx;
2421 unsigned int transhdr_len;
3d64bd67 2422 unsigned int dst_size = 0, temp, subtype = get_aead_subtype(tfm);
1f479e4c 2423 unsigned int kctx_len = 0, dnents, snents;
2debd332 2424 unsigned int authsize = crypto_aead_authsize(tfm);
2f47d580 2425 int error = -EINVAL;
1f479e4c 2426 u8 *ivptr;
2debd332
HJ		 2427 int null = 0;
2428 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2429 GFP_ATOMIC;
567be3a5
AS		 2430 struct adapter *adap = padap(ctx->dev);
2431 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2debd332 2432
2f47d580
HJ		 2433 if (req->cryptlen == 0)
2434 return NULL;
2debd332 2435
4262c98a
HJ		 2436 reqctx->b0_len = 0;
2437 error = chcr_aead_common_init(req);
2438 if (error)
2439 return ERR_PTR(error);
2440
3d64bd67 2441 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL ||
4262c98a 2442 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
2debd332 2443 null = 1;
2debd332 2444 }
1f479e4c
HJ		 2445 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen +
2446 (reqctx->op ? -authsize : authsize), CHCR_DST_SG_SIZE, 0);
5abc8db0 2447 dnents += MIN_AUTH_SG; // For IV
1f479e4c
HJ		 2448 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
2449 CHCR_SRC_SG_SIZE, 0);
2f47d580 2450 dst_size = get_space_for_phys_dsgl(dnents);
ff462ddf 2451 kctx_len = (KEY_CONTEXT_CTX_LEN_G(ntohl(aeadctx->key_ctx_hdr)) << 4)
2debd332
HJ		 2452 - sizeof(chcr_req->key_ctx);
2453 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
1f479e4c 2454 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <
2f47d580 2455 SGE_MAX_WR_LEN;
1f479e4c
HJ		 2456 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16)
2457 : (sgl_len(snents) * 8);
2f47d580 2458 transhdr_len += temp;
125d01ca 2459 transhdr_len = roundup(transhdr_len, 16);
2f47d580
HJ		 2460
2461 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
4262c98a 2462 transhdr_len, reqctx->op)) {
ee0863ba 2463 atomic_inc(&adap->chcr_stats.fallback);
4262c98a
HJ		 2464 chcr_aead_common_exit(req);
2465 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
0e93708d 2466 }
1f479e4c 2467 skb = alloc_skb(transhdr_len, flags);
5fe8c711
HJ		 2468 if (!skb) {
2469 error = -ENOMEM;
2debd332 2470 goto err;
5fe8c711 2471 }
2debd332 2472
de77b966 2473 chcr_req = __skb_put_zero(skb, transhdr_len);
2debd332 2474
4262c98a 2475 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
2debd332
HJ		 2476
2477 /*
2478 * Input order is AAD,IV and Payload. where IV should be included as
2479 * the part of authdata. All other fields should be filled according
2480 * to the hardware spec
2481 */
2482 chcr_req->sec_cpl.op_ivinsrtofst =
567be3a5 2483 FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
1f479e4c 2484 chcr_req->sec_cpl.pldlen = htonl(req->assoclen + IV + req->cryptlen);
2debd332 2485 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
1f479e4c
HJ		 2486 null ? 0 : 1 + IV,
2487 null ? 0 : IV + req->assoclen,
2488 req->assoclen + IV + 1,
2f47d580 2489 (temp & 0x1F0) >> 4);
2debd332 2490 chcr_req->sec_cpl.cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(
2f47d580 2491 temp & 0xF,
1f479e4c 2492 null ? 0 : req->assoclen + IV + 1,
2f47d580 2493 temp, temp);
3d64bd67
HJ		 2494 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL ||
2495 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA)
2496 temp = CHCR_SCMD_CIPHER_MODE_AES_CTR;
2497 else
2498 temp = CHCR_SCMD_CIPHER_MODE_AES_CBC;
4262c98a
HJ		 2499 chcr_req->sec_cpl.seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op,
2500 (reqctx->op == CHCR_ENCRYPT_OP) ? 1 : 0,
3d64bd67 2501 temp,
2debd332 2502 actx->auth_mode, aeadctx->hmac_ctrl,
2f47d580 2503 IV >> 1);
2debd332 2504 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2f47d580 2505 0, 0, dst_size);
2debd332
HJ		 2506
2507 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
4262c98a 2508 if (reqctx->op == CHCR_ENCRYPT_OP ||
3d64bd67
HJ		 2509 subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2510 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL)
2debd332
HJ		 2511 memcpy(chcr_req->key_ctx.key, aeadctx->key,
2512 aeadctx->enckey_len);
2513 else
2514 memcpy(chcr_req->key_ctx.key, actx->dec_rrkey,
2515 aeadctx->enckey_len);
2516
125d01ca
HJ		 2517 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
2518 actx->h_iopad, kctx_len - roundup(aeadctx->enckey_len, 16));
1f479e4c
HJ		 2519 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
2520 ivptr = (u8 *)(phys_cpl + 1) + dst_size;
2521 ulptx = (struct ulptx_sgl *)(ivptr + IV);
3d64bd67
HJ		 2522 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
2523 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
1f479e4c
HJ		 2524 memcpy(ivptr, aeadctx->nonce, CTR_RFC3686_NONCE_SIZE);
2525 memcpy(ivptr + CTR_RFC3686_NONCE_SIZE, req->iv,
3d64bd67 2526 CTR_RFC3686_IV_SIZE);
1f479e4c 2527 *(__be32 *)(ivptr + CTR_RFC3686_NONCE_SIZE +
3d64bd67
HJ		 2528 CTR_RFC3686_IV_SIZE) = cpu_to_be32(1);
2529 } else {
1f479e4c 2530 memcpy(ivptr, req->iv, IV);
3d64bd67 2531 }
1f479e4c
HJ		 2532 chcr_add_aead_dst_ent(req, phys_cpl, qid);
2533 chcr_add_aead_src_ent(req, ulptx);
ee0863ba 2534 atomic_inc(&adap->chcr_stats.cipher_rqst);
1f479e4c
HJ		 2535 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
2536 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0);
2f47d580
HJ		 2537 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
2538 transhdr_len, temp, 0);
2debd332 2539 reqctx->skb = skb;
2debd332
HJ		 2540
2541 return skb;
2debd332 2542err:
4262c98a 2543 chcr_aead_common_exit(req);
2f47d580 2544
5fe8c711 2545 return ERR_PTR(error);
2debd332
HJ		 2546}
2547
6dad4e8a
AG		 2548int chcr_aead_dma_map(struct device *dev,
2549 struct aead_request *req,
2550 unsigned short op_type)
2f47d580
HJ		 2551{
2552 int error;
2553 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2554 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2555 unsigned int authsize = crypto_aead_authsize(tfm);
2556 int dst_size;
2557
2558 dst_size = req->assoclen + req->cryptlen + (op_type ?
d91a3159 2559 0 : authsize);
2f47d580
HJ		 2560 if (!req->cryptlen || !dst_size)
2561 return 0;
4262c98a 2562 reqctx->iv_dma = dma_map_single(dev, reqctx->iv, (IV + reqctx->b0_len),
2f47d580
HJ		 2563 DMA_BIDIRECTIONAL);
2564 if (dma_mapping_error(dev, reqctx->iv_dma))
2565 return -ENOMEM;
4262c98a
HJ		 2566 if (reqctx->b0_len)
2567 reqctx->b0_dma = reqctx->iv_dma + IV;
2568 else
2569 reqctx->b0_dma = 0;
2f47d580 2570 if (req->src == req->dst) {
9195189e
AS		 2571 error = dma_map_sg(dev, req->src,
2572 sg_nents_for_len(req->src, dst_size),
2573 DMA_BIDIRECTIONAL);
2f47d580
HJ		 2574 if (!error)
2575 goto err;
2576 } else {
2577 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2578 DMA_TO_DEVICE);
2579 if (!error)
2580 goto err;
2581 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2582 DMA_FROM_DEVICE);
2583 if (!error) {
2584 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2585 DMA_TO_DEVICE);
2586 goto err;
2587 }
2588 }
2589
2590 return 0;
2591err:
2592 dma_unmap_single(dev, reqctx->iv_dma, IV, DMA_BIDIRECTIONAL);
2593 return -ENOMEM;
2594}
2595
6dad4e8a
AG		 2596void chcr_aead_dma_unmap(struct device *dev,
2597 struct aead_request *req,
2598 unsigned short op_type)
2f47d580
HJ		 2599{
2600 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2601 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2602 unsigned int authsize = crypto_aead_authsize(tfm);
2603 int dst_size;
2604
2605 dst_size = req->assoclen + req->cryptlen + (op_type ?
d91a3159 2606 0 : authsize);
2f47d580
HJ		 2607 if (!req->cryptlen || !dst_size)
2608 return;
2609
4262c98a 2610 dma_unmap_single(dev, reqctx->iv_dma, (IV + reqctx->b0_len),
2f47d580
HJ		 2611 DMA_BIDIRECTIONAL);
2612 if (req->src == req->dst) {
d91a3159
DSK		 2613 dma_unmap_sg(dev, req->src,
2614 sg_nents_for_len(req->src, dst_size),
2615 DMA_BIDIRECTIONAL);
2f47d580
HJ		 2616 } else {
2617 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2618 DMA_TO_DEVICE);
2619 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2620 DMA_FROM_DEVICE);
2621 }
2622}
2623
6dad4e8a 2624void chcr_add_aead_src_ent(struct aead_request *req,
1f479e4c 2625 struct ulptx_sgl *ulptx)
2f47d580
HJ		 2626{
2627 struct ulptx_walk ulp_walk;
2628 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2629
2630 if (reqctx->imm) {
2631 u8 *buf = (u8 *)ulptx;
2632
4262c98a 2633 if (reqctx->b0_len) {
2f47d580
HJ		 2634 memcpy(buf, reqctx->scratch_pad, reqctx->b0_len);
2635 buf += reqctx->b0_len;
2636 }
2637 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
1f479e4c 2638 buf, req->cryptlen + req->assoclen, 0);
2f47d580
HJ		 2639 } else {
2640 ulptx_walk_init(&ulp_walk, ulptx);
4262c98a 2641 if (reqctx->b0_len)
2f47d580 2642 ulptx_walk_add_page(&ulp_walk, reqctx->b0_len,
c4f6d44d 2643 reqctx->b0_dma);
1f479e4c
HJ		 2644 ulptx_walk_add_sg(&ulp_walk, req->src, req->cryptlen +
2645 req->assoclen, 0);
2f47d580
HJ		 2646 ulptx_walk_end(&ulp_walk);
2647 }
2648}
2649
6dad4e8a
AG		 2650void chcr_add_aead_dst_ent(struct aead_request *req,
2651 struct cpl_rx_phys_dsgl *phys_cpl,
6dad4e8a 2652 unsigned short qid)
2f47d580
HJ		 2653{
2654 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2655 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2656 struct dsgl_walk dsgl_walk;
2657 unsigned int authsize = crypto_aead_authsize(tfm);
add92a81 2658 struct chcr_context *ctx = a_ctx(tfm);
2f47d580 2659 u32 temp;
567be3a5 2660 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2f47d580
HJ		 2661
2662 dsgl_walk_init(&dsgl_walk, phys_cpl);
c4f6d44d 2663 dsgl_walk_add_page(&dsgl_walk, IV + reqctx->b0_len, reqctx->iv_dma);
1f479e4c
HJ		 2664 temp = req->assoclen + req->cryptlen +
2665 (reqctx->op ? -authsize : authsize);
2666 dsgl_walk_add_sg(&dsgl_walk, req->dst, temp, 0);
567be3a5 2667 dsgl_walk_end(&dsgl_walk, qid, rx_channel_id);
2f47d580
HJ		 2668}
2669
7cea6d3e 2670void chcr_add_cipher_src_ent(struct skcipher_request *req,
335bcc4a 2671 void *ulptx,
6dad4e8a 2672 struct cipher_wr_param *wrparam)
2f47d580
HJ		 2673{
2674 struct ulptx_walk ulp_walk;
7cea6d3e 2675 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
335bcc4a 2676 u8 *buf = ulptx;
2f47d580 2677
335bcc4a
HJ		 2678 memcpy(buf, reqctx->iv, IV);
2679 buf += IV;
2f47d580 2680 if (reqctx->imm) {
2f47d580
HJ		 2681 sg_pcopy_to_buffer(req->src, sg_nents(req->src),
2682 buf, wrparam->bytes, reqctx->processed);
2683 } else {
335bcc4a 2684 ulptx_walk_init(&ulp_walk, (struct ulptx_sgl *)buf);
2f47d580
HJ		 2685 ulptx_walk_add_sg(&ulp_walk, reqctx->srcsg, wrparam->bytes,
2686 reqctx->src_ofst);
2687 reqctx->srcsg = ulp_walk.last_sg;
2688 reqctx->src_ofst = ulp_walk.last_sg_len;
2689 ulptx_walk_end(&ulp_walk);
2690 }
2691}
2692
7cea6d3e 2693void chcr_add_cipher_dst_ent(struct skcipher_request *req,
6dad4e8a
AG		 2694 struct cpl_rx_phys_dsgl *phys_cpl,
2695 struct cipher_wr_param *wrparam,
2696 unsigned short qid)
2f47d580 2697{
7cea6d3e
AB		 2698 struct chcr_skcipher_req_ctx *reqctx = skcipher_request_ctx(req);
2699 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(wrparam->req);
add92a81 2700 struct chcr_context *ctx = c_ctx(tfm);
2f47d580 2701 struct dsgl_walk dsgl_walk;
567be3a5 2702 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2f47d580
HJ		 2703
2704 dsgl_walk_init(&dsgl_walk, phys_cpl);
2f47d580
HJ		 2705 dsgl_walk_add_sg(&dsgl_walk, reqctx->dstsg, wrparam->bytes,
2706 reqctx->dst_ofst);
2707 reqctx->dstsg = dsgl_walk.last_sg;
2708 reqctx->dst_ofst = dsgl_walk.last_sg_len;
567be3a5 2709 dsgl_walk_end(&dsgl_walk, qid, rx_channel_id);
2f47d580
HJ		 2710}
2711
6dad4e8a
AG		 2712void chcr_add_hash_src_ent(struct ahash_request *req,
2713 struct ulptx_sgl *ulptx,
2714 struct hash_wr_param *param)
2f47d580
HJ		 2715{
2716 struct ulptx_walk ulp_walk;
2717 struct chcr_ahash_req_ctx *reqctx = ahash_request_ctx(req);
2718
5110e655 2719 if (reqctx->hctx_wr.imm) {
2f47d580
HJ		 2720 u8 *buf = (u8 *)ulptx;
2721
2722 if (param->bfr_len) {
2723 memcpy(buf, reqctx->reqbfr, param->bfr_len);
2724 buf += param->bfr_len;
2725 }
5110e655
HJ		 2726
2727 sg_pcopy_to_buffer(reqctx->hctx_wr.srcsg,
2728 sg_nents(reqctx->hctx_wr.srcsg), buf,
2729 param->sg_len, 0);
2f47d580
HJ		 2730 } else {
2731 ulptx_walk_init(&ulp_walk, ulptx);
2732 if (param->bfr_len)
2733 ulptx_walk_add_page(&ulp_walk, param->bfr_len,
c4f6d44d 2734 reqctx->hctx_wr.dma_addr);
5110e655
HJ		 2735 ulptx_walk_add_sg(&ulp_walk, reqctx->hctx_wr.srcsg,
2736 param->sg_len, reqctx->hctx_wr.src_ofst);
2737 reqctx->hctx_wr.srcsg = ulp_walk.last_sg;
2738 reqctx->hctx_wr.src_ofst = ulp_walk.last_sg_len;
db6deea4 2739 ulptx_walk_end(&ulp_walk);
2f47d580
HJ		 2740 }
2741}
2742
6dad4e8a
AG		 2743int chcr_hash_dma_map(struct device *dev,
2744 struct ahash_request *req)
2f47d580
HJ		 2745{
2746 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2747 int error = 0;
2748
2749 if (!req->nbytes)
2750 return 0;
2751 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2752 DMA_TO_DEVICE);
2753 if (!error)
7814f552 2754 return -ENOMEM;
5110e655 2755 req_ctx->hctx_wr.is_sg_map = 1;
2f47d580
HJ		 2756 return 0;
2757}
2758
6dad4e8a
AG		 2759void chcr_hash_dma_unmap(struct device *dev,
2760 struct ahash_request *req)
2f47d580
HJ		 2761{
2762 struct chcr_ahash_req_ctx *req_ctx = ahash_request_ctx(req);
2763
2764 if (!req->nbytes)
2765 return;
2766
2767 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2768 DMA_TO_DEVICE);
5110e655 2769 req_ctx->hctx_wr.is_sg_map = 0;
2f47d580
HJ		 2770
2771}
2772
6dad4e8a 2773int chcr_cipher_dma_map(struct device *dev,
7cea6d3e 2774 struct skcipher_request *req)
2f47d580
HJ		 2775{
2776 int error;
2f47d580
HJ		 2777
2778 if (req->src == req->dst) {
2779 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2780 DMA_BIDIRECTIONAL);
2781 if (!error)
2782 goto err;
2783 } else {
2784 error = dma_map_sg(dev, req->src, sg_nents(req->src),
2785 DMA_TO_DEVICE);
2786 if (!error)
2787 goto err;
2788 error = dma_map_sg(dev, req->dst, sg_nents(req->dst),
2789 DMA_FROM_DEVICE);
2790 if (!error) {
2791 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2792 DMA_TO_DEVICE);
2793 goto err;
2794 }
2795 }
2796
2797 return 0;
2798err:
2f47d580
HJ		 2799 return -ENOMEM;
2800}
6dad4e8a
AG		 2801
2802void chcr_cipher_dma_unmap(struct device *dev,
7cea6d3e 2803 struct skcipher_request *req)
2f47d580 2804{
2f47d580
HJ		 2805 if (req->src == req->dst) {
2806 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2807 DMA_BIDIRECTIONAL);
2808 } else {
2809 dma_unmap_sg(dev, req->src, sg_nents(req->src),
2810 DMA_TO_DEVICE);
2811 dma_unmap_sg(dev, req->dst, sg_nents(req->dst),
2812 DMA_FROM_DEVICE);
2813 }
2814}
2815
2debd332
HJ		 2816static int set_msg_len(u8 *block, unsigned int msglen, int csize)
2817{
2818 __be32 data;
2819
2820 memset(block, 0, csize);
2821 block += csize;
2822
2823 if (csize >= 4)
2824 csize = 4;
2825 else if (msglen > (unsigned int)(1 << (8 * csize)))
2826 return -EOVERFLOW;
2827
2828 data = cpu_to_be32(msglen);
2829 memcpy(block - csize, (u8 *)&data + 4 - csize, csize);
2830
2831 return 0;
2832}
2833
66af86d9 2834static int generate_b0(struct aead_request *req, u8 *ivptr,
2debd332
HJ		 2835 unsigned short op_type)
2836{
2837 unsigned int l, lp, m;
2838 int rc;
2839 struct crypto_aead *aead = crypto_aead_reqtfm(req);
2840 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2841 u8 *b0 = reqctx->scratch_pad;
2842
2843 m = crypto_aead_authsize(aead);
2844
1f479e4c 2845 memcpy(b0, ivptr, 16);
2debd332
HJ		 2846
2847 lp = b0[0];
2848 l = lp + 1;
2849
2850 /* set m, bits 3-5 */
2851 *b0 |= (8 * ((m - 2) / 2));
2852
2853 /* set adata, bit 6, if associated data is used */
2854 if (req->assoclen)
2855 *b0 |= 64;
2856 rc = set_msg_len(b0 + 16 - l,
2857 (op_type == CHCR_DECRYPT_OP) ?
2858 req->cryptlen - m : req->cryptlen, l);
66af86d9
Y		 2859
2860 return rc;
2debd332
HJ		 2861}
2862
2863static inline int crypto_ccm_check_iv(const u8 *iv)
2864{
2865 /* 2 <= L <= 8, so 1 <= L' <= 7. */
2866 if (iv[0] < 1 || iv[0] > 7)
2867 return -EINVAL;
2868
2869 return 0;
2870}
2871
2872static int ccm_format_packet(struct aead_request *req,
1f479e4c 2873 u8 *ivptr,
2debd332 2874 unsigned int sub_type,
4262c98a
HJ		 2875 unsigned short op_type,
2876 unsigned int assoclen)
2debd332
HJ		 2877{
2878 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
1f479e4c
HJ		 2879 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2880 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 2881 int rc = 0;
2882
2debd332 2883 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
1f479e4c
HJ		 2884 ivptr[0] = 3;
2885 memcpy(ivptr + 1, &aeadctx->salt[0], 3);
2886 memcpy(ivptr + 4, req->iv, 8);
2887 memset(ivptr + 12, 0, 4);
2debd332 2888 } else {
1f479e4c 2889 memcpy(ivptr, req->iv, 16);
2debd332 2890 }
4262c98a
HJ		 2891 if (assoclen)
2892 *((unsigned short *)(reqctx->scratch_pad + 16)) =
2893 htons(assoclen);
2894
66af86d9 2895 rc = generate_b0(req, ivptr, op_type);
2debd332 2896 /* zero the ctr value */
1f479e4c 2897 memset(ivptr + 15 - ivptr[0], 0, ivptr[0] + 1);
2debd332
HJ		 2898 return rc;
2899}
2900
2901static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu *sec_cpl,
2902 unsigned int dst_size,
2903 struct aead_request *req,
2f47d580 2904 unsigned short op_type)
2debd332
HJ		 2905{
2906 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
567be3a5
AS		 2907 struct chcr_context *ctx = a_ctx(tfm);
2908 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2909 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2debd332
HJ		 2910 unsigned int cipher_mode = CHCR_SCMD_CIPHER_MODE_AES_CCM;
2911 unsigned int mac_mode = CHCR_SCMD_AUTH_MODE_CBCMAC;
567be3a5 2912 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2debd332
HJ		 2913 unsigned int ccm_xtra;
2914 unsigned char tag_offset = 0, auth_offset = 0;
2debd332
HJ		 2915 unsigned int assoclen;
2916
2917 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2918 assoclen = req->assoclen - 8;
2919 else
2920 assoclen = req->assoclen;
2921 ccm_xtra = CCM_B0_SIZE +
2922 ((assoclen) ? CCM_AAD_FIELD_SIZE : 0);
2923
2924 auth_offset = req->cryptlen ?
1f479e4c 2925 (req->assoclen + IV + 1 + ccm_xtra) : 0;
2debd332
HJ		 2926 if (op_type == CHCR_DECRYPT_OP) {
2927 if (crypto_aead_authsize(tfm) != req->cryptlen)
2928 tag_offset = crypto_aead_authsize(tfm);
2929 else
2930 auth_offset = 0;
2931 }
2932
567be3a5 2933 sec_cpl->op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(rx_channel_id, 2, 1);
2debd332 2934 sec_cpl->pldlen =
1f479e4c 2935 htonl(req->assoclen + IV + req->cryptlen + ccm_xtra);
2debd332
HJ		 2936 /* For CCM there wil be b0 always. So AAD start will be 1 always */
2937 sec_cpl->aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
1f479e4c
HJ		 2938 1 + IV, IV + assoclen + ccm_xtra,
2939 req->assoclen + IV + 1 + ccm_xtra, 0);
2debd332
HJ		 2940
2941 sec_cpl->cipherstop_lo_authinsert = FILL_SEC_CPL_AUTHINSERT(0,
2942 auth_offset, tag_offset,
2943 (op_type == CHCR_ENCRYPT_OP) ? 0 :
2944 crypto_aead_authsize(tfm));
2945 sec_cpl->seqno_numivs = FILL_SEC_CPL_SCMD0_SEQNO(op_type,
2946 (op_type == CHCR_ENCRYPT_OP) ? 0 : 1,
0a7bd30c 2947 cipher_mode, mac_mode,
2f47d580 2948 aeadctx->hmac_ctrl, IV >> 1);
2debd332
HJ		 2949
2950 sec_cpl->ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2f47d580 2951 0, dst_size);
2debd332
HJ		 2952}
2953
1efb892b
CIK		 2954static int aead_ccm_validate_input(unsigned short op_type,
2955 struct aead_request *req,
2956 struct chcr_aead_ctx *aeadctx,
2957 unsigned int sub_type)
2debd332
HJ		 2958{
2959 if (sub_type != CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309) {
2960 if (crypto_ccm_check_iv(req->iv)) {
2961 pr_err("CCM: IV check fails\n");
2962 return -EINVAL;
2963 }
2964 } else {
2965 if (req->assoclen != 16 && req->assoclen != 20) {
2966 pr_err("RFC4309: Invalid AAD length %d\n",
2967 req->assoclen);
2968 return -EINVAL;
2969 }
2970 }
2debd332
HJ		 2971 return 0;
2972}
2973
2debd332
HJ		 2974static struct sk_buff *create_aead_ccm_wr(struct aead_request *req,
2975 unsigned short qid,
4262c98a 2976 int size)
2debd332
HJ		 2977{
2978 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
2f47d580 2979 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 2980 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
2981 struct sk_buff *skb = NULL;
2982 struct chcr_wr *chcr_req;
2983 struct cpl_rx_phys_dsgl *phys_cpl;
2f47d580
HJ		 2984 struct ulptx_sgl *ulptx;
2985 unsigned int transhdr_len;
1f479e4c 2986 unsigned int dst_size = 0, kctx_len, dnents, temp, snents;
2f47d580 2987 unsigned int sub_type, assoclen = req->assoclen;
2debd332 2988 unsigned int authsize = crypto_aead_authsize(tfm);
2f47d580 2989 int error = -EINVAL;
1f479e4c 2990 u8 *ivptr;
2debd332
HJ		 2991 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
2992 GFP_ATOMIC;
2f47d580 2993 struct adapter *adap = padap(a_ctx(tfm)->dev);
2debd332 2994
2f47d580
HJ		 2995 sub_type = get_aead_subtype(tfm);
2996 if (sub_type == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309)
2997 assoclen -= 8;
4262c98a
HJ		 2998 reqctx->b0_len = CCM_B0_SIZE + (assoclen ? CCM_AAD_FIELD_SIZE : 0);
2999 error = chcr_aead_common_init(req);
2f47d580
HJ		 3000 if (error)
3001 return ERR_PTR(error);
0e93708d 3002
4262c98a 3003 error = aead_ccm_validate_input(reqctx->op, req, aeadctx, sub_type);
5fe8c711 3004 if (error)
2debd332 3005 goto err;
1f479e4c 3006 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen
4262c98a 3007 + (reqctx->op ? -authsize : authsize),
1f479e4c 3008 CHCR_DST_SG_SIZE, 0);
e1a018e6 3009 dnents += MIN_CCM_SG; // For IV and B0
2f47d580 3010 dst_size = get_space_for_phys_dsgl(dnents);
1f479e4c
HJ		 3011 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
3012 CHCR_SRC_SG_SIZE, 0);
3013 snents += MIN_CCM_SG; //For B0
125d01ca 3014 kctx_len = roundup(aeadctx->enckey_len, 16) * 2;
2debd332 3015 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
1f479e4c 3016 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen +
2f47d580 3017 reqctx->b0_len) <= SGE_MAX_WR_LEN;
1f479e4c 3018 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen +
125d01ca 3019 reqctx->b0_len, 16) :
1f479e4c 3020 (sgl_len(snents) * 8);
2f47d580 3021 transhdr_len += temp;
125d01ca 3022 transhdr_len = roundup(transhdr_len, 16);
2f47d580
HJ		 3023
3024 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE -
1f479e4c 3025 reqctx->b0_len, transhdr_len, reqctx->op)) {
ee0863ba 3026 atomic_inc(&adap->chcr_stats.fallback);
4262c98a
HJ		 3027 chcr_aead_common_exit(req);
3028 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
0e93708d 3029 }
1f479e4c 3030 skb = alloc_skb(transhdr_len, flags);
2debd332 3031
5fe8c711
HJ		 3032 if (!skb) {
3033 error = -ENOMEM;
2debd332 3034 goto err;
5fe8c711 3035 }
2debd332 3036
1f479e4c 3037 chcr_req = __skb_put_zero(skb, transhdr_len);
2debd332 3038
4262c98a 3039 fill_sec_cpl_for_aead(&chcr_req->sec_cpl, dst_size, req, reqctx->op);
2debd332
HJ		 3040
3041 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3042 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
125d01ca
HJ		 3043 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3044 aeadctx->key, aeadctx->enckey_len);
2debd332
HJ		 3045
3046 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
1f479e4c
HJ		 3047 ivptr = (u8 *)(phys_cpl + 1) + dst_size;
3048 ulptx = (struct ulptx_sgl *)(ivptr + IV);
3049 error = ccm_format_packet(req, ivptr, sub_type, reqctx->op, assoclen);
5fe8c711 3050 if (error)
2debd332 3051 goto dstmap_fail;
1f479e4c
HJ		 3052 chcr_add_aead_dst_ent(req, phys_cpl, qid);
3053 chcr_add_aead_src_ent(req, ulptx);
2debd332 3054
ee0863ba 3055 atomic_inc(&adap->chcr_stats.aead_rqst);
1f479e4c
HJ		 3056 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
3057 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen +
2f47d580
HJ		 3058 reqctx->b0_len) : 0);
3059 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, 0,
3060 transhdr_len, temp, 0);
2debd332 3061 reqctx->skb = skb;
2f47d580 3062
2debd332
HJ		 3063 return skb;
3064dstmap_fail:
3065 kfree_skb(skb);
2debd332 3066err:
4262c98a 3067 chcr_aead_common_exit(req);
5fe8c711 3068 return ERR_PTR(error);
2debd332
HJ		 3069}
3070
3071static struct sk_buff *create_gcm_wr(struct aead_request *req,
3072 unsigned short qid,
4262c98a 3073 int size)
2debd332
HJ		 3074{
3075 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
567be3a5
AS		 3076 struct chcr_context *ctx = a_ctx(tfm);
3077 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2debd332
HJ		 3078 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3079 struct sk_buff *skb = NULL;
3080 struct chcr_wr *chcr_req;
3081 struct cpl_rx_phys_dsgl *phys_cpl;
2f47d580 3082 struct ulptx_sgl *ulptx;
1f479e4c 3083 unsigned int transhdr_len, dnents = 0, snents;
2f47d580 3084 unsigned int dst_size = 0, temp = 0, kctx_len, assoclen = req->assoclen;
2debd332 3085 unsigned int authsize = crypto_aead_authsize(tfm);
2f47d580 3086 int error = -EINVAL;
1f479e4c 3087 u8 *ivptr;
2debd332
HJ		 3088 gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
3089 GFP_ATOMIC;
567be3a5
AS		 3090 struct adapter *adap = padap(ctx->dev);
3091 unsigned int rx_channel_id = reqctx->rxqidx / ctx->rxq_perchan;
2debd332 3092
2f47d580
HJ		 3093 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106)
3094 assoclen = req->assoclen - 8;
2debd332 3095
4262c98a
HJ		 3096 reqctx->b0_len = 0;
3097 error = chcr_aead_common_init(req);
e1a018e6
HJ		 3098 if (error)
3099 return ERR_PTR(error);
1f479e4c 3100 dnents = sg_nents_xlen(req->dst, req->assoclen + req->cryptlen +
4262c98a 3101 (reqctx->op ? -authsize : authsize),
1f479e4c
HJ		 3102 CHCR_DST_SG_SIZE, 0);
3103 snents = sg_nents_xlen(req->src, req->assoclen + req->cryptlen,
3104 CHCR_SRC_SG_SIZE, 0);
e1a018e6 3105 dnents += MIN_GCM_SG; // For IV
2f47d580 3106 dst_size = get_space_for_phys_dsgl(dnents);
125d01ca 3107 kctx_len = roundup(aeadctx->enckey_len, 16) + AEAD_H_SIZE;
2debd332 3108 transhdr_len = CIPHER_TRANSHDR_SIZE(kctx_len, dst_size);
1f479e4c 3109 reqctx->imm = (transhdr_len + req->assoclen + req->cryptlen) <=
2f47d580 3110 SGE_MAX_WR_LEN;
1f479e4c
HJ		 3111 temp = reqctx->imm ? roundup(req->assoclen + req->cryptlen, 16) :
3112 (sgl_len(snents) * 8);
2f47d580 3113 transhdr_len += temp;
125d01ca 3114 transhdr_len = roundup(transhdr_len, 16);
2f47d580 3115 if (chcr_aead_need_fallback(req, dnents, T6_MAX_AAD_SIZE,
4262c98a
HJ		 3116 transhdr_len, reqctx->op)) {
3117
ee0863ba 3118 atomic_inc(&adap->chcr_stats.fallback);
4262c98a
HJ		 3119 chcr_aead_common_exit(req);
3120 return ERR_PTR(chcr_aead_fallback(req, reqctx->op));
0e93708d 3121 }
1f479e4c 3122 skb = alloc_skb(transhdr_len, flags);
5fe8c711
HJ		 3123 if (!skb) {
3124 error = -ENOMEM;
2debd332 3125 goto err;
5fe8c711 3126 }
2debd332 3127
de77b966 3128 chcr_req = __skb_put_zero(skb, transhdr_len);
2debd332 3129
2f47d580 3130 //Offset of tag from end
4262c98a 3131 temp = (reqctx->op == CHCR_ENCRYPT_OP) ? 0 : authsize;
2debd332 3132 chcr_req->sec_cpl.op_ivinsrtofst = FILL_SEC_CPL_OP_IVINSR(
567be3a5 3133 rx_channel_id, 2, 1);
0e93708d 3134 chcr_req->sec_cpl.pldlen =
1f479e4c 3135 htonl(req->assoclen + IV + req->cryptlen);
2debd332 3136 chcr_req->sec_cpl.aadstart_cipherstop_hi = FILL_SEC_CPL_CIPHERSTOP_HI(
1f479e4c
HJ		 3137 assoclen ? 1 + IV : 0,
3138 assoclen ? IV + assoclen : 0,
3139 req->assoclen + IV + 1, 0);
e1a018e6 3140 chcr_req->sec_cpl.cipherstop_lo_authinsert =
1f479e4c 3141 FILL_SEC_CPL_AUTHINSERT(0, req->assoclen + IV + 1,
2f47d580 3142 temp, temp);
e1a018e6 3143 chcr_req->sec_cpl.seqno_numivs =
4262c98a 3144 FILL_SEC_CPL_SCMD0_SEQNO(reqctx->op, (reqctx->op ==
2debd332
HJ		 3145 CHCR_ENCRYPT_OP) ? 1 : 0,
3146 CHCR_SCMD_CIPHER_MODE_AES_GCM,
0a7bd30c 3147 CHCR_SCMD_AUTH_MODE_GHASH,
2f47d580 3148 aeadctx->hmac_ctrl, IV >> 1);
2debd332 3149 chcr_req->sec_cpl.ivgen_hdrlen = FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2f47d580 3150 0, 0, dst_size);
2debd332
HJ		 3151 chcr_req->key_ctx.ctx_hdr = aeadctx->key_ctx_hdr;
3152 memcpy(chcr_req->key_ctx.key, aeadctx->key, aeadctx->enckey_len);
125d01ca
HJ		 3153 memcpy(chcr_req->key_ctx.key + roundup(aeadctx->enckey_len, 16),
3154 GCM_CTX(aeadctx)->ghash_h, AEAD_H_SIZE);
2debd332 3155
1f479e4c
HJ		 3156 phys_cpl = (struct cpl_rx_phys_dsgl *)((u8 *)(chcr_req + 1) + kctx_len);
3157 ivptr = (u8 *)(phys_cpl + 1) + dst_size;
2debd332
HJ		 3158 /* prepare a 16 byte iv */
3159 /* S A L T | IV | 0x00000001 */
3160 if (get_aead_subtype(tfm) ==
3161 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106) {
1f479e4c
HJ		 3162 memcpy(ivptr, aeadctx->salt, 4);
3163 memcpy(ivptr + 4, req->iv, GCM_RFC4106_IV_SIZE);
2debd332 3164 } else {
1f479e4c 3165 memcpy(ivptr, req->iv, GCM_AES_IV_SIZE);
2debd332 3166 }
1f479e4c 3167 *((unsigned int *)(ivptr + 12)) = htonl(0x01);
2debd332 3168
1f479e4c 3169 ulptx = (struct ulptx_sgl *)(ivptr + 16);
2debd332 3170
1f479e4c
HJ		 3171 chcr_add_aead_dst_ent(req, phys_cpl, qid);
3172 chcr_add_aead_src_ent(req, ulptx);
ee0863ba 3173 atomic_inc(&adap->chcr_stats.aead_rqst);
1f479e4c
HJ		 3174 temp = sizeof(struct cpl_rx_phys_dsgl) + dst_size + IV +
3175 kctx_len + (reqctx->imm ? (req->assoclen + req->cryptlen) : 0);
2f47d580
HJ		 3176 create_wreq(a_ctx(tfm), chcr_req, &req->base, reqctx->imm, size,
3177 transhdr_len, temp, reqctx->verify);
2debd332 3178 reqctx->skb = skb;
2debd332
HJ		 3179 return skb;
3180
2debd332 3181err:
4262c98a 3182 chcr_aead_common_exit(req);
5fe8c711 3183 return ERR_PTR(error);
2debd332
HJ		 3184}
3185
3186
3187
3188static int chcr_aead_cra_init(struct crypto_aead *tfm)
3189{
2f47d580 3190 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
0e93708d
HJ		 3191 struct aead_alg *alg = crypto_aead_alg(tfm);
3192
3193 aeadctx->sw_cipher = crypto_alloc_aead(alg->base.cra_name, 0,
5fe8c711
HJ		 3194 CRYPTO_ALG_NEED_FALLBACK |
3195 CRYPTO_ALG_ASYNC);
0e93708d
HJ		 3196 if (IS_ERR(aeadctx->sw_cipher))
3197 return PTR_ERR(aeadctx->sw_cipher);
3198 crypto_aead_set_reqsize(tfm, max(sizeof(struct chcr_aead_reqctx),
3199 sizeof(struct aead_request) +
3200 crypto_aead_reqsize(aeadctx->sw_cipher)));
2f47d580 3201 return chcr_device_init(a_ctx(tfm));
2debd332
HJ		 3202}
3203
3204static void chcr_aead_cra_exit(struct crypto_aead *tfm)
3205{
2f47d580 3206 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
0e93708d 3207
0e93708d 3208 crypto_free_aead(aeadctx->sw_cipher);
2debd332
HJ		 3209}
3210
3211static int chcr_authenc_null_setauthsize(struct crypto_aead *tfm,
3212 unsigned int authsize)
3213{
2f47d580 3214 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3215
3216 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NOP;
3217 aeadctx->mayverify = VERIFY_HW;
0e93708d 3218 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 3219}
3220static int chcr_authenc_setauthsize(struct crypto_aead *tfm,
3221 unsigned int authsize)
3222{
2f47d580 3223 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3224 u32 maxauth = crypto_aead_maxauthsize(tfm);
3225
3226 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
3227 * true for sha1. authsize == 12 condition should be before
3228 * authsize == (maxauth >> 1)
3229 */
3230 if (authsize == ICV_4) {
3231 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3232 aeadctx->mayverify = VERIFY_HW;
3233 } else if (authsize == ICV_6) {
3234 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3235 aeadctx->mayverify = VERIFY_HW;
3236 } else if (authsize == ICV_10) {
3237 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3238 aeadctx->mayverify = VERIFY_HW;
3239 } else if (authsize == ICV_12) {
3240 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3241 aeadctx->mayverify = VERIFY_HW;
3242 } else if (authsize == ICV_14) {
3243 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3244 aeadctx->mayverify = VERIFY_HW;
3245 } else if (authsize == (maxauth >> 1)) {
3246 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3247 aeadctx->mayverify = VERIFY_HW;
3248 } else if (authsize == maxauth) {
3249 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3250 aeadctx->mayverify = VERIFY_HW;
3251 } else {
3252 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3253 aeadctx->mayverify = VERIFY_SW;
3254 }
0e93708d 3255 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 3256}
3257
3258
3259static int chcr_gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
3260{
2f47d580 3261 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3262
3263 switch (authsize) {
3264 case ICV_4:
3265 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3266 aeadctx->mayverify = VERIFY_HW;
3267 break;
3268 case ICV_8:
3269 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3270 aeadctx->mayverify = VERIFY_HW;
3271 break;
3272 case ICV_12:
fc6176a2
CIK		 3273 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3274 aeadctx->mayverify = VERIFY_HW;
2debd332
HJ		 3275 break;
3276 case ICV_14:
fc6176a2
CIK		 3277 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3278 aeadctx->mayverify = VERIFY_HW;
2debd332
HJ		 3279 break;
3280 case ICV_16:
3281 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3282 aeadctx->mayverify = VERIFY_HW;
3283 break;
3284 case ICV_13:
3285 case ICV_15:
3286 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3287 aeadctx->mayverify = VERIFY_SW;
3288 break;
3289 default:
2debd332
HJ		 3290 return -EINVAL;
3291 }
0e93708d 3292 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 3293}
3294
3295static int chcr_4106_4309_setauthsize(struct crypto_aead *tfm,
3296 unsigned int authsize)
3297{
2f47d580 3298 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3299
3300 switch (authsize) {
3301 case ICV_8:
3302 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3303 aeadctx->mayverify = VERIFY_HW;
3304 break;
3305 case ICV_12:
3306 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3307 aeadctx->mayverify = VERIFY_HW;
3308 break;
3309 case ICV_16:
3310 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3311 aeadctx->mayverify = VERIFY_HW;
3312 break;
3313 default:
2debd332
HJ		 3314 return -EINVAL;
3315 }
0e93708d 3316 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 3317}
3318
3319static int chcr_ccm_setauthsize(struct crypto_aead *tfm,
3320 unsigned int authsize)
3321{
2f47d580 3322 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(tfm));
2debd332
HJ		 3323
3324 switch (authsize) {
3325 case ICV_4:
3326 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL1;
3327 aeadctx->mayverify = VERIFY_HW;
3328 break;
3329 case ICV_6:
3330 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL2;
3331 aeadctx->mayverify = VERIFY_HW;
3332 break;
3333 case ICV_8:
3334 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_DIV2;
3335 aeadctx->mayverify = VERIFY_HW;
3336 break;
3337 case ICV_10:
3338 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366;
3339 aeadctx->mayverify = VERIFY_HW;
3340 break;
3341 case ICV_12:
3342 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT;
3343 aeadctx->mayverify = VERIFY_HW;
3344 break;
3345 case ICV_14:
3346 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_PL3;
3347 aeadctx->mayverify = VERIFY_HW;
3348 break;
3349 case ICV_16:
3350 aeadctx->hmac_ctrl = CHCR_SCMD_HMAC_CTRL_NO_TRUNC;
3351 aeadctx->mayverify = VERIFY_HW;
3352 break;
3353 default:
2debd332
HJ		 3354 return -EINVAL;
3355 }
0e93708d 3356 return crypto_aead_setauthsize(aeadctx->sw_cipher, authsize);
2debd332
HJ		 3357}
3358
0e93708d 3359static int chcr_ccm_common_setkey(struct crypto_aead *aead,
2debd332
HJ		 3360 const u8 *key,
3361 unsigned int keylen)
3362{
2f47d580 3363 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
2debd332
HJ		 3364 unsigned char ck_size, mk_size;
3365 int key_ctx_size = 0;
3366
125d01ca 3367 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) * 2;
2debd332 3368 if (keylen == AES_KEYSIZE_128) {
2debd332 3369 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
125d01ca 3370 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
2debd332
HJ		 3371 } else if (keylen == AES_KEYSIZE_192) {
3372 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3373 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_192;
3374 } else if (keylen == AES_KEYSIZE_256) {
3375 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3376 mk_size = CHCR_KEYCTX_MAC_KEY_SIZE_256;
3377 } else {
2debd332
HJ		 3378 aeadctx->enckey_len = 0;
3379 return -EINVAL;
3380 }
3381 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, mk_size, 0, 0,
3382 key_ctx_size >> 4);
0e93708d
HJ		 3383 memcpy(aeadctx->key, key, keylen);
3384 aeadctx->enckey_len = keylen;
3385
2debd332
HJ		 3386 return 0;
3387}
3388
0e93708d
HJ		 3389static int chcr_aead_ccm_setkey(struct crypto_aead *aead,
3390 const u8 *key,
3391 unsigned int keylen)
3392{
2f47d580 3393 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
0e93708d
HJ		 3394 int error;
3395
3396 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3397 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3398 CRYPTO_TFM_REQ_MASK);
3399 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
0e93708d
HJ		 3400 if (error)
3401 return error;
3402 return chcr_ccm_common_setkey(aead, key, keylen);
3403}
3404
2debd332
HJ		 3405static int chcr_aead_rfc4309_setkey(struct crypto_aead *aead, const u8 *key,
3406 unsigned int keylen)
3407{
2f47d580 3408 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
4dbeae42 3409 int error;
2debd332
HJ		 3410
3411 if (keylen < 3) {
2debd332
HJ		 3412 aeadctx->enckey_len = 0;
3413 return -EINVAL;
3414 }
4dbeae42
HJ		 3415 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3416 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead) &
3417 CRYPTO_TFM_REQ_MASK);
3418 error = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
4dbeae42
HJ		 3419 if (error)
3420 return error;
2debd332
HJ		 3421 keylen -= 3;
3422 memcpy(aeadctx->salt, key + keylen, 3);
0e93708d 3423 return chcr_ccm_common_setkey(aead, key, keylen);
2debd332
HJ		 3424}
3425
3426static int chcr_gcm_setkey(struct crypto_aead *aead, const u8 *key,
3427 unsigned int keylen)
3428{
2f47d580 3429 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(aead));
2debd332 3430 struct chcr_gcm_ctx *gctx = GCM_CTX(aeadctx);
2debd332
HJ		 3431 unsigned int ck_size;
3432 int ret = 0, key_ctx_size = 0;
571c47ab 3433 struct crypto_aes_ctx aes;
2debd332 3434
0e93708d
HJ		 3435 aeadctx->enckey_len = 0;
3436 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3437 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(aead)
3438 & CRYPTO_TFM_REQ_MASK);
3439 ret = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
0e93708d
HJ		 3440 if (ret)
3441 goto out;
3442
7c2cf1c4
HJ		 3443 if (get_aead_subtype(aead) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3444 keylen > 3) {
2debd332
HJ		 3445 keylen -= 4; /* nonce/salt is present in the last 4 bytes */
3446 memcpy(aeadctx->salt, key + keylen, 4);
3447 }
3448 if (keylen == AES_KEYSIZE_128) {
3449 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3450 } else if (keylen == AES_KEYSIZE_192) {
3451 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3452 } else if (keylen == AES_KEYSIZE_256) {
3453 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3454 } else {
0e93708d 3455 pr_err("GCM: Invalid key length %d\n", keylen);
2debd332
HJ		 3456 ret = -EINVAL;
3457 goto out;
3458 }
3459
3460 memcpy(aeadctx->key, key, keylen);
3461 aeadctx->enckey_len = keylen;
125d01ca 3462 key_ctx_size = sizeof(struct _key_ctx) + roundup(keylen, 16) +
2debd332 3463 AEAD_H_SIZE;
125d01ca 3464 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
2debd332
HJ		 3465 CHCR_KEYCTX_MAC_KEY_SIZE_128,
3466 0, 0,
3467 key_ctx_size >> 4);
8356ea51
HJ		 3468 /* Calculate the H = CIPH(K, 0 repeated 16 times).
3469 * It will go in key context
2debd332 3470 */
571c47ab 3471 ret = aes_expandkey(&aes, key, keylen);
2debd332
HJ		 3472 if (ret) {
3473 aeadctx->enckey_len = 0;
571c47ab 3474 goto out;
2debd332
HJ		 3475 }
3476 memset(gctx->ghash_h, 0, AEAD_H_SIZE);
571c47ab
AB		 3477 aes_encrypt(&aes, gctx->ghash_h, gctx->ghash_h);
3478 memzero_explicit(&aes, sizeof(aes));
2debd332 3479
2debd332
HJ		 3480out:
3481 return ret;
3482}
3483
3484static int chcr_authenc_setkey(struct crypto_aead *authenc, const u8 *key,
3485 unsigned int keylen)
3486{
2f47d580 3487 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
2debd332
HJ		 3488 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3489 /* it contains auth and cipher key both*/
3490 struct crypto_authenc_keys keys;
3d64bd67 3491 unsigned int bs, subtype;
2debd332
HJ		 3492 unsigned int max_authsize = crypto_aead_alg(authenc)->maxauthsize;
3493 int err = 0, i, key_ctx_len = 0;
3494 unsigned char ck_size = 0;
3495 unsigned char pad[CHCR_HASH_MAX_BLOCK_SIZE_128] = { 0 };
ec1bca94 3496 struct crypto_shash *base_hash = ERR_PTR(-EINVAL);
2debd332
HJ		 3497 struct algo_param param;
3498 int align;
3499 u8 *o_ptr = NULL;
3500
0e93708d
HJ		 3501 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3502 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3503 & CRYPTO_TFM_REQ_MASK);
3504 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
0e93708d
HJ		 3505 if (err)
3506 goto out;
3507
674f368a 3508 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
2debd332 3509 goto out;
2debd332
HJ		 3510
3511 if (get_alg_config(&param, max_authsize)) {
3512 pr_err("chcr : Unsupported digest size\n");
3513 goto out;
3514 }
3d64bd67
HJ		 3515 subtype = get_aead_subtype(authenc);
3516 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3517 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3518 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3519 goto out;
3520 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3521 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3522 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3523 }
2debd332
HJ		 3524 if (keys.enckeylen == AES_KEYSIZE_128) {
3525 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3526 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3527 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3528 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3529 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3530 } else {
3531 pr_err("chcr : Unsupported cipher key\n");
3532 goto out;
3533 }
3534
3535 /* Copy only encryption key. We use authkey to generate h(ipad) and
3536 * h(opad) so authkey is not needed again. authkeylen size have the
3537 * size of the hash digest size.
3538 */
3539 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3540 aeadctx->enckey_len = keys.enckeylen;
3d64bd67
HJ		 3541 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3542 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
2debd332 3543
3d64bd67
HJ		 3544 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3545 aeadctx->enckey_len << 3);
3546 }
2debd332
HJ		 3547 base_hash = chcr_alloc_shash(max_authsize);
3548 if (IS_ERR(base_hash)) {
3549 pr_err("chcr : Base driver cannot be loaded\n");
0e93708d 3550 aeadctx->enckey_len = 0;
eb526531 3551 memzero_explicit(&keys, sizeof(keys));
0e93708d 3552 return -EINVAL;
324429d7 3553 }
2debd332
HJ		 3554 {
3555 SHASH_DESC_ON_STACK(shash, base_hash);
6faa0f57 3556
2debd332 3557 shash->tfm = base_hash;
2debd332
HJ		 3558 bs = crypto_shash_blocksize(base_hash);
3559 align = KEYCTX_ALIGN_PAD(max_authsize);
3560 o_ptr = actx->h_iopad + param.result_size + align;
3561
3562 if (keys.authkeylen > bs) {
3563 err = crypto_shash_digest(shash, keys.authkey,
3564 keys.authkeylen,
3565 o_ptr);
3566 if (err) {
3567 pr_err("chcr : Base driver cannot be loaded\n");
3568 goto out;
3569 }
3570 keys.authkeylen = max_authsize;
3571 } else
3572 memcpy(o_ptr, keys.authkey, keys.authkeylen);
3573
3574 /* Compute the ipad-digest*/
3575 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3576 memcpy(pad, o_ptr, keys.authkeylen);
3577 for (i = 0; i < bs >> 2; i++)
3578 *((unsigned int *)pad + i) ^= IPAD_DATA;
3579
3580 if (chcr_compute_partial_hash(shash, pad, actx->h_iopad,
3581 max_authsize))
3582 goto out;
3583 /* Compute the opad-digest */
3584 memset(pad + keys.authkeylen, 0, bs - keys.authkeylen);
3585 memcpy(pad, o_ptr, keys.authkeylen);
3586 for (i = 0; i < bs >> 2; i++)
3587 *((unsigned int *)pad + i) ^= OPAD_DATA;
3588
3589 if (chcr_compute_partial_hash(shash, pad, o_ptr, max_authsize))
3590 goto out;
3591
3592 /* convert the ipad and opad digest to network order */
3593 chcr_change_order(actx->h_iopad, param.result_size);
3594 chcr_change_order(o_ptr, param.result_size);
3595 key_ctx_len = sizeof(struct _key_ctx) +
125d01ca 3596 roundup(keys.enckeylen, 16) +
2debd332
HJ		 3597 (param.result_size + align) * 2;
3598 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, param.mk_size,
3599 0, 1, key_ctx_len >> 4);
3600 actx->auth_mode = param.auth_mode;
3601 chcr_free_shash(base_hash);
3602
eb526531 3603 memzero_explicit(&keys, sizeof(keys));
2debd332
HJ		 3604 return 0;
3605 }
3606out:
3607 aeadctx->enckey_len = 0;
eb526531 3608 memzero_explicit(&keys, sizeof(keys));
ec1bca94 3609 if (!IS_ERR(base_hash))
2debd332
HJ		 3610 chcr_free_shash(base_hash);
3611 return -EINVAL;
324429d7
HS		 3612}
3613
2debd332
HJ		 3614static int chcr_aead_digest_null_setkey(struct crypto_aead *authenc,
3615 const u8 *key, unsigned int keylen)
3616{
2f47d580 3617 struct chcr_aead_ctx *aeadctx = AEAD_CTX(a_ctx(authenc));
2debd332
HJ		 3618 struct chcr_authenc_ctx *actx = AUTHENC_CTX(aeadctx);
3619 struct crypto_authenc_keys keys;
0e93708d 3620 int err;
2debd332 3621 /* it contains auth and cipher key both*/
3d64bd67 3622 unsigned int subtype;
2debd332
HJ		 3623 int key_ctx_len = 0;
3624 unsigned char ck_size = 0;
3625
0e93708d
HJ		 3626 crypto_aead_clear_flags(aeadctx->sw_cipher, CRYPTO_TFM_REQ_MASK);
3627 crypto_aead_set_flags(aeadctx->sw_cipher, crypto_aead_get_flags(authenc)
3628 & CRYPTO_TFM_REQ_MASK);
3629 err = crypto_aead_setkey(aeadctx->sw_cipher, key, keylen);
0e93708d
HJ		 3630 if (err)
3631 goto out;
3632
674f368a 3633 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
2debd332 3634 goto out;
674f368a 3635
3d64bd67
HJ		 3636 subtype = get_aead_subtype(authenc);
3637 if (subtype == CRYPTO_ALG_SUB_TYPE_CTR_SHA ||
3638 subtype == CRYPTO_ALG_SUB_TYPE_CTR_NULL) {
3639 if (keys.enckeylen < CTR_RFC3686_NONCE_SIZE)
3640 goto out;
3641 memcpy(aeadctx->nonce, keys.enckey + (keys.enckeylen
3642 - CTR_RFC3686_NONCE_SIZE), CTR_RFC3686_NONCE_SIZE);
3643 keys.enckeylen -= CTR_RFC3686_NONCE_SIZE;
3644 }
2debd332
HJ		 3645 if (keys.enckeylen == AES_KEYSIZE_128) {
3646 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
3647 } else if (keys.enckeylen == AES_KEYSIZE_192) {
3648 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_192;
3649 } else if (keys.enckeylen == AES_KEYSIZE_256) {
3650 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_256;
3651 } else {
3d64bd67 3652 pr_err("chcr : Unsupported cipher key %d\n", keys.enckeylen);
2debd332
HJ		 3653 goto out;
3654 }
3655 memcpy(aeadctx->key, keys.enckey, keys.enckeylen);
3656 aeadctx->enckey_len = keys.enckeylen;
3d64bd67
HJ		 3657 if (subtype == CRYPTO_ALG_SUB_TYPE_CBC_SHA ||
3658 subtype == CRYPTO_ALG_SUB_TYPE_CBC_NULL) {
3659 get_aes_decrypt_key(actx->dec_rrkey, aeadctx->key,
3660 aeadctx->enckey_len << 3);
3661 }
125d01ca 3662 key_ctx_len = sizeof(struct _key_ctx) + roundup(keys.enckeylen, 16);
2debd332
HJ		 3663
3664 aeadctx->key_ctx_hdr = FILL_KEY_CTX_HDR(ck_size, CHCR_KEYCTX_NO_KEY, 0,
3665 0, key_ctx_len >> 4);
3666 actx->auth_mode = CHCR_SCMD_AUTH_MODE_NOP;
eb526531 3667 memzero_explicit(&keys, sizeof(keys));
2debd332
HJ		 3668 return 0;
3669out:
3670 aeadctx->enckey_len = 0;
eb526531 3671 memzero_explicit(&keys, sizeof(keys));
2debd332
HJ		 3672 return -EINVAL;
3673}
6dad4e8a
AG		 3674
3675static int chcr_aead_op(struct aead_request *req,
6dad4e8a
AG		 3676 int size,
3677 create_wr_t create_wr_fn)
3678{
3679 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
fef4912b 3680 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
567be3a5
AS		 3681 struct chcr_context *ctx = a_ctx(tfm);
3682 struct uld_ctx *u_ctx = ULD_CTX(ctx);
6dad4e8a 3683 struct sk_buff *skb;
fef4912b 3684 struct chcr_dev *cdev;
6dad4e8a 3685
fef4912b
HJ		 3686 cdev = a_ctx(tfm)->dev;
3687 if (!cdev) {
6dad4e8a
AG		 3688 pr_err("chcr : %s : No crypto device.\n", __func__);
3689 return -ENXIO;
3690 }
fef4912b
HJ		 3691
3692 if (chcr_inc_wrcount(cdev)) {
3693 /* Detach state for CHCR means lldi or padap is freed.
3694 * We cannot increment fallback here.
3695 */
3696 return chcr_aead_fallback(req, reqctx->op);
3697 }
3698
6dad4e8a 3699 if (cxgb4_is_crypto_q_full(u_ctx->lldi.ports[0],
567be3a5
AS		 3700 reqctx->txqidx) &&
3701 (!(req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG))) {
fef4912b 3702 chcr_dec_wrcount(cdev);
6faa0f57 3703 return -ENOSPC;
6dad4e8a
AG		 3704 }
3705
d91a3159
DSK		 3706 if (get_aead_subtype(tfm) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106 &&
3707 crypto_ipsec_check_assoclen(req->assoclen) != 0) {
3708 pr_err("RFC4106: Invalid value of assoclen %d\n",
3709 req->assoclen);
3710 return -EINVAL;
3711 }
3712
6dad4e8a 3713 /* Form a WR from req */
567be3a5 3714 skb = create_wr_fn(req, u_ctx->lldi.rxq_ids[reqctx->rxqidx], size);
6dad4e8a 3715
b04a27ca 3716 if (IS_ERR_OR_NULL(skb)) {
fef4912b 3717 chcr_dec_wrcount(cdev);
b04a27ca 3718 return PTR_ERR_OR_ZERO(skb);
fef4912b 3719 }
6dad4e8a
AG		 3720
3721 skb->dev = u_ctx->lldi.ports[0];
567be3a5 3722 set_wr_txq(skb, CPL_PRIORITY_DATA, reqctx->txqidx);
6dad4e8a 3723 chcr_send_wr(skb);
567be3a5 3724 return -EINPROGRESS;
6dad4e8a
AG		 3725}
3726
2debd332
HJ		 3727static int chcr_aead_encrypt(struct aead_request *req)
3728{
3729 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
3730 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
567be3a5
AS		 3731 struct chcr_context *ctx = a_ctx(tfm);
3732 unsigned int cpu;
3733
3734 cpu = get_cpu();
3735 reqctx->txqidx = cpu % ctx->ntxq;
3736 reqctx->rxqidx = cpu % ctx->nrxq;
3737 put_cpu();
2debd332
HJ		 3738
3739 reqctx->verify = VERIFY_HW;
4262c98a 3740 reqctx->op = CHCR_ENCRYPT_OP;
2debd332
HJ		 3741
3742 switch (get_aead_subtype(tfm)) {
3d64bd67
HJ		 3743 case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3744 case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3745 case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3746 case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
4262c98a 3747 return chcr_aead_op(req, 0, create_authenc_wr);
2debd332
HJ		 3748 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3749 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
4262c98a 3750 return chcr_aead_op(req, 0, create_aead_ccm_wr);
2debd332 3751 default:
4262c98a 3752 return chcr_aead_op(req, 0, create_gcm_wr);
2debd332
HJ		 3753 }
3754}
3755
3756static int chcr_aead_decrypt(struct aead_request *req)
3757{
3758 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
567be3a5
AS		 3759 struct chcr_context *ctx = a_ctx(tfm);
3760 struct chcr_aead_ctx *aeadctx = AEAD_CTX(ctx);
2debd332
HJ		 3761 struct chcr_aead_reqctx *reqctx = aead_request_ctx(req);
3762 int size;
567be3a5
AS		 3763 unsigned int cpu;
3764
3765 cpu = get_cpu();
3766 reqctx->txqidx = cpu % ctx->ntxq;
3767 reqctx->rxqidx = cpu % ctx->nrxq;
3768 put_cpu();
2debd332
HJ		 3769
3770 if (aeadctx->mayverify == VERIFY_SW) {
3771 size = crypto_aead_maxauthsize(tfm);
3772 reqctx->verify = VERIFY_SW;
3773 } else {
3774 size = 0;
3775 reqctx->verify = VERIFY_HW;
3776 }
4262c98a 3777 reqctx->op = CHCR_DECRYPT_OP;
2debd332 3778 switch (get_aead_subtype(tfm)) {
3d64bd67
HJ		 3779 case CRYPTO_ALG_SUB_TYPE_CBC_SHA:
3780 case CRYPTO_ALG_SUB_TYPE_CTR_SHA:
3781 case CRYPTO_ALG_SUB_TYPE_CBC_NULL:
3782 case CRYPTO_ALG_SUB_TYPE_CTR_NULL:
4262c98a 3783 return chcr_aead_op(req, size, create_authenc_wr);
2debd332
HJ		 3784 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM:
3785 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309:
4262c98a 3786 return chcr_aead_op(req, size, create_aead_ccm_wr);
2debd332 3787 default:
4262c98a 3788 return chcr_aead_op(req, size, create_gcm_wr);
2debd332
HJ		 3789 }
3790}
3791
324429d7
HS		 3792static struct chcr_alg_template driver_algs[] = {
3793 /* AES-CBC */
3794 {
7cea6d3e 3795 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CBC,
324429d7 3796 .is_registered = 0,
7cea6d3e
AB		 3797 .alg.skcipher = {
3798 .base.cra_name = "cbc(aes)",
3799 .base.cra_driver_name = "cbc-aes-chcr",
3800 .base.cra_blocksize = AES_BLOCK_SIZE,
3801
3802 .init = chcr_init_tfm,
3803 .exit = chcr_exit_tfm,
3804 .min_keysize = AES_MIN_KEY_SIZE,
3805 .max_keysize = AES_MAX_KEY_SIZE,
3806 .ivsize = AES_BLOCK_SIZE,
3807 .setkey = chcr_aes_cbc_setkey,
3808 .encrypt = chcr_aes_encrypt,
3809 .decrypt = chcr_aes_decrypt,
324429d7 3810 }
324429d7
HS		 3811 },
3812 {
7cea6d3e 3813 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_XTS,
324429d7 3814 .is_registered = 0,
7cea6d3e
AB		 3815 .alg.skcipher = {
3816 .base.cra_name = "xts(aes)",
3817 .base.cra_driver_name = "xts-aes-chcr",
3818 .base.cra_blocksize = AES_BLOCK_SIZE,
3819
3820 .init = chcr_init_tfm,
3821 .exit = chcr_exit_tfm,
3822 .min_keysize = 2 * AES_MIN_KEY_SIZE,
3823 .max_keysize = 2 * AES_MAX_KEY_SIZE,
3824 .ivsize = AES_BLOCK_SIZE,
3825 .setkey = chcr_aes_xts_setkey,
3826 .encrypt = chcr_aes_encrypt,
3827 .decrypt = chcr_aes_decrypt,
324429d7 3828 }
b8fd1f41
HJ		 3829 },
3830 {
7cea6d3e 3831 .type = CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_SUB_TYPE_CTR,
b8fd1f41 3832 .is_registered = 0,
7cea6d3e
AB		 3833 .alg.skcipher = {
3834 .base.cra_name = "ctr(aes)",
3835 .base.cra_driver_name = "ctr-aes-chcr",
3836 .base.cra_blocksize = 1,
3837
3838 .init = chcr_init_tfm,
3839 .exit = chcr_exit_tfm,
3840 .min_keysize = AES_MIN_KEY_SIZE,
3841 .max_keysize = AES_MAX_KEY_SIZE,
3842 .ivsize = AES_BLOCK_SIZE,
3843 .setkey = chcr_aes_ctr_setkey,
3844 .encrypt = chcr_aes_encrypt,
3845 .decrypt = chcr_aes_decrypt,
b8fd1f41
HJ		 3846 }
3847 },
3848 {
7cea6d3e 3849 .type = CRYPTO_ALG_TYPE_SKCIPHER |
b8fd1f41
HJ		 3850 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686,
3851 .is_registered = 0,
7cea6d3e
AB		 3852 .alg.skcipher = {
3853 .base.cra_name = "rfc3686(ctr(aes))",
3854 .base.cra_driver_name = "rfc3686-ctr-aes-chcr",
3855 .base.cra_blocksize = 1,
3856
3857 .init = chcr_rfc3686_init,
3858 .exit = chcr_exit_tfm,
3859 .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3860 .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE,
3861 .ivsize = CTR_RFC3686_IV_SIZE,
3862 .setkey = chcr_aes_rfc3686_setkey,
3863 .encrypt = chcr_aes_encrypt,
3864 .decrypt = chcr_aes_decrypt,
324429d7
HS		 3865 }
3866 },
3867 /* SHA */
3868 {
3869 .type = CRYPTO_ALG_TYPE_AHASH,
3870 .is_registered = 0,
3871 .alg.hash = {
3872 .halg.digestsize = SHA1_DIGEST_SIZE,
3873 .halg.base = {
3874 .cra_name = "sha1",
3875 .cra_driver_name = "sha1-chcr",
3876 .cra_blocksize = SHA1_BLOCK_SIZE,
3877 }
3878 }
3879 },
3880 {
3881 .type = CRYPTO_ALG_TYPE_AHASH,
3882 .is_registered = 0,
3883 .alg.hash = {
3884 .halg.digestsize = SHA256_DIGEST_SIZE,
3885 .halg.base = {
3886 .cra_name = "sha256",
3887 .cra_driver_name = "sha256-chcr",
3888 .cra_blocksize = SHA256_BLOCK_SIZE,
3889 }
3890 }
3891 },
3892 {
3893 .type = CRYPTO_ALG_TYPE_AHASH,
3894 .is_registered = 0,
3895 .alg.hash = {
3896 .halg.digestsize = SHA224_DIGEST_SIZE,
3897 .halg.base = {
3898 .cra_name = "sha224",
3899 .cra_driver_name = "sha224-chcr",
3900 .cra_blocksize = SHA224_BLOCK_SIZE,
3901 }
3902 }
3903 },
3904 {
3905 .type = CRYPTO_ALG_TYPE_AHASH,
3906 .is_registered = 0,
3907 .alg.hash = {
3908 .halg.digestsize = SHA384_DIGEST_SIZE,
3909 .halg.base = {
3910 .cra_name = "sha384",
3911 .cra_driver_name = "sha384-chcr",
3912 .cra_blocksize = SHA384_BLOCK_SIZE,
3913 }
3914 }
3915 },
3916 {
3917 .type = CRYPTO_ALG_TYPE_AHASH,
3918 .is_registered = 0,
3919 .alg.hash = {
3920 .halg.digestsize = SHA512_DIGEST_SIZE,
3921 .halg.base = {
3922 .cra_name = "sha512",
3923 .cra_driver_name = "sha512-chcr",
3924 .cra_blocksize = SHA512_BLOCK_SIZE,
3925 }
3926 }
3927 },
3928 /* HMAC */
3929 {
3930 .type = CRYPTO_ALG_TYPE_HMAC,
3931 .is_registered = 0,
3932 .alg.hash = {
3933 .halg.digestsize = SHA1_DIGEST_SIZE,
3934 .halg.base = {
3935 .cra_name = "hmac(sha1)",
2debd332 3936 .cra_driver_name = "hmac-sha1-chcr",
324429d7
HS		 3937 .cra_blocksize = SHA1_BLOCK_SIZE,
3938 }
3939 }
3940 },
3941 {
3942 .type = CRYPTO_ALG_TYPE_HMAC,
3943 .is_registered = 0,
3944 .alg.hash = {
3945 .halg.digestsize = SHA224_DIGEST_SIZE,
3946 .halg.base = {
3947 .cra_name = "hmac(sha224)",
2debd332 3948 .cra_driver_name = "hmac-sha224-chcr",
324429d7
HS		 3949 .cra_blocksize = SHA224_BLOCK_SIZE,
3950 }
3951 }
3952 },
3953 {
3954 .type = CRYPTO_ALG_TYPE_HMAC,
3955 .is_registered = 0,
3956 .alg.hash = {
3957 .halg.digestsize = SHA256_DIGEST_SIZE,
3958 .halg.base = {
3959 .cra_name = "hmac(sha256)",
2debd332 3960 .cra_driver_name = "hmac-sha256-chcr",
324429d7
HS		 3961 .cra_blocksize = SHA256_BLOCK_SIZE,
3962 }
3963 }
3964 },
3965 {
3966 .type = CRYPTO_ALG_TYPE_HMAC,
3967 .is_registered = 0,
3968 .alg.hash = {
3969 .halg.digestsize = SHA384_DIGEST_SIZE,
3970 .halg.base = {
3971 .cra_name = "hmac(sha384)",
2debd332 3972 .cra_driver_name = "hmac-sha384-chcr",
324429d7
HS		 3973 .cra_blocksize = SHA384_BLOCK_SIZE,
3974 }
3975 }
3976 },
3977 {
3978 .type = CRYPTO_ALG_TYPE_HMAC,
3979 .is_registered = 0,
3980 .alg.hash = {
3981 .halg.digestsize = SHA512_DIGEST_SIZE,
3982 .halg.base = {
3983 .cra_name = "hmac(sha512)",
2debd332 3984 .cra_driver_name = "hmac-sha512-chcr",
324429d7
HS		 3985 .cra_blocksize = SHA512_BLOCK_SIZE,
3986 }
3987 }
3988 },
2debd332
HJ		 3989 /* Add AEAD Algorithms */
3990 {
3991 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_GCM,
3992 .is_registered = 0,
3993 .alg.aead = {
3994 .base = {
3995 .cra_name = "gcm(aes)",
3996 .cra_driver_name = "gcm-aes-chcr",
3997 .cra_blocksize = 1,
e29abda5 3998 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 3999 .cra_ctxsize = sizeof(struct chcr_context) +
4000 sizeof(struct chcr_aead_ctx) +
4001 sizeof(struct chcr_gcm_ctx),
4002 },
8f6acb7f 4003 .ivsize = GCM_AES_IV_SIZE,
2debd332
HJ		 4004 .maxauthsize = GHASH_DIGEST_SIZE,
4005 .setkey = chcr_gcm_setkey,
4006 .setauthsize = chcr_gcm_setauthsize,
4007 }
4008 },
4009 {
4010 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106,
4011 .is_registered = 0,
4012 .alg.aead = {
4013 .base = {
4014 .cra_name = "rfc4106(gcm(aes))",
4015 .cra_driver_name = "rfc4106-gcm-aes-chcr",
4016 .cra_blocksize = 1,
e29abda5 4017 .cra_priority = CHCR_AEAD_PRIORITY + 1,
2debd332
HJ		 4018 .cra_ctxsize = sizeof(struct chcr_context) +
4019 sizeof(struct chcr_aead_ctx) +
4020 sizeof(struct chcr_gcm_ctx),
4021
4022 },
8f6acb7f 4023 .ivsize = GCM_RFC4106_IV_SIZE,
2debd332
HJ		 4024 .maxauthsize = GHASH_DIGEST_SIZE,
4025 .setkey = chcr_gcm_setkey,
4026 .setauthsize = chcr_4106_4309_setauthsize,
4027 }
4028 },
4029 {
4030 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_CCM,
4031 .is_registered = 0,
4032 .alg.aead = {
4033 .base = {
4034 .cra_name = "ccm(aes)",
4035 .cra_driver_name = "ccm-aes-chcr",
4036 .cra_blocksize = 1,
e29abda5 4037 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 4038 .cra_ctxsize = sizeof(struct chcr_context) +
4039 sizeof(struct chcr_aead_ctx),
4040
4041 },
4042 .ivsize = AES_BLOCK_SIZE,
4043 .maxauthsize = GHASH_DIGEST_SIZE,
4044 .setkey = chcr_aead_ccm_setkey,
4045 .setauthsize = chcr_ccm_setauthsize,
4046 }
4047 },
4048 {
4049 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309,
4050 .is_registered = 0,
4051 .alg.aead = {
4052 .base = {
4053 .cra_name = "rfc4309(ccm(aes))",
4054 .cra_driver_name = "rfc4309-ccm-aes-chcr",
4055 .cra_blocksize = 1,
e29abda5 4056 .cra_priority = CHCR_AEAD_PRIORITY + 1,
2debd332
HJ		 4057 .cra_ctxsize = sizeof(struct chcr_context) +
4058 sizeof(struct chcr_aead_ctx),
4059
4060 },
4061 .ivsize = 8,
4062 .maxauthsize = GHASH_DIGEST_SIZE,
4063 .setkey = chcr_aead_rfc4309_setkey,
4064 .setauthsize = chcr_4106_4309_setauthsize,
4065 }
4066 },
4067 {
3d64bd67 4068 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
2debd332
HJ		 4069 .is_registered = 0,
4070 .alg.aead = {
4071 .base = {
4072 .cra_name = "authenc(hmac(sha1),cbc(aes))",
4073 .cra_driver_name =
4074 "authenc-hmac-sha1-cbc-aes-chcr",
4075 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 4076 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 4077 .cra_ctxsize = sizeof(struct chcr_context) +
4078 sizeof(struct chcr_aead_ctx) +
4079 sizeof(struct chcr_authenc_ctx),
4080
4081 },
4082 .ivsize = AES_BLOCK_SIZE,
4083 .maxauthsize = SHA1_DIGEST_SIZE,
4084 .setkey = chcr_authenc_setkey,
4085 .setauthsize = chcr_authenc_setauthsize,
4086 }
4087 },
4088 {
3d64bd67 4089 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
2debd332
HJ		 4090 .is_registered = 0,
4091 .alg.aead = {
4092 .base = {
4093
4094 .cra_name = "authenc(hmac(sha256),cbc(aes))",
4095 .cra_driver_name =
4096 "authenc-hmac-sha256-cbc-aes-chcr",
4097 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 4098 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 4099 .cra_ctxsize = sizeof(struct chcr_context) +
4100 sizeof(struct chcr_aead_ctx) +
4101 sizeof(struct chcr_authenc_ctx),
4102
4103 },
4104 .ivsize = AES_BLOCK_SIZE,
4105 .maxauthsize = SHA256_DIGEST_SIZE,
4106 .setkey = chcr_authenc_setkey,
4107 .setauthsize = chcr_authenc_setauthsize,
4108 }
4109 },
4110 {
3d64bd67 4111 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
2debd332
HJ		 4112 .is_registered = 0,
4113 .alg.aead = {
4114 .base = {
4115 .cra_name = "authenc(hmac(sha224),cbc(aes))",
4116 .cra_driver_name =
4117 "authenc-hmac-sha224-cbc-aes-chcr",
4118 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 4119 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 4120 .cra_ctxsize = sizeof(struct chcr_context) +
4121 sizeof(struct chcr_aead_ctx) +
4122 sizeof(struct chcr_authenc_ctx),
4123 },
4124 .ivsize = AES_BLOCK_SIZE,
4125 .maxauthsize = SHA224_DIGEST_SIZE,
4126 .setkey = chcr_authenc_setkey,
4127 .setauthsize = chcr_authenc_setauthsize,
4128 }
4129 },
4130 {
3d64bd67 4131 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
2debd332
HJ		 4132 .is_registered = 0,
4133 .alg.aead = {
4134 .base = {
4135 .cra_name = "authenc(hmac(sha384),cbc(aes))",
4136 .cra_driver_name =
4137 "authenc-hmac-sha384-cbc-aes-chcr",
4138 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 4139 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 4140 .cra_ctxsize = sizeof(struct chcr_context) +
4141 sizeof(struct chcr_aead_ctx) +
4142 sizeof(struct chcr_authenc_ctx),
4143
4144 },
4145 .ivsize = AES_BLOCK_SIZE,
4146 .maxauthsize = SHA384_DIGEST_SIZE,
4147 .setkey = chcr_authenc_setkey,
4148 .setauthsize = chcr_authenc_setauthsize,
4149 }
4150 },
4151 {
3d64bd67 4152 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_SHA,
2debd332
HJ		 4153 .is_registered = 0,
4154 .alg.aead = {
4155 .base = {
4156 .cra_name = "authenc(hmac(sha512),cbc(aes))",
4157 .cra_driver_name =
4158 "authenc-hmac-sha512-cbc-aes-chcr",
4159 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 4160 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 4161 .cra_ctxsize = sizeof(struct chcr_context) +
4162 sizeof(struct chcr_aead_ctx) +
4163 sizeof(struct chcr_authenc_ctx),
4164
4165 },
4166 .ivsize = AES_BLOCK_SIZE,
4167 .maxauthsize = SHA512_DIGEST_SIZE,
4168 .setkey = chcr_authenc_setkey,
4169 .setauthsize = chcr_authenc_setauthsize,
4170 }
4171 },
4172 {
3d64bd67 4173 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CBC_NULL,
2debd332
HJ		 4174 .is_registered = 0,
4175 .alg.aead = {
4176 .base = {
4177 .cra_name = "authenc(digest_null,cbc(aes))",
4178 .cra_driver_name =
4179 "authenc-digest_null-cbc-aes-chcr",
4180 .cra_blocksize = AES_BLOCK_SIZE,
e29abda5 4181 .cra_priority = CHCR_AEAD_PRIORITY,
2debd332
HJ		 4182 .cra_ctxsize = sizeof(struct chcr_context) +
4183 sizeof(struct chcr_aead_ctx) +
4184 sizeof(struct chcr_authenc_ctx),
4185
4186 },
4187 .ivsize = AES_BLOCK_SIZE,
4188 .maxauthsize = 0,
4189 .setkey = chcr_aead_digest_null_setkey,
4190 .setauthsize = chcr_authenc_null_setauthsize,
4191 }
4192 },
3d64bd67
HJ		 4193 {
4194 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4195 .is_registered = 0,
4196 .alg.aead = {
4197 .base = {
4198 .cra_name = "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4199 .cra_driver_name =
4200 "authenc-hmac-sha1-rfc3686-ctr-aes-chcr",
4201 .cra_blocksize = 1,
4202 .cra_priority = CHCR_AEAD_PRIORITY,
4203 .cra_ctxsize = sizeof(struct chcr_context) +
4204 sizeof(struct chcr_aead_ctx) +
4205 sizeof(struct chcr_authenc_ctx),
4206
4207 },
4208 .ivsize = CTR_RFC3686_IV_SIZE,
4209 .maxauthsize = SHA1_DIGEST_SIZE,
4210 .setkey = chcr_authenc_setkey,
4211 .setauthsize = chcr_authenc_setauthsize,
4212 }
4213 },
4214 {
4215 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4216 .is_registered = 0,
4217 .alg.aead = {
4218 .base = {
4219
4220 .cra_name = "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4221 .cra_driver_name =
4222 "authenc-hmac-sha256-rfc3686-ctr-aes-chcr",
4223 .cra_blocksize = 1,
4224 .cra_priority = CHCR_AEAD_PRIORITY,
4225 .cra_ctxsize = sizeof(struct chcr_context) +
4226 sizeof(struct chcr_aead_ctx) +
4227 sizeof(struct chcr_authenc_ctx),
4228
4229 },
4230 .ivsize = CTR_RFC3686_IV_SIZE,
4231 .maxauthsize = SHA256_DIGEST_SIZE,
4232 .setkey = chcr_authenc_setkey,
4233 .setauthsize = chcr_authenc_setauthsize,
4234 }
4235 },
4236 {
4237 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4238 .is_registered = 0,
4239 .alg.aead = {
4240 .base = {
4241 .cra_name = "authenc(hmac(sha224),rfc3686(ctr(aes)))",
4242 .cra_driver_name =
4243 "authenc-hmac-sha224-rfc3686-ctr-aes-chcr",
4244 .cra_blocksize = 1,
4245 .cra_priority = CHCR_AEAD_PRIORITY,
4246 .cra_ctxsize = sizeof(struct chcr_context) +
4247 sizeof(struct chcr_aead_ctx) +
4248 sizeof(struct chcr_authenc_ctx),
4249 },
4250 .ivsize = CTR_RFC3686_IV_SIZE,
4251 .maxauthsize = SHA224_DIGEST_SIZE,
4252 .setkey = chcr_authenc_setkey,
4253 .setauthsize = chcr_authenc_setauthsize,
4254 }
4255 },
4256 {
4257 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4258 .is_registered = 0,
4259 .alg.aead = {
4260 .base = {
4261 .cra_name = "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4262 .cra_driver_name =
4263 "authenc-hmac-sha384-rfc3686-ctr-aes-chcr",
4264 .cra_blocksize = 1,
4265 .cra_priority = CHCR_AEAD_PRIORITY,
4266 .cra_ctxsize = sizeof(struct chcr_context) +
4267 sizeof(struct chcr_aead_ctx) +
4268 sizeof(struct chcr_authenc_ctx),
4269
4270 },
4271 .ivsize = CTR_RFC3686_IV_SIZE,
4272 .maxauthsize = SHA384_DIGEST_SIZE,
4273 .setkey = chcr_authenc_setkey,
4274 .setauthsize = chcr_authenc_setauthsize,
4275 }
4276 },
4277 {
4278 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_SHA,
4279 .is_registered = 0,
4280 .alg.aead = {
4281 .base = {
4282 .cra_name = "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4283 .cra_driver_name =
4284 "authenc-hmac-sha512-rfc3686-ctr-aes-chcr",
4285 .cra_blocksize = 1,
4286 .cra_priority = CHCR_AEAD_PRIORITY,
4287 .cra_ctxsize = sizeof(struct chcr_context) +
4288 sizeof(struct chcr_aead_ctx) +
4289 sizeof(struct chcr_authenc_ctx),
4290
4291 },
4292 .ivsize = CTR_RFC3686_IV_SIZE,
4293 .maxauthsize = SHA512_DIGEST_SIZE,
4294 .setkey = chcr_authenc_setkey,
4295 .setauthsize = chcr_authenc_setauthsize,
4296 }
4297 },
4298 {
4299 .type = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_SUB_TYPE_CTR_NULL,
4300 .is_registered = 0,
4301 .alg.aead = {
4302 .base = {
4303 .cra_name = "authenc(digest_null,rfc3686(ctr(aes)))",
4304 .cra_driver_name =
4305 "authenc-digest_null-rfc3686-ctr-aes-chcr",
4306 .cra_blocksize = 1,
4307 .cra_priority = CHCR_AEAD_PRIORITY,
4308 .cra_ctxsize = sizeof(struct chcr_context) +
4309 sizeof(struct chcr_aead_ctx) +
4310 sizeof(struct chcr_authenc_ctx),
4311
4312 },
4313 .ivsize = CTR_RFC3686_IV_SIZE,
4314 .maxauthsize = 0,
4315 .setkey = chcr_aead_digest_null_setkey,
4316 .setauthsize = chcr_authenc_null_setauthsize,
4317 }
4318 },
324429d7
HS		 4319};
4320
4321/*
4322 * chcr_unregister_alg - Deregister crypto algorithms with
4323 * kernel framework.
4324 */
4325static int chcr_unregister_alg(void)
4326{
4327 int i;
4328
4329 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4330 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
7cea6d3e 4331 case CRYPTO_ALG_TYPE_SKCIPHER:
324429d7 4332 if (driver_algs[i].is_registered)
7cea6d3e
AB		 4333 crypto_unregister_skcipher(
4334 &driver_algs[i].alg.skcipher);
324429d7 4335 break;
2debd332
HJ		 4336 case CRYPTO_ALG_TYPE_AEAD:
4337 if (driver_algs[i].is_registered)
4338 crypto_unregister_aead(
4339 &driver_algs[i].alg.aead);
4340 break;
324429d7
HS		 4341 case CRYPTO_ALG_TYPE_AHASH:
4342 if (driver_algs[i].is_registered)
4343 crypto_unregister_ahash(
4344 &driver_algs[i].alg.hash);
4345 break;
4346 }
4347 driver_algs[i].is_registered = 0;
4348 }
4349 return 0;
4350}
4351
4352#define SZ_AHASH_CTX sizeof(struct chcr_context)
4353#define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
4354#define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
324429d7
HS		 4355
4356/*
4357 * chcr_register_alg - Register crypto algorithms with kernel framework.
4358 */
4359static int chcr_register_alg(void)
4360{
4361 struct crypto_alg ai;
4362 struct ahash_alg *a_hash;
4363 int err = 0, i;
4364 char *name = NULL;
4365
4366 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
4367 if (driver_algs[i].is_registered)
4368 continue;
4369 switch (driver_algs[i].type & CRYPTO_ALG_TYPE_MASK) {
7cea6d3e
AB		 4370 case CRYPTO_ALG_TYPE_SKCIPHER:
4371 driver_algs[i].alg.skcipher.base.cra_priority =
b8fd1f41 4372 CHCR_CRA_PRIORITY;
7cea6d3e
AB		 4373 driver_algs[i].alg.skcipher.base.cra_module = THIS_MODULE;
4374 driver_algs[i].alg.skcipher.base.cra_flags =
4375 CRYPTO_ALG_TYPE_SKCIPHER | CRYPTO_ALG_ASYNC |
b8fd1f41 4376 CRYPTO_ALG_NEED_FALLBACK;
7cea6d3e 4377 driver_algs[i].alg.skcipher.base.cra_ctxsize =
b8fd1f41
HJ		 4378 sizeof(struct chcr_context) +
4379 sizeof(struct ablk_ctx);
7cea6d3e
AB		 4380 driver_algs[i].alg.skcipher.base.cra_alignmask = 0;
4381
4382 err = crypto_register_skcipher(&driver_algs[i].alg.skcipher);
4383 name = driver_algs[i].alg.skcipher.base.cra_driver_name;
324429d7 4384 break;
2debd332 4385 case CRYPTO_ALG_TYPE_AEAD:
2debd332 4386 driver_algs[i].alg.aead.base.cra_flags =
3f4a537a 4387 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK;
2debd332
HJ		 4388 driver_algs[i].alg.aead.encrypt = chcr_aead_encrypt;
4389 driver_algs[i].alg.aead.decrypt = chcr_aead_decrypt;
4390 driver_algs[i].alg.aead.init = chcr_aead_cra_init;
4391 driver_algs[i].alg.aead.exit = chcr_aead_cra_exit;
4392 driver_algs[i].alg.aead.base.cra_module = THIS_MODULE;
4393 err = crypto_register_aead(&driver_algs[i].alg.aead);
4394 name = driver_algs[i].alg.aead.base.cra_driver_name;
4395 break;
324429d7
HS		 4396 case CRYPTO_ALG_TYPE_AHASH:
4397 a_hash = &driver_algs[i].alg.hash;
4398 a_hash->update = chcr_ahash_update;
4399 a_hash->final = chcr_ahash_final;
4400 a_hash->finup = chcr_ahash_finup;
4401 a_hash->digest = chcr_ahash_digest;
4402 a_hash->export = chcr_ahash_export;
4403 a_hash->import = chcr_ahash_import;
4404 a_hash->halg.statesize = SZ_AHASH_REQ_CTX;
4405 a_hash->halg.base.cra_priority = CHCR_CRA_PRIORITY;
4406 a_hash->halg.base.cra_module = THIS_MODULE;
6a38f622 4407 a_hash->halg.base.cra_flags = CRYPTO_ALG_ASYNC;
324429d7
HS		 4408 a_hash->halg.base.cra_alignmask = 0;
4409 a_hash->halg.base.cra_exit = NULL;
324429d7
HS		 4410
4411 if (driver_algs[i].type == CRYPTO_ALG_TYPE_HMAC) {
4412 a_hash->halg.base.cra_init = chcr_hmac_cra_init;
4413 a_hash->halg.base.cra_exit = chcr_hmac_cra_exit;
4414 a_hash->init = chcr_hmac_init;
4415 a_hash->setkey = chcr_ahash_setkey;
4416 a_hash->halg.base.cra_ctxsize = SZ_AHASH_H_CTX;
4417 } else {
4418 a_hash->init = chcr_sha_init;
4419 a_hash->halg.base.cra_ctxsize = SZ_AHASH_CTX;
4420 a_hash->halg.base.cra_init = chcr_sha_cra_init;
4421 }
4422 err = crypto_register_ahash(&driver_algs[i].alg.hash);
4423 ai = driver_algs[i].alg.hash.halg.base;
4424 name = ai.cra_driver_name;
4425 break;
4426 }
4427 if (err) {
4428 pr_err("chcr : %s : Algorithm registration failed\n",
4429 name);
4430 goto register_err;
4431 } else {
4432 driver_algs[i].is_registered = 1;
4433 }
4434 }
4435 return 0;
4436
4437register_err:
4438 chcr_unregister_alg();
4439 return err;
4440}
4441
4442/*
4443 * start_crypto - Register the crypto algorithms.
4444 * This should called once when the first device comesup. After this
4445 * kernel will start calling driver APIs for crypto operations.
4446 */
4447int start_crypto(void)
4448{
4449 return chcr_register_alg();
4450}
4451
4452/*
4453 * stop_crypto - Deregister all the crypto algorithms with kernel.
4454 * This should be called once when the last device goes down. After this
4455 * kernel will not call the driver API for crypto operations.
4456 */
4457int stop_crypto(void)
4458{
4459 chcr_unregister_alg();
4460 return 0;
4461}
Ubuntu Hirsute Linux kernel mirror