2 * This file is part of the Chelsio T6 Crypto driver for Linux.
4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved.
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * Written and Maintained by:
35 * Manoj Malviya (manojmalviya@chelsio.com)
36 * Atul Gupta (atul.gupta@chelsio.com)
37 * Jitendra Lulla (jlulla@chelsio.com)
38 * Yeshaswi M R Gowda (yeshaswi@chelsio.com)
39 * Harsh Jain (harsh@chelsio.com)
42 #define pr_fmt(fmt) "chcr:" fmt
44 #include <linux/kernel.h>
45 #include <linux/module.h>
46 #include <linux/crypto.h>
47 #include <linux/skbuff.h>
48 #include <linux/rtnetlink.h>
49 #include <linux/highmem.h>
50 #include <linux/scatterlist.h>
52 #include <crypto/aes.h>
53 #include <crypto/algapi.h>
54 #include <crypto/hash.h>
55 #include <crypto/gcm.h>
56 #include <crypto/sha.h>
57 #include <crypto/authenc.h>
58 #include <crypto/ctr.h>
59 #include <crypto/gf128mul.h>
60 #include <crypto/internal/aead.h>
61 #include <crypto/null.h>
62 #include <crypto/internal/skcipher.h>
63 #include <crypto/aead.h>
64 #include <crypto/scatterwalk.h>
65 #include <crypto/internal/hash.h>
69 #include "chcr_core.h"
70 #include "chcr_algo.h"
71 #include "chcr_crypto.h"
73 #define IV AES_BLOCK_SIZE
75 static unsigned int sgl_ent_len
[] = {
76 0, 0, 16, 24, 40, 48, 64, 72, 88,
77 96, 112, 120, 136, 144, 160, 168, 184,
78 192, 208, 216, 232, 240, 256, 264, 280,
79 288, 304, 312, 328, 336, 352, 360, 376
82 static unsigned int dsgl_ent_len
[] = {
83 0, 32, 32, 48, 48, 64, 64, 80, 80,
84 112, 112, 128, 128, 144, 144, 160, 160,
85 192, 192, 208, 208, 224, 224, 240, 240,
86 272, 272, 288, 288, 304, 304, 320, 320
89 static u32 round_constant
[11] = {
90 0x01000000, 0x02000000, 0x04000000, 0x08000000,
91 0x10000000, 0x20000000, 0x40000000, 0x80000000,
92 0x1B000000, 0x36000000, 0x6C000000
95 static int chcr_handle_cipher_resp(struct skcipher_request
*req
,
96 unsigned char *input
, int err
);
98 static inline struct chcr_aead_ctx
*AEAD_CTX(struct chcr_context
*ctx
)
100 return ctx
->crypto_ctx
->aeadctx
;
103 static inline struct ablk_ctx
*ABLK_CTX(struct chcr_context
*ctx
)
105 return ctx
->crypto_ctx
->ablkctx
;
108 static inline struct hmac_ctx
*HMAC_CTX(struct chcr_context
*ctx
)
110 return ctx
->crypto_ctx
->hmacctx
;
113 static inline struct chcr_gcm_ctx
*GCM_CTX(struct chcr_aead_ctx
*gctx
)
115 return gctx
->ctx
->gcm
;
118 static inline struct chcr_authenc_ctx
*AUTHENC_CTX(struct chcr_aead_ctx
*gctx
)
120 return gctx
->ctx
->authenc
;
123 static inline struct uld_ctx
*ULD_CTX(struct chcr_context
*ctx
)
125 return container_of(ctx
->dev
, struct uld_ctx
, dev
);
128 static inline int is_ofld_imm(const struct sk_buff
*skb
)
130 return (skb
->len
<= SGE_MAX_WR_LEN
);
133 static inline void chcr_init_hctx_per_wr(struct chcr_ahash_req_ctx
*reqctx
)
135 memset(&reqctx
->hctx_wr
, 0, sizeof(struct chcr_hctx_per_wr
));
138 static int sg_nents_xlen(struct scatterlist
*sg
, unsigned int reqlen
,
144 unsigned int skip_len
= 0;
147 if (sg_dma_len(sg
) <= skip
) {
148 skip
-= sg_dma_len(sg
);
157 while (sg
&& reqlen
) {
158 less
= min(reqlen
, sg_dma_len(sg
) - skip_len
);
159 nents
+= DIV_ROUND_UP(less
, entlen
);
167 static inline int get_aead_subtype(struct crypto_aead
*aead
)
169 struct aead_alg
*alg
= crypto_aead_alg(aead
);
170 struct chcr_alg_template
*chcr_crypto_alg
=
171 container_of(alg
, struct chcr_alg_template
, alg
.aead
);
172 return chcr_crypto_alg
->type
& CRYPTO_ALG_SUB_TYPE_MASK
;
175 void chcr_verify_tag(struct aead_request
*req
, u8
*input
, int *err
)
177 u8 temp
[SHA512_DIGEST_SIZE
];
178 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
179 int authsize
= crypto_aead_authsize(tfm
);
180 struct cpl_fw6_pld
*fw6_pld
;
183 fw6_pld
= (struct cpl_fw6_pld
*)input
;
184 if ((get_aead_subtype(tfm
) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106
) ||
185 (get_aead_subtype(tfm
) == CRYPTO_ALG_SUB_TYPE_AEAD_GCM
)) {
186 cmp
= crypto_memneq(&fw6_pld
->data
[2], (fw6_pld
+ 1), authsize
);
189 sg_pcopy_to_buffer(req
->src
, sg_nents(req
->src
), temp
,
190 authsize
, req
->assoclen
+
191 req
->cryptlen
- authsize
);
192 cmp
= crypto_memneq(temp
, (fw6_pld
+ 1), authsize
);
200 static int chcr_inc_wrcount(struct chcr_dev
*dev
)
202 if (dev
->state
== CHCR_DETACH
)
204 atomic_inc(&dev
->inflight
);
208 static inline void chcr_dec_wrcount(struct chcr_dev
*dev
)
210 atomic_dec(&dev
->inflight
);
213 static inline int chcr_handle_aead_resp(struct aead_request
*req
,
214 unsigned char *input
,
217 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
218 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
219 struct chcr_dev
*dev
= a_ctx(tfm
)->dev
;
221 chcr_aead_common_exit(req
);
222 if (reqctx
->verify
== VERIFY_SW
) {
223 chcr_verify_tag(req
, input
, &err
);
224 reqctx
->verify
= VERIFY_HW
;
226 chcr_dec_wrcount(dev
);
227 req
->base
.complete(&req
->base
, err
);
232 static void get_aes_decrypt_key(unsigned char *dec_key
,
233 const unsigned char *key
,
234 unsigned int keylength
)
242 case AES_KEYLENGTH_128BIT
:
243 nk
= KEYLENGTH_4BYTES
;
244 nr
= NUMBER_OF_ROUNDS_10
;
246 case AES_KEYLENGTH_192BIT
:
247 nk
= KEYLENGTH_6BYTES
;
248 nr
= NUMBER_OF_ROUNDS_12
;
250 case AES_KEYLENGTH_256BIT
:
251 nk
= KEYLENGTH_8BYTES
;
252 nr
= NUMBER_OF_ROUNDS_14
;
257 for (i
= 0; i
< nk
; i
++)
258 w_ring
[i
] = be32_to_cpu(*(u32
*)&key
[4 * i
]);
261 temp
= w_ring
[nk
- 1];
262 while (i
+ nk
< (nr
+ 1) * 4) {
265 temp
= (temp
<< 8) | (temp
>> 24);
266 temp
= aes_ks_subword(temp
);
267 temp
^= round_constant
[i
/ nk
];
268 } else if (nk
== 8 && (i
% 4 == 0)) {
269 temp
= aes_ks_subword(temp
);
271 w_ring
[i
% nk
] ^= temp
;
272 temp
= w_ring
[i
% nk
];
276 for (k
= 0, j
= i
% nk
; k
< nk
; k
++) {
277 *((u32
*)dec_key
+ k
) = htonl(w_ring
[j
]);
284 static struct crypto_shash
*chcr_alloc_shash(unsigned int ds
)
286 struct crypto_shash
*base_hash
= ERR_PTR(-EINVAL
);
289 case SHA1_DIGEST_SIZE
:
290 base_hash
= crypto_alloc_shash("sha1", 0, 0);
292 case SHA224_DIGEST_SIZE
:
293 base_hash
= crypto_alloc_shash("sha224", 0, 0);
295 case SHA256_DIGEST_SIZE
:
296 base_hash
= crypto_alloc_shash("sha256", 0, 0);
298 case SHA384_DIGEST_SIZE
:
299 base_hash
= crypto_alloc_shash("sha384", 0, 0);
301 case SHA512_DIGEST_SIZE
:
302 base_hash
= crypto_alloc_shash("sha512", 0, 0);
309 static int chcr_compute_partial_hash(struct shash_desc
*desc
,
310 char *iopad
, char *result_hash
,
313 struct sha1_state sha1_st
;
314 struct sha256_state sha256_st
;
315 struct sha512_state sha512_st
;
318 if (digest_size
== SHA1_DIGEST_SIZE
) {
319 error
= crypto_shash_init(desc
) ?:
320 crypto_shash_update(desc
, iopad
, SHA1_BLOCK_SIZE
) ?:
321 crypto_shash_export(desc
, (void *)&sha1_st
);
322 memcpy(result_hash
, sha1_st
.state
, SHA1_DIGEST_SIZE
);
323 } else if (digest_size
== SHA224_DIGEST_SIZE
) {
324 error
= crypto_shash_init(desc
) ?:
325 crypto_shash_update(desc
, iopad
, SHA256_BLOCK_SIZE
) ?:
326 crypto_shash_export(desc
, (void *)&sha256_st
);
327 memcpy(result_hash
, sha256_st
.state
, SHA256_DIGEST_SIZE
);
329 } else if (digest_size
== SHA256_DIGEST_SIZE
) {
330 error
= crypto_shash_init(desc
) ?:
331 crypto_shash_update(desc
, iopad
, SHA256_BLOCK_SIZE
) ?:
332 crypto_shash_export(desc
, (void *)&sha256_st
);
333 memcpy(result_hash
, sha256_st
.state
, SHA256_DIGEST_SIZE
);
335 } else if (digest_size
== SHA384_DIGEST_SIZE
) {
336 error
= crypto_shash_init(desc
) ?:
337 crypto_shash_update(desc
, iopad
, SHA512_BLOCK_SIZE
) ?:
338 crypto_shash_export(desc
, (void *)&sha512_st
);
339 memcpy(result_hash
, sha512_st
.state
, SHA512_DIGEST_SIZE
);
341 } else if (digest_size
== SHA512_DIGEST_SIZE
) {
342 error
= crypto_shash_init(desc
) ?:
343 crypto_shash_update(desc
, iopad
, SHA512_BLOCK_SIZE
) ?:
344 crypto_shash_export(desc
, (void *)&sha512_st
);
345 memcpy(result_hash
, sha512_st
.state
, SHA512_DIGEST_SIZE
);
348 pr_err("Unknown digest size %d\n", digest_size
);
353 static void chcr_change_order(char *buf
, int ds
)
357 if (ds
== SHA512_DIGEST_SIZE
) {
358 for (i
= 0; i
< (ds
/ sizeof(u64
)); i
++)
359 *((__be64
*)buf
+ i
) =
360 cpu_to_be64(*((u64
*)buf
+ i
));
362 for (i
= 0; i
< (ds
/ sizeof(u32
)); i
++)
363 *((__be32
*)buf
+ i
) =
364 cpu_to_be32(*((u32
*)buf
+ i
));
368 static inline int is_hmac(struct crypto_tfm
*tfm
)
370 struct crypto_alg
*alg
= tfm
->__crt_alg
;
371 struct chcr_alg_template
*chcr_crypto_alg
=
372 container_of(__crypto_ahash_alg(alg
), struct chcr_alg_template
,
374 if (chcr_crypto_alg
->type
== CRYPTO_ALG_TYPE_HMAC
)
379 static inline void dsgl_walk_init(struct dsgl_walk
*walk
,
380 struct cpl_rx_phys_dsgl
*dsgl
)
384 walk
->to
= (struct phys_sge_pairs
*)(dsgl
+ 1);
387 static inline void dsgl_walk_end(struct dsgl_walk
*walk
, unsigned short qid
,
390 struct cpl_rx_phys_dsgl
*phys_cpl
;
392 phys_cpl
= walk
->dsgl
;
394 phys_cpl
->op_to_tid
= htonl(CPL_RX_PHYS_DSGL_OPCODE_V(CPL_RX_PHYS_DSGL
)
395 | CPL_RX_PHYS_DSGL_ISRDMA_V(0));
396 phys_cpl
->pcirlxorder_to_noofsgentr
=
397 htonl(CPL_RX_PHYS_DSGL_PCIRLXORDER_V(0) |
398 CPL_RX_PHYS_DSGL_PCINOSNOOP_V(0) |
399 CPL_RX_PHYS_DSGL_PCITPHNTENB_V(0) |
400 CPL_RX_PHYS_DSGL_PCITPHNT_V(0) |
401 CPL_RX_PHYS_DSGL_DCAID_V(0) |
402 CPL_RX_PHYS_DSGL_NOOFSGENTR_V(walk
->nents
));
403 phys_cpl
->rss_hdr_int
.opcode
= CPL_RX_PHYS_ADDR
;
404 phys_cpl
->rss_hdr_int
.qid
= htons(qid
);
405 phys_cpl
->rss_hdr_int
.hash_val
= 0;
406 phys_cpl
->rss_hdr_int
.channel
= pci_chan_id
;
409 static inline void dsgl_walk_add_page(struct dsgl_walk
*walk
,
418 walk
->to
->len
[j
% 8] = htons(size
);
419 walk
->to
->addr
[j
% 8] = cpu_to_be64(addr
);
426 static void dsgl_walk_add_sg(struct dsgl_walk
*walk
,
427 struct scatterlist
*sg
,
432 unsigned int left_size
= slen
, len
= 0;
433 unsigned int j
= walk
->nents
;
439 if (sg_dma_len(sg
) <= skip
) {
440 skip
-= sg_dma_len(sg
);
449 while (left_size
&& sg
) {
450 len
= min_t(u32
, left_size
, sg_dma_len(sg
) - skip_len
);
453 ent_len
= min_t(u32
, len
, CHCR_DST_SG_SIZE
);
454 walk
->to
->len
[j
% 8] = htons(ent_len
);
455 walk
->to
->addr
[j
% 8] = cpu_to_be64(sg_dma_address(sg
) +
464 walk
->last_sg_len
= min_t(u32
, left_size
, sg_dma_len(sg
) -
465 skip_len
) + skip_len
;
466 left_size
-= min_t(u32
, left_size
, sg_dma_len(sg
) - skip_len
);
473 static inline void ulptx_walk_init(struct ulptx_walk
*walk
,
474 struct ulptx_sgl
*ulp
)
479 walk
->pair
= ulp
->sge
;
480 walk
->last_sg
= NULL
;
481 walk
->last_sg_len
= 0;
484 static inline void ulptx_walk_end(struct ulptx_walk
*walk
)
486 walk
->sgl
->cmd_nsge
= htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL
) |
487 ULPTX_NSGE_V(walk
->nents
));
491 static inline void ulptx_walk_add_page(struct ulptx_walk
*walk
,
498 if (walk
->nents
== 0) {
499 walk
->sgl
->len0
= cpu_to_be32(size
);
500 walk
->sgl
->addr0
= cpu_to_be64(addr
);
502 walk
->pair
->addr
[walk
->pair_idx
] = cpu_to_be64(addr
);
503 walk
->pair
->len
[walk
->pair_idx
] = cpu_to_be32(size
);
504 walk
->pair_idx
= !walk
->pair_idx
;
511 static void ulptx_walk_add_sg(struct ulptx_walk
*walk
,
512 struct scatterlist
*sg
,
523 if (sg_dma_len(sg
) <= skip
) {
524 skip
-= sg_dma_len(sg
);
532 WARN(!sg
, "SG should not be null here\n");
533 if (sg
&& (walk
->nents
== 0)) {
534 small
= min_t(unsigned int, sg_dma_len(sg
) - skip_len
, len
);
535 sgmin
= min_t(unsigned int, small
, CHCR_SRC_SG_SIZE
);
536 walk
->sgl
->len0
= cpu_to_be32(sgmin
);
537 walk
->sgl
->addr0
= cpu_to_be64(sg_dma_address(sg
) + skip_len
);
541 walk
->last_sg_len
= sgmin
+ skip_len
;
543 if (sg_dma_len(sg
) == skip_len
) {
550 small
= min(sg_dma_len(sg
) - skip_len
, len
);
551 sgmin
= min_t(unsigned int, small
, CHCR_SRC_SG_SIZE
);
552 walk
->pair
->len
[walk
->pair_idx
] = cpu_to_be32(sgmin
);
553 walk
->pair
->addr
[walk
->pair_idx
] =
554 cpu_to_be64(sg_dma_address(sg
) + skip_len
);
555 walk
->pair_idx
= !walk
->pair_idx
;
562 walk
->last_sg_len
= skip_len
;
563 if (sg_dma_len(sg
) == skip_len
) {
570 static inline int get_cryptoalg_subtype(struct crypto_skcipher
*tfm
)
572 struct skcipher_alg
*alg
= crypto_skcipher_alg(tfm
);
573 struct chcr_alg_template
*chcr_crypto_alg
=
574 container_of(alg
, struct chcr_alg_template
, alg
.skcipher
);
576 return chcr_crypto_alg
->type
& CRYPTO_ALG_SUB_TYPE_MASK
;
579 static int cxgb4_is_crypto_q_full(struct net_device
*dev
, unsigned int idx
)
581 struct adapter
*adap
= netdev2adap(dev
);
582 struct sge_uld_txq_info
*txq_info
=
583 adap
->sge
.uld_txq_info
[CXGB4_TX_CRYPTO
];
584 struct sge_uld_txq
*txq
;
588 txq
= &txq_info
->uldtxq
[idx
];
589 spin_lock(&txq
->sendq
.lock
);
592 spin_unlock(&txq
->sendq
.lock
);
597 static int generate_copy_rrkey(struct ablk_ctx
*ablkctx
,
598 struct _key_ctx
*key_ctx
)
600 if (ablkctx
->ciph_mode
== CHCR_SCMD_CIPHER_MODE_AES_CBC
) {
601 memcpy(key_ctx
->key
, ablkctx
->rrkey
, ablkctx
->enckey_len
);
604 ablkctx
->key
+ (ablkctx
->enckey_len
>> 1),
605 ablkctx
->enckey_len
>> 1);
606 memcpy(key_ctx
->key
+ (ablkctx
->enckey_len
>> 1),
607 ablkctx
->rrkey
, ablkctx
->enckey_len
>> 1);
612 static int chcr_hash_ent_in_wr(struct scatterlist
*src
,
615 unsigned int srcskip
)
619 int soffset
= 0, sless
;
621 if (sg_dma_len(src
) == srcskip
) {
625 while (src
&& space
> (sgl_ent_len
[srcsg
+ 1])) {
626 sless
= min_t(unsigned int, sg_dma_len(src
) - soffset
- srcskip
,
631 if (sg_dma_len(src
) == (soffset
+ srcskip
)) {
640 static int chcr_sg_ent_in_wr(struct scatterlist
*src
,
641 struct scatterlist
*dst
,
644 unsigned int srcskip
,
645 unsigned int dstskip
)
647 int srclen
= 0, dstlen
= 0;
648 int srcsg
= minsg
, dstsg
= minsg
;
649 int offset
= 0, soffset
= 0, less
, sless
= 0;
651 if (sg_dma_len(src
) == srcskip
) {
655 if (sg_dma_len(dst
) == dstskip
) {
661 space
> (sgl_ent_len
[srcsg
+ 1] + dsgl_ent_len
[dstsg
])) {
662 sless
= min_t(unsigned int, sg_dma_len(src
) - srcskip
- soffset
,
667 while (dst
&& ((dstsg
+ 1) <= MAX_DSGL_ENT
) &&
668 space
> (sgl_ent_len
[srcsg
] + dsgl_ent_len
[dstsg
+ 1])) {
669 if (srclen
<= dstlen
)
671 less
= min_t(unsigned int, sg_dma_len(dst
) - offset
-
672 dstskip
, CHCR_DST_SG_SIZE
);
675 if ((offset
+ dstskip
) == sg_dma_len(dst
)) {
683 if ((soffset
+ srcskip
) == sg_dma_len(src
)) {
690 return min(srclen
, dstlen
);
693 static int chcr_cipher_fallback(struct crypto_sync_skcipher
*cipher
,
695 struct scatterlist
*src
,
696 struct scatterlist
*dst
,
699 unsigned short op_type
)
703 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq
, cipher
);
705 skcipher_request_set_sync_tfm(subreq
, cipher
);
706 skcipher_request_set_callback(subreq
, flags
, NULL
, NULL
);
707 skcipher_request_set_crypt(subreq
, src
, dst
,
710 err
= op_type
? crypto_skcipher_decrypt(subreq
) :
711 crypto_skcipher_encrypt(subreq
);
712 skcipher_request_zero(subreq
);
718 static inline int get_qidxs(struct crypto_async_request
*req
,
719 unsigned int *txqidx
, unsigned int *rxqidx
)
721 struct crypto_tfm
*tfm
= req
->tfm
;
724 switch (tfm
->__crt_alg
->cra_flags
& CRYPTO_ALG_TYPE_MASK
) {
725 case CRYPTO_ALG_TYPE_AEAD
:
727 struct aead_request
*aead_req
=
728 container_of(req
, struct aead_request
, base
);
729 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(aead_req
);
730 *txqidx
= reqctx
->txqidx
;
731 *rxqidx
= reqctx
->rxqidx
;
734 case CRYPTO_ALG_TYPE_SKCIPHER
:
736 struct skcipher_request
*sk_req
=
737 container_of(req
, struct skcipher_request
, base
);
738 struct chcr_skcipher_req_ctx
*reqctx
=
739 skcipher_request_ctx(sk_req
);
740 *txqidx
= reqctx
->txqidx
;
741 *rxqidx
= reqctx
->rxqidx
;
744 case CRYPTO_ALG_TYPE_AHASH
:
746 struct ahash_request
*ahash_req
=
747 container_of(req
, struct ahash_request
, base
);
748 struct chcr_ahash_req_ctx
*reqctx
=
749 ahash_request_ctx(ahash_req
);
750 *txqidx
= reqctx
->txqidx
;
751 *rxqidx
= reqctx
->rxqidx
;
756 /* should never get here */
763 static inline void create_wreq(struct chcr_context
*ctx
,
764 struct chcr_wr
*chcr_req
,
765 struct crypto_async_request
*req
,
772 struct uld_ctx
*u_ctx
= ULD_CTX(ctx
);
773 unsigned int tx_channel_id
, rx_channel_id
;
774 unsigned int txqidx
= 0, rxqidx
= 0;
775 unsigned int qid
, fid
;
777 get_qidxs(req
, &txqidx
, &rxqidx
);
778 qid
= u_ctx
->lldi
.rxq_ids
[rxqidx
];
779 fid
= u_ctx
->lldi
.rxq_ids
[0];
780 tx_channel_id
= txqidx
/ ctx
->txq_perchan
;
781 rx_channel_id
= rxqidx
/ ctx
->rxq_perchan
;
784 chcr_req
->wreq
.op_to_cctx_size
= FILL_WR_OP_CCTX_SIZE
;
785 chcr_req
->wreq
.pld_size_hash_size
=
786 htonl(FW_CRYPTO_LOOKASIDE_WR_HASH_SIZE_V(hash_sz
));
787 chcr_req
->wreq
.len16_pkd
=
788 htonl(FW_CRYPTO_LOOKASIDE_WR_LEN16_V(DIV_ROUND_UP(len16
, 16)));
789 chcr_req
->wreq
.cookie
= cpu_to_be64((uintptr_t)req
);
790 chcr_req
->wreq
.rx_chid_to_rx_q_id
= FILL_WR_RX_Q_ID(rx_channel_id
, qid
,
793 chcr_req
->ulptx
.cmd_dest
= FILL_ULPTX_CMD_DEST(tx_channel_id
, fid
);
794 chcr_req
->ulptx
.len
= htonl((DIV_ROUND_UP(len16
, 16) -
795 ((sizeof(chcr_req
->wreq
)) >> 4)));
796 chcr_req
->sc_imm
.cmd_more
= FILL_CMD_MORE(!imm
);
797 chcr_req
->sc_imm
.len
= cpu_to_be32(sizeof(struct cpl_tx_sec_pdu
) +
798 sizeof(chcr_req
->key_ctx
) + sc_len
);
802 * create_cipher_wr - form the WR for cipher operations
804 * @ctx: crypto driver context of the request.
805 * @qid: ingress qid where response of this WR should be received.
806 * @op_type: encryption or decryption
808 static struct sk_buff
*create_cipher_wr(struct cipher_wr_param
*wrparam
)
810 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(wrparam
->req
);
811 struct chcr_context
*ctx
= c_ctx(tfm
);
812 struct ablk_ctx
*ablkctx
= ABLK_CTX(ctx
);
813 struct sk_buff
*skb
= NULL
;
814 struct chcr_wr
*chcr_req
;
815 struct cpl_rx_phys_dsgl
*phys_cpl
;
816 struct ulptx_sgl
*ulptx
;
817 struct chcr_skcipher_req_ctx
*reqctx
=
818 skcipher_request_ctx(wrparam
->req
);
819 unsigned int temp
= 0, transhdr_len
, dst_size
;
822 unsigned int kctx_len
;
823 gfp_t flags
= wrparam
->req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
?
824 GFP_KERNEL
: GFP_ATOMIC
;
825 struct adapter
*adap
= padap(ctx
->dev
);
826 unsigned int rx_channel_id
= reqctx
->rxqidx
/ ctx
->rxq_perchan
;
828 nents
= sg_nents_xlen(reqctx
->dstsg
, wrparam
->bytes
, CHCR_DST_SG_SIZE
,
830 dst_size
= get_space_for_phys_dsgl(nents
);
831 kctx_len
= roundup(ablkctx
->enckey_len
, 16);
832 transhdr_len
= CIPHER_TRANSHDR_SIZE(kctx_len
, dst_size
);
833 nents
= sg_nents_xlen(reqctx
->srcsg
, wrparam
->bytes
,
834 CHCR_SRC_SG_SIZE
, reqctx
->src_ofst
);
835 temp
= reqctx
->imm
? roundup(wrparam
->bytes
, 16) :
836 (sgl_len(nents
) * 8);
837 transhdr_len
+= temp
;
838 transhdr_len
= roundup(transhdr_len
, 16);
839 skb
= alloc_skb(SGE_MAX_WR_LEN
, flags
);
844 chcr_req
= __skb_put_zero(skb
, transhdr_len
);
845 chcr_req
->sec_cpl
.op_ivinsrtofst
=
846 FILL_SEC_CPL_OP_IVINSR(rx_channel_id
, 2, 1);
848 chcr_req
->sec_cpl
.pldlen
= htonl(IV
+ wrparam
->bytes
);
849 chcr_req
->sec_cpl
.aadstart_cipherstop_hi
=
850 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, IV
+ 1, 0);
852 chcr_req
->sec_cpl
.cipherstop_lo_authinsert
=
853 FILL_SEC_CPL_AUTHINSERT(0, 0, 0, 0);
854 chcr_req
->sec_cpl
.seqno_numivs
= FILL_SEC_CPL_SCMD0_SEQNO(reqctx
->op
, 0,
857 chcr_req
->sec_cpl
.ivgen_hdrlen
= FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 0,
860 chcr_req
->key_ctx
.ctx_hdr
= ablkctx
->key_ctx_hdr
;
861 if ((reqctx
->op
== CHCR_DECRYPT_OP
) &&
862 (!(get_cryptoalg_subtype(tfm
) ==
863 CRYPTO_ALG_SUB_TYPE_CTR
)) &&
864 (!(get_cryptoalg_subtype(tfm
) ==
865 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686
))) {
866 generate_copy_rrkey(ablkctx
, &chcr_req
->key_ctx
);
868 if ((ablkctx
->ciph_mode
== CHCR_SCMD_CIPHER_MODE_AES_CBC
) ||
869 (ablkctx
->ciph_mode
== CHCR_SCMD_CIPHER_MODE_AES_CTR
)) {
870 memcpy(chcr_req
->key_ctx
.key
, ablkctx
->key
,
871 ablkctx
->enckey_len
);
873 memcpy(chcr_req
->key_ctx
.key
, ablkctx
->key
+
874 (ablkctx
->enckey_len
>> 1),
875 ablkctx
->enckey_len
>> 1);
876 memcpy(chcr_req
->key_ctx
.key
+
877 (ablkctx
->enckey_len
>> 1),
879 ablkctx
->enckey_len
>> 1);
882 phys_cpl
= (struct cpl_rx_phys_dsgl
*)((u8
*)(chcr_req
+ 1) + kctx_len
);
883 ulptx
= (struct ulptx_sgl
*)((u8
*)(phys_cpl
+ 1) + dst_size
);
884 chcr_add_cipher_src_ent(wrparam
->req
, ulptx
, wrparam
);
885 chcr_add_cipher_dst_ent(wrparam
->req
, phys_cpl
, wrparam
, wrparam
->qid
);
887 atomic_inc(&adap
->chcr_stats
.cipher_rqst
);
888 temp
= sizeof(struct cpl_rx_phys_dsgl
) + dst_size
+ kctx_len
+ IV
889 + (reqctx
->imm
? (wrparam
->bytes
) : 0);
890 create_wreq(c_ctx(tfm
), chcr_req
, &(wrparam
->req
->base
), reqctx
->imm
, 0,
892 ablkctx
->ciph_mode
== CHCR_SCMD_CIPHER_MODE_AES_CBC
);
895 if (reqctx
->op
&& (ablkctx
->ciph_mode
==
896 CHCR_SCMD_CIPHER_MODE_AES_CBC
))
897 sg_pcopy_to_buffer(wrparam
->req
->src
,
898 sg_nents(wrparam
->req
->src
), wrparam
->req
->iv
, 16,
899 reqctx
->processed
+ wrparam
->bytes
- AES_BLOCK_SIZE
);
903 return ERR_PTR(error
);
906 static inline int chcr_keyctx_ck_size(unsigned int keylen
)
910 if (keylen
== AES_KEYSIZE_128
)
911 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_128
;
912 else if (keylen
== AES_KEYSIZE_192
)
913 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_192
;
914 else if (keylen
== AES_KEYSIZE_256
)
915 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_256
;
921 static int chcr_cipher_fallback_setkey(struct crypto_skcipher
*cipher
,
925 struct ablk_ctx
*ablkctx
= ABLK_CTX(c_ctx(cipher
));
927 crypto_sync_skcipher_clear_flags(ablkctx
->sw_cipher
,
928 CRYPTO_TFM_REQ_MASK
);
929 crypto_sync_skcipher_set_flags(ablkctx
->sw_cipher
,
930 cipher
->base
.crt_flags
& CRYPTO_TFM_REQ_MASK
);
931 return crypto_sync_skcipher_setkey(ablkctx
->sw_cipher
, key
, keylen
);
934 static int chcr_aes_cbc_setkey(struct crypto_skcipher
*cipher
,
938 struct ablk_ctx
*ablkctx
= ABLK_CTX(c_ctx(cipher
));
939 unsigned int ck_size
, context_size
;
943 err
= chcr_cipher_fallback_setkey(cipher
, key
, keylen
);
947 ck_size
= chcr_keyctx_ck_size(keylen
);
948 alignment
= ck_size
== CHCR_KEYCTX_CIPHER_KEY_SIZE_192
? 8 : 0;
949 memcpy(ablkctx
->key
, key
, keylen
);
950 ablkctx
->enckey_len
= keylen
;
951 get_aes_decrypt_key(ablkctx
->rrkey
, ablkctx
->key
, keylen
<< 3);
952 context_size
= (KEY_CONTEXT_HDR_SALT_AND_PAD
+
953 keylen
+ alignment
) >> 4;
955 ablkctx
->key_ctx_hdr
= FILL_KEY_CTX_HDR(ck_size
, CHCR_KEYCTX_NO_KEY
,
957 ablkctx
->ciph_mode
= CHCR_SCMD_CIPHER_MODE_AES_CBC
;
960 ablkctx
->enckey_len
= 0;
965 static int chcr_aes_ctr_setkey(struct crypto_skcipher
*cipher
,
969 struct ablk_ctx
*ablkctx
= ABLK_CTX(c_ctx(cipher
));
970 unsigned int ck_size
, context_size
;
974 err
= chcr_cipher_fallback_setkey(cipher
, key
, keylen
);
977 ck_size
= chcr_keyctx_ck_size(keylen
);
978 alignment
= (ck_size
== CHCR_KEYCTX_CIPHER_KEY_SIZE_192
) ? 8 : 0;
979 memcpy(ablkctx
->key
, key
, keylen
);
980 ablkctx
->enckey_len
= keylen
;
981 context_size
= (KEY_CONTEXT_HDR_SALT_AND_PAD
+
982 keylen
+ alignment
) >> 4;
984 ablkctx
->key_ctx_hdr
= FILL_KEY_CTX_HDR(ck_size
, CHCR_KEYCTX_NO_KEY
,
986 ablkctx
->ciph_mode
= CHCR_SCMD_CIPHER_MODE_AES_CTR
;
990 ablkctx
->enckey_len
= 0;
995 static int chcr_aes_rfc3686_setkey(struct crypto_skcipher
*cipher
,
999 struct ablk_ctx
*ablkctx
= ABLK_CTX(c_ctx(cipher
));
1000 unsigned int ck_size
, context_size
;
1004 if (keylen
< CTR_RFC3686_NONCE_SIZE
)
1006 memcpy(ablkctx
->nonce
, key
+ (keylen
- CTR_RFC3686_NONCE_SIZE
),
1007 CTR_RFC3686_NONCE_SIZE
);
1009 keylen
-= CTR_RFC3686_NONCE_SIZE
;
1010 err
= chcr_cipher_fallback_setkey(cipher
, key
, keylen
);
1014 ck_size
= chcr_keyctx_ck_size(keylen
);
1015 alignment
= (ck_size
== CHCR_KEYCTX_CIPHER_KEY_SIZE_192
) ? 8 : 0;
1016 memcpy(ablkctx
->key
, key
, keylen
);
1017 ablkctx
->enckey_len
= keylen
;
1018 context_size
= (KEY_CONTEXT_HDR_SALT_AND_PAD
+
1019 keylen
+ alignment
) >> 4;
1021 ablkctx
->key_ctx_hdr
= FILL_KEY_CTX_HDR(ck_size
, CHCR_KEYCTX_NO_KEY
,
1022 0, 0, context_size
);
1023 ablkctx
->ciph_mode
= CHCR_SCMD_CIPHER_MODE_AES_CTR
;
1027 ablkctx
->enckey_len
= 0;
1031 static void ctr_add_iv(u8
*dstiv
, u8
*srciv
, u32 add
)
1033 unsigned int size
= AES_BLOCK_SIZE
;
1034 __be32
*b
= (__be32
*)(dstiv
+ size
);
1037 memcpy(dstiv
, srciv
, AES_BLOCK_SIZE
);
1038 for (; size
>= 4; size
-= 4) {
1039 prev
= be32_to_cpu(*--b
);
1041 *b
= cpu_to_be32(c
);
1049 static unsigned int adjust_ctr_overflow(u8
*iv
, u32 bytes
)
1051 __be32
*b
= (__be32
*)(iv
+ AES_BLOCK_SIZE
);
1053 u32 temp
= be32_to_cpu(*--b
);
1056 c
= (u64
)temp
+ 1; // No of block can processed withou overflow
1057 if ((bytes
/ AES_BLOCK_SIZE
) > c
)
1058 bytes
= c
* AES_BLOCK_SIZE
;
1062 static int chcr_update_tweak(struct skcipher_request
*req
, u8
*iv
,
1065 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
1066 struct ablk_ctx
*ablkctx
= ABLK_CTX(c_ctx(tfm
));
1067 struct chcr_skcipher_req_ctx
*reqctx
= skcipher_request_ctx(req
);
1068 struct crypto_aes_ctx aes
;
1071 unsigned int keylen
;
1072 int round
= reqctx
->last_req_len
/ AES_BLOCK_SIZE
;
1073 int round8
= round
/ 8;
1075 memcpy(iv
, reqctx
->iv
, AES_BLOCK_SIZE
);
1077 keylen
= ablkctx
->enckey_len
/ 2;
1078 key
= ablkctx
->key
+ keylen
;
1079 ret
= aes_expandkey(&aes
, key
, keylen
);
1082 aes_encrypt(&aes
, iv
, iv
);
1083 for (i
= 0; i
< round8
; i
++)
1084 gf128mul_x8_ble((le128
*)iv
, (le128
*)iv
);
1086 for (i
= 0; i
< (round
% 8); i
++)
1087 gf128mul_x_ble((le128
*)iv
, (le128
*)iv
);
1090 aes_decrypt(&aes
, iv
, iv
);
1092 memzero_explicit(&aes
, sizeof(aes
));
1096 static int chcr_update_cipher_iv(struct skcipher_request
*req
,
1097 struct cpl_fw6_pld
*fw6_pld
, u8
*iv
)
1099 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
1100 struct chcr_skcipher_req_ctx
*reqctx
= skcipher_request_ctx(req
);
1101 int subtype
= get_cryptoalg_subtype(tfm
);
1104 if (subtype
== CRYPTO_ALG_SUB_TYPE_CTR
)
1105 ctr_add_iv(iv
, req
->iv
, (reqctx
->processed
/
1107 else if (subtype
== CRYPTO_ALG_SUB_TYPE_CTR_RFC3686
)
1108 *(__be32
*)(reqctx
->iv
+ CTR_RFC3686_NONCE_SIZE
+
1109 CTR_RFC3686_IV_SIZE
) = cpu_to_be32((reqctx
->processed
/
1110 AES_BLOCK_SIZE
) + 1);
1111 else if (subtype
== CRYPTO_ALG_SUB_TYPE_XTS
)
1112 ret
= chcr_update_tweak(req
, iv
, 0);
1113 else if (subtype
== CRYPTO_ALG_SUB_TYPE_CBC
) {
1115 /*Updated before sending last WR*/
1116 memcpy(iv
, req
->iv
, AES_BLOCK_SIZE
);
1118 memcpy(iv
, &fw6_pld
->data
[2], AES_BLOCK_SIZE
);
1125 /* We need separate function for final iv because in rfc3686 Initial counter
1126 * starts from 1 and buffer size of iv is 8 byte only which remains constant
1127 * for subsequent update requests
1130 static int chcr_final_cipher_iv(struct skcipher_request
*req
,
1131 struct cpl_fw6_pld
*fw6_pld
, u8
*iv
)
1133 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
1134 struct chcr_skcipher_req_ctx
*reqctx
= skcipher_request_ctx(req
);
1135 int subtype
= get_cryptoalg_subtype(tfm
);
1138 if (subtype
== CRYPTO_ALG_SUB_TYPE_CTR
)
1139 ctr_add_iv(iv
, req
->iv
, DIV_ROUND_UP(reqctx
->processed
,
1141 else if (subtype
== CRYPTO_ALG_SUB_TYPE_XTS
) {
1142 if (!reqctx
->partial_req
)
1143 memcpy(iv
, reqctx
->iv
, AES_BLOCK_SIZE
);
1145 ret
= chcr_update_tweak(req
, iv
, 1);
1147 else if (subtype
== CRYPTO_ALG_SUB_TYPE_CBC
) {
1148 /*Already updated for Decrypt*/
1150 memcpy(iv
, &fw6_pld
->data
[2], AES_BLOCK_SIZE
);
1157 static int chcr_handle_cipher_resp(struct skcipher_request
*req
,
1158 unsigned char *input
, int err
)
1160 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
1161 struct chcr_context
*ctx
= c_ctx(tfm
);
1162 struct uld_ctx
*u_ctx
= ULD_CTX(c_ctx(tfm
));
1163 struct ablk_ctx
*ablkctx
= ABLK_CTX(c_ctx(tfm
));
1164 struct sk_buff
*skb
;
1165 struct cpl_fw6_pld
*fw6_pld
= (struct cpl_fw6_pld
*)input
;
1166 struct chcr_skcipher_req_ctx
*reqctx
= skcipher_request_ctx(req
);
1167 struct cipher_wr_param wrparam
;
1168 struct chcr_dev
*dev
= c_ctx(tfm
)->dev
;
1173 if (req
->cryptlen
== reqctx
->processed
) {
1174 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm
))->lldi
.pdev
->dev
,
1176 err
= chcr_final_cipher_iv(req
, fw6_pld
, req
->iv
);
1181 bytes
= chcr_sg_ent_in_wr(reqctx
->srcsg
, reqctx
->dstsg
, 0,
1182 CIP_SPACE_LEFT(ablkctx
->enckey_len
),
1183 reqctx
->src_ofst
, reqctx
->dst_ofst
);
1184 if ((bytes
+ reqctx
->processed
) >= req
->cryptlen
)
1185 bytes
= req
->cryptlen
- reqctx
->processed
;
1187 bytes
= rounddown(bytes
, 16);
1189 /*CTR mode counter overfloa*/
1190 bytes
= req
->cryptlen
- reqctx
->processed
;
1192 err
= chcr_update_cipher_iv(req
, fw6_pld
, reqctx
->iv
);
1196 if (unlikely(bytes
== 0)) {
1197 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm
))->lldi
.pdev
->dev
,
1199 err
= chcr_cipher_fallback(ablkctx
->sw_cipher
,
1209 if (get_cryptoalg_subtype(tfm
) ==
1210 CRYPTO_ALG_SUB_TYPE_CTR
)
1211 bytes
= adjust_ctr_overflow(reqctx
->iv
, bytes
);
1212 wrparam
.qid
= u_ctx
->lldi
.rxq_ids
[reqctx
->rxqidx
];
1214 wrparam
.bytes
= bytes
;
1215 skb
= create_cipher_wr(&wrparam
);
1217 pr_err("chcr : %s : Failed to form WR. No memory\n", __func__
);
1221 skb
->dev
= u_ctx
->lldi
.ports
[0];
1222 set_wr_txq(skb
, CPL_PRIORITY_DATA
, reqctx
->txqidx
);
1224 reqctx
->last_req_len
= bytes
;
1225 reqctx
->processed
+= bytes
;
1226 if (get_cryptoalg_subtype(tfm
) ==
1227 CRYPTO_ALG_SUB_TYPE_CBC
&& req
->base
.flags
==
1228 CRYPTO_TFM_REQ_MAY_SLEEP
) {
1229 complete(&ctx
->cbc_aes_aio_done
);
1233 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm
))->lldi
.pdev
->dev
, req
);
1235 if (get_cryptoalg_subtype(tfm
) ==
1236 CRYPTO_ALG_SUB_TYPE_CBC
&& req
->base
.flags
==
1237 CRYPTO_TFM_REQ_MAY_SLEEP
) {
1238 complete(&ctx
->cbc_aes_aio_done
);
1240 chcr_dec_wrcount(dev
);
1241 req
->base
.complete(&req
->base
, err
);
1245 static int process_cipher(struct skcipher_request
*req
,
1247 struct sk_buff
**skb
,
1248 unsigned short op_type
)
1250 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
1251 unsigned int ivsize
= crypto_skcipher_ivsize(tfm
);
1252 struct chcr_skcipher_req_ctx
*reqctx
= skcipher_request_ctx(req
);
1253 struct ablk_ctx
*ablkctx
= ABLK_CTX(c_ctx(tfm
));
1254 struct cipher_wr_param wrparam
;
1255 int bytes
, err
= -EINVAL
;
1257 reqctx
->processed
= 0;
1258 reqctx
->partial_req
= 0;
1261 if ((ablkctx
->enckey_len
== 0) || (ivsize
> AES_BLOCK_SIZE
) ||
1262 (req
->cryptlen
== 0) ||
1263 (req
->cryptlen
% crypto_skcipher_blocksize(tfm
))) {
1264 pr_err("AES: Invalid value of Key Len %d nbytes %d IV Len %d\n",
1265 ablkctx
->enckey_len
, req
->cryptlen
, ivsize
);
1269 err
= chcr_cipher_dma_map(&ULD_CTX(c_ctx(tfm
))->lldi
.pdev
->dev
, req
);
1272 if (req
->cryptlen
< (SGE_MAX_WR_LEN
- (sizeof(struct chcr_wr
) +
1274 sizeof(struct cpl_rx_phys_dsgl
) +
1277 /* Can be sent as Imm*/
1278 unsigned int dnents
= 0, transhdr_len
, phys_dsgl
, kctx_len
;
1280 dnents
= sg_nents_xlen(req
->dst
, req
->cryptlen
,
1281 CHCR_DST_SG_SIZE
, 0);
1282 phys_dsgl
= get_space_for_phys_dsgl(dnents
);
1283 kctx_len
= roundup(ablkctx
->enckey_len
, 16);
1284 transhdr_len
= CIPHER_TRANSHDR_SIZE(kctx_len
, phys_dsgl
);
1285 reqctx
->imm
= (transhdr_len
+ IV
+ req
->cryptlen
) <=
1287 bytes
= IV
+ req
->cryptlen
;
1294 bytes
= chcr_sg_ent_in_wr(req
->src
, req
->dst
, 0,
1295 CIP_SPACE_LEFT(ablkctx
->enckey_len
),
1297 if ((bytes
+ reqctx
->processed
) >= req
->cryptlen
)
1298 bytes
= req
->cryptlen
- reqctx
->processed
;
1300 bytes
= rounddown(bytes
, 16);
1302 bytes
= req
->cryptlen
;
1304 if (get_cryptoalg_subtype(tfm
) ==
1305 CRYPTO_ALG_SUB_TYPE_CTR
) {
1306 bytes
= adjust_ctr_overflow(req
->iv
, bytes
);
1308 if (get_cryptoalg_subtype(tfm
) ==
1309 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686
) {
1310 memcpy(reqctx
->iv
, ablkctx
->nonce
, CTR_RFC3686_NONCE_SIZE
);
1311 memcpy(reqctx
->iv
+ CTR_RFC3686_NONCE_SIZE
, req
->iv
,
1312 CTR_RFC3686_IV_SIZE
);
1314 /* initialize counter portion of counter block */
1315 *(__be32
*)(reqctx
->iv
+ CTR_RFC3686_NONCE_SIZE
+
1316 CTR_RFC3686_IV_SIZE
) = cpu_to_be32(1);
1320 memcpy(reqctx
->iv
, req
->iv
, IV
);
1322 if (unlikely(bytes
== 0)) {
1323 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm
))->lldi
.pdev
->dev
,
1325 err
= chcr_cipher_fallback(ablkctx
->sw_cipher
,
1334 reqctx
->op
= op_type
;
1335 reqctx
->srcsg
= req
->src
;
1336 reqctx
->dstsg
= req
->dst
;
1337 reqctx
->src_ofst
= 0;
1338 reqctx
->dst_ofst
= 0;
1341 wrparam
.bytes
= bytes
;
1342 *skb
= create_cipher_wr(&wrparam
);
1344 err
= PTR_ERR(*skb
);
1347 reqctx
->processed
= bytes
;
1348 reqctx
->last_req_len
= bytes
;
1349 reqctx
->partial_req
= !!(req
->cryptlen
- reqctx
->processed
);
1353 chcr_cipher_dma_unmap(&ULD_CTX(c_ctx(tfm
))->lldi
.pdev
->dev
, req
);
1358 static int chcr_aes_encrypt(struct skcipher_request
*req
)
1360 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
1361 struct chcr_skcipher_req_ctx
*reqctx
= skcipher_request_ctx(req
);
1362 struct chcr_dev
*dev
= c_ctx(tfm
)->dev
;
1363 struct sk_buff
*skb
= NULL
;
1365 struct uld_ctx
*u_ctx
= ULD_CTX(c_ctx(tfm
));
1366 struct chcr_context
*ctx
= c_ctx(tfm
);
1370 reqctx
->txqidx
= cpu
% ctx
->ntxq
;
1371 reqctx
->rxqidx
= cpu
% ctx
->nrxq
;
1374 err
= chcr_inc_wrcount(dev
);
1377 if (unlikely(cxgb4_is_crypto_q_full(u_ctx
->lldi
.ports
[0],
1379 (!(req
->base
.flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
)))) {
1384 err
= process_cipher(req
, u_ctx
->lldi
.rxq_ids
[reqctx
->rxqidx
],
1385 &skb
, CHCR_ENCRYPT_OP
);
1388 skb
->dev
= u_ctx
->lldi
.ports
[0];
1389 set_wr_txq(skb
, CPL_PRIORITY_DATA
, reqctx
->txqidx
);
1391 if (get_cryptoalg_subtype(tfm
) ==
1392 CRYPTO_ALG_SUB_TYPE_CBC
&& req
->base
.flags
==
1393 CRYPTO_TFM_REQ_MAY_SLEEP
) {
1394 reqctx
->partial_req
= 1;
1395 wait_for_completion(&ctx
->cbc_aes_aio_done
);
1397 return -EINPROGRESS
;
1399 chcr_dec_wrcount(dev
);
1403 static int chcr_aes_decrypt(struct skcipher_request
*req
)
1405 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
1406 struct chcr_skcipher_req_ctx
*reqctx
= skcipher_request_ctx(req
);
1407 struct uld_ctx
*u_ctx
= ULD_CTX(c_ctx(tfm
));
1408 struct chcr_dev
*dev
= c_ctx(tfm
)->dev
;
1409 struct sk_buff
*skb
= NULL
;
1411 struct chcr_context
*ctx
= c_ctx(tfm
);
1415 reqctx
->txqidx
= cpu
% ctx
->ntxq
;
1416 reqctx
->rxqidx
= cpu
% ctx
->nrxq
;
1419 err
= chcr_inc_wrcount(dev
);
1423 if (unlikely(cxgb4_is_crypto_q_full(u_ctx
->lldi
.ports
[0],
1425 (!(req
->base
.flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
))))
1427 err
= process_cipher(req
, u_ctx
->lldi
.rxq_ids
[reqctx
->rxqidx
],
1428 &skb
, CHCR_DECRYPT_OP
);
1431 skb
->dev
= u_ctx
->lldi
.ports
[0];
1432 set_wr_txq(skb
, CPL_PRIORITY_DATA
, reqctx
->txqidx
);
1434 return -EINPROGRESS
;
1436 static int chcr_device_init(struct chcr_context
*ctx
)
1438 struct uld_ctx
*u_ctx
= NULL
;
1439 int txq_perchan
, ntxq
;
1440 int err
= 0, rxq_perchan
;
1443 u_ctx
= assign_chcr_device();
1445 pr_err("chcr device assignment fails\n");
1448 ctx
->dev
= &u_ctx
->dev
;
1449 ntxq
= u_ctx
->lldi
.ntxq
;
1450 rxq_perchan
= u_ctx
->lldi
.nrxq
/ u_ctx
->lldi
.nchan
;
1451 txq_perchan
= ntxq
/ u_ctx
->lldi
.nchan
;
1453 ctx
->nrxq
= u_ctx
->lldi
.nrxq
;
1454 ctx
->rxq_perchan
= rxq_perchan
;
1455 ctx
->txq_perchan
= txq_perchan
;
1461 static int chcr_init_tfm(struct crypto_skcipher
*tfm
)
1463 struct skcipher_alg
*alg
= crypto_skcipher_alg(tfm
);
1464 struct chcr_context
*ctx
= crypto_skcipher_ctx(tfm
);
1465 struct ablk_ctx
*ablkctx
= ABLK_CTX(ctx
);
1467 ablkctx
->sw_cipher
= crypto_alloc_sync_skcipher(alg
->base
.cra_name
, 0,
1468 CRYPTO_ALG_NEED_FALLBACK
);
1469 if (IS_ERR(ablkctx
->sw_cipher
)) {
1470 pr_err("failed to allocate fallback for %s\n", alg
->base
.cra_name
);
1471 return PTR_ERR(ablkctx
->sw_cipher
);
1473 init_completion(&ctx
->cbc_aes_aio_done
);
1474 crypto_skcipher_set_reqsize(tfm
, sizeof(struct chcr_skcipher_req_ctx
));
1476 return chcr_device_init(ctx
);
1479 static int chcr_rfc3686_init(struct crypto_skcipher
*tfm
)
1481 struct skcipher_alg
*alg
= crypto_skcipher_alg(tfm
);
1482 struct chcr_context
*ctx
= crypto_skcipher_ctx(tfm
);
1483 struct ablk_ctx
*ablkctx
= ABLK_CTX(ctx
);
1485 /*RFC3686 initialises IV counter value to 1, rfc3686(ctr(aes))
1486 * cannot be used as fallback in chcr_handle_cipher_response
1488 ablkctx
->sw_cipher
= crypto_alloc_sync_skcipher("ctr(aes)", 0,
1489 CRYPTO_ALG_NEED_FALLBACK
);
1490 if (IS_ERR(ablkctx
->sw_cipher
)) {
1491 pr_err("failed to allocate fallback for %s\n", alg
->base
.cra_name
);
1492 return PTR_ERR(ablkctx
->sw_cipher
);
1494 crypto_skcipher_set_reqsize(tfm
, sizeof(struct chcr_skcipher_req_ctx
));
1495 return chcr_device_init(ctx
);
1499 static void chcr_exit_tfm(struct crypto_skcipher
*tfm
)
1501 struct chcr_context
*ctx
= crypto_skcipher_ctx(tfm
);
1502 struct ablk_ctx
*ablkctx
= ABLK_CTX(ctx
);
1504 crypto_free_sync_skcipher(ablkctx
->sw_cipher
);
1507 static int get_alg_config(struct algo_param
*params
,
1508 unsigned int auth_size
)
1510 switch (auth_size
) {
1511 case SHA1_DIGEST_SIZE
:
1512 params
->mk_size
= CHCR_KEYCTX_MAC_KEY_SIZE_160
;
1513 params
->auth_mode
= CHCR_SCMD_AUTH_MODE_SHA1
;
1514 params
->result_size
= SHA1_DIGEST_SIZE
;
1516 case SHA224_DIGEST_SIZE
:
1517 params
->mk_size
= CHCR_KEYCTX_MAC_KEY_SIZE_256
;
1518 params
->auth_mode
= CHCR_SCMD_AUTH_MODE_SHA224
;
1519 params
->result_size
= SHA256_DIGEST_SIZE
;
1521 case SHA256_DIGEST_SIZE
:
1522 params
->mk_size
= CHCR_KEYCTX_MAC_KEY_SIZE_256
;
1523 params
->auth_mode
= CHCR_SCMD_AUTH_MODE_SHA256
;
1524 params
->result_size
= SHA256_DIGEST_SIZE
;
1526 case SHA384_DIGEST_SIZE
:
1527 params
->mk_size
= CHCR_KEYCTX_MAC_KEY_SIZE_512
;
1528 params
->auth_mode
= CHCR_SCMD_AUTH_MODE_SHA512_384
;
1529 params
->result_size
= SHA512_DIGEST_SIZE
;
1531 case SHA512_DIGEST_SIZE
:
1532 params
->mk_size
= CHCR_KEYCTX_MAC_KEY_SIZE_512
;
1533 params
->auth_mode
= CHCR_SCMD_AUTH_MODE_SHA512_512
;
1534 params
->result_size
= SHA512_DIGEST_SIZE
;
1537 pr_err("chcr : ERROR, unsupported digest size\n");
1543 static inline void chcr_free_shash(struct crypto_shash
*base_hash
)
1545 crypto_free_shash(base_hash
);
1549 * create_hash_wr - Create hash work request
1550 * @req - Cipher req base
1552 static struct sk_buff
*create_hash_wr(struct ahash_request
*req
,
1553 struct hash_wr_param
*param
)
1555 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(req
);
1556 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
1557 struct chcr_context
*ctx
= h_ctx(tfm
);
1558 struct hmac_ctx
*hmacctx
= HMAC_CTX(ctx
);
1559 struct sk_buff
*skb
= NULL
;
1560 struct uld_ctx
*u_ctx
= ULD_CTX(ctx
);
1561 struct chcr_wr
*chcr_req
;
1562 struct ulptx_sgl
*ulptx
;
1563 unsigned int nents
= 0, transhdr_len
;
1564 unsigned int temp
= 0;
1565 gfp_t flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
? GFP_KERNEL
:
1567 struct adapter
*adap
= padap(h_ctx(tfm
)->dev
);
1569 unsigned int rx_channel_id
= req_ctx
->rxqidx
/ ctx
->rxq_perchan
;
1571 transhdr_len
= HASH_TRANSHDR_SIZE(param
->kctx_len
);
1572 req_ctx
->hctx_wr
.imm
= (transhdr_len
+ param
->bfr_len
+
1573 param
->sg_len
) <= SGE_MAX_WR_LEN
;
1574 nents
= sg_nents_xlen(req_ctx
->hctx_wr
.srcsg
, param
->sg_len
,
1575 CHCR_SRC_SG_SIZE
, req_ctx
->hctx_wr
.src_ofst
);
1576 nents
+= param
->bfr_len
? 1 : 0;
1577 transhdr_len
+= req_ctx
->hctx_wr
.imm
? roundup(param
->bfr_len
+
1578 param
->sg_len
, 16) : (sgl_len(nents
) * 8);
1579 transhdr_len
= roundup(transhdr_len
, 16);
1581 skb
= alloc_skb(transhdr_len
, flags
);
1583 return ERR_PTR(-ENOMEM
);
1584 chcr_req
= __skb_put_zero(skb
, transhdr_len
);
1586 chcr_req
->sec_cpl
.op_ivinsrtofst
=
1587 FILL_SEC_CPL_OP_IVINSR(rx_channel_id
, 2, 0);
1589 chcr_req
->sec_cpl
.pldlen
= htonl(param
->bfr_len
+ param
->sg_len
);
1591 chcr_req
->sec_cpl
.aadstart_cipherstop_hi
=
1592 FILL_SEC_CPL_CIPHERSTOP_HI(0, 0, 0, 0);
1593 chcr_req
->sec_cpl
.cipherstop_lo_authinsert
=
1594 FILL_SEC_CPL_AUTHINSERT(0, 1, 0, 0);
1595 chcr_req
->sec_cpl
.seqno_numivs
=
1596 FILL_SEC_CPL_SCMD0_SEQNO(0, 0, 0, param
->alg_prm
.auth_mode
,
1597 param
->opad_needed
, 0);
1599 chcr_req
->sec_cpl
.ivgen_hdrlen
=
1600 FILL_SEC_CPL_IVGEN_HDRLEN(param
->last
, param
->more
, 0, 1, 0, 0);
1602 memcpy(chcr_req
->key_ctx
.key
, req_ctx
->partial_hash
,
1603 param
->alg_prm
.result_size
);
1605 if (param
->opad_needed
)
1606 memcpy(chcr_req
->key_ctx
.key
+
1607 ((param
->alg_prm
.result_size
<= 32) ? 32 :
1608 CHCR_HASH_MAX_DIGEST_SIZE
),
1609 hmacctx
->opad
, param
->alg_prm
.result_size
);
1611 chcr_req
->key_ctx
.ctx_hdr
= FILL_KEY_CTX_HDR(CHCR_KEYCTX_NO_KEY
,
1612 param
->alg_prm
.mk_size
, 0,
1615 sizeof(chcr_req
->key_ctx
)) >> 4));
1616 chcr_req
->sec_cpl
.scmd1
= cpu_to_be64((u64
)param
->scmd1
);
1617 ulptx
= (struct ulptx_sgl
*)((u8
*)(chcr_req
+ 1) + param
->kctx_len
+
1619 if (param
->bfr_len
!= 0) {
1620 req_ctx
->hctx_wr
.dma_addr
=
1621 dma_map_single(&u_ctx
->lldi
.pdev
->dev
, req_ctx
->reqbfr
,
1622 param
->bfr_len
, DMA_TO_DEVICE
);
1623 if (dma_mapping_error(&u_ctx
->lldi
.pdev
->dev
,
1624 req_ctx
->hctx_wr
. dma_addr
)) {
1628 req_ctx
->hctx_wr
.dma_len
= param
->bfr_len
;
1630 req_ctx
->hctx_wr
.dma_addr
= 0;
1632 chcr_add_hash_src_ent(req
, ulptx
, param
);
1633 /* Request upto max wr size */
1634 temp
= param
->kctx_len
+ DUMMY_BYTES
+ (req_ctx
->hctx_wr
.imm
?
1635 (param
->sg_len
+ param
->bfr_len
) : 0);
1636 atomic_inc(&adap
->chcr_stats
.digest_rqst
);
1637 create_wreq(h_ctx(tfm
), chcr_req
, &req
->base
, req_ctx
->hctx_wr
.imm
,
1638 param
->hash_size
, transhdr_len
,
1640 req_ctx
->hctx_wr
.skb
= skb
;
1644 return ERR_PTR(error
);
1647 static int chcr_ahash_update(struct ahash_request
*req
)
1649 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(req
);
1650 struct crypto_ahash
*rtfm
= crypto_ahash_reqtfm(req
);
1651 struct uld_ctx
*u_ctx
= ULD_CTX(h_ctx(rtfm
));
1652 struct chcr_context
*ctx
= h_ctx(rtfm
);
1653 struct chcr_dev
*dev
= h_ctx(rtfm
)->dev
;
1654 struct sk_buff
*skb
;
1655 u8 remainder
= 0, bs
;
1656 unsigned int nbytes
= req
->nbytes
;
1657 struct hash_wr_param params
;
1662 req_ctx
->txqidx
= cpu
% ctx
->ntxq
;
1663 req_ctx
->rxqidx
= cpu
% ctx
->nrxq
;
1666 bs
= crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm
));
1668 if (nbytes
+ req_ctx
->reqlen
>= bs
) {
1669 remainder
= (nbytes
+ req_ctx
->reqlen
) % bs
;
1670 nbytes
= nbytes
+ req_ctx
->reqlen
- remainder
;
1672 sg_pcopy_to_buffer(req
->src
, sg_nents(req
->src
), req_ctx
->reqbfr
1673 + req_ctx
->reqlen
, nbytes
, 0);
1674 req_ctx
->reqlen
+= nbytes
;
1677 error
= chcr_inc_wrcount(dev
);
1680 /* Detach state for CHCR means lldi or padap is freed. Increasing
1681 * inflight count for dev guarantees that lldi and padap is valid
1683 if (unlikely(cxgb4_is_crypto_q_full(u_ctx
->lldi
.ports
[0],
1685 (!(req
->base
.flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
)))) {
1690 chcr_init_hctx_per_wr(req_ctx
);
1691 error
= chcr_hash_dma_map(&u_ctx
->lldi
.pdev
->dev
, req
);
1696 get_alg_config(¶ms
.alg_prm
, crypto_ahash_digestsize(rtfm
));
1697 params
.kctx_len
= roundup(params
.alg_prm
.result_size
, 16);
1698 params
.sg_len
= chcr_hash_ent_in_wr(req
->src
, !!req_ctx
->reqlen
,
1699 HASH_SPACE_LEFT(params
.kctx_len
), 0);
1700 if (params
.sg_len
> req
->nbytes
)
1701 params
.sg_len
= req
->nbytes
;
1702 params
.sg_len
= rounddown(params
.sg_len
+ req_ctx
->reqlen
, bs
) -
1704 params
.opad_needed
= 0;
1707 params
.bfr_len
= req_ctx
->reqlen
;
1709 req_ctx
->hctx_wr
.srcsg
= req
->src
;
1711 params
.hash_size
= params
.alg_prm
.result_size
;
1712 req_ctx
->data_len
+= params
.sg_len
+ params
.bfr_len
;
1713 skb
= create_hash_wr(req
, ¶ms
);
1715 error
= PTR_ERR(skb
);
1719 req_ctx
->hctx_wr
.processed
+= params
.sg_len
;
1722 swap(req_ctx
->reqbfr
, req_ctx
->skbfr
);
1723 sg_pcopy_to_buffer(req
->src
, sg_nents(req
->src
),
1724 req_ctx
->reqbfr
, remainder
, req
->nbytes
-
1727 req_ctx
->reqlen
= remainder
;
1728 skb
->dev
= u_ctx
->lldi
.ports
[0];
1729 set_wr_txq(skb
, CPL_PRIORITY_DATA
, req_ctx
->txqidx
);
1731 return -EINPROGRESS
;
1733 chcr_hash_dma_unmap(&u_ctx
->lldi
.pdev
->dev
, req
);
1735 chcr_dec_wrcount(dev
);
1739 static void create_last_hash_block(char *bfr_ptr
, unsigned int bs
, u64 scmd1
)
1741 memset(bfr_ptr
, 0, bs
);
1744 *(__be64
*)(bfr_ptr
+ 56) = cpu_to_be64(scmd1
<< 3);
1746 *(__be64
*)(bfr_ptr
+ 120) = cpu_to_be64(scmd1
<< 3);
1749 static int chcr_ahash_final(struct ahash_request
*req
)
1751 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(req
);
1752 struct crypto_ahash
*rtfm
= crypto_ahash_reqtfm(req
);
1753 struct chcr_dev
*dev
= h_ctx(rtfm
)->dev
;
1754 struct hash_wr_param params
;
1755 struct sk_buff
*skb
;
1756 struct uld_ctx
*u_ctx
= ULD_CTX(h_ctx(rtfm
));
1757 struct chcr_context
*ctx
= h_ctx(rtfm
);
1758 u8 bs
= crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm
));
1763 req_ctx
->txqidx
= cpu
% ctx
->ntxq
;
1764 req_ctx
->rxqidx
= cpu
% ctx
->nrxq
;
1767 error
= chcr_inc_wrcount(dev
);
1771 chcr_init_hctx_per_wr(req_ctx
);
1772 if (is_hmac(crypto_ahash_tfm(rtfm
)))
1773 params
.opad_needed
= 1;
1775 params
.opad_needed
= 0;
1777 req_ctx
->hctx_wr
.isfinal
= 1;
1778 get_alg_config(¶ms
.alg_prm
, crypto_ahash_digestsize(rtfm
));
1779 params
.kctx_len
= roundup(params
.alg_prm
.result_size
, 16);
1780 if (is_hmac(crypto_ahash_tfm(rtfm
))) {
1781 params
.opad_needed
= 1;
1782 params
.kctx_len
*= 2;
1784 params
.opad_needed
= 0;
1787 req_ctx
->hctx_wr
.result
= 1;
1788 params
.bfr_len
= req_ctx
->reqlen
;
1789 req_ctx
->data_len
+= params
.bfr_len
+ params
.sg_len
;
1790 req_ctx
->hctx_wr
.srcsg
= req
->src
;
1791 if (req_ctx
->reqlen
== 0) {
1792 create_last_hash_block(req_ctx
->reqbfr
, bs
, req_ctx
->data_len
);
1796 params
.bfr_len
= bs
;
1799 params
.scmd1
= req_ctx
->data_len
;
1803 params
.hash_size
= crypto_ahash_digestsize(rtfm
);
1804 skb
= create_hash_wr(req
, ¶ms
);
1806 error
= PTR_ERR(skb
);
1809 req_ctx
->reqlen
= 0;
1810 skb
->dev
= u_ctx
->lldi
.ports
[0];
1811 set_wr_txq(skb
, CPL_PRIORITY_DATA
, req_ctx
->txqidx
);
1813 return -EINPROGRESS
;
1815 chcr_dec_wrcount(dev
);
1819 static int chcr_ahash_finup(struct ahash_request
*req
)
1821 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(req
);
1822 struct crypto_ahash
*rtfm
= crypto_ahash_reqtfm(req
);
1823 struct chcr_dev
*dev
= h_ctx(rtfm
)->dev
;
1824 struct uld_ctx
*u_ctx
= ULD_CTX(h_ctx(rtfm
));
1825 struct chcr_context
*ctx
= h_ctx(rtfm
);
1826 struct sk_buff
*skb
;
1827 struct hash_wr_param params
;
1833 req_ctx
->txqidx
= cpu
% ctx
->ntxq
;
1834 req_ctx
->rxqidx
= cpu
% ctx
->nrxq
;
1837 bs
= crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm
));
1838 error
= chcr_inc_wrcount(dev
);
1842 if (unlikely(cxgb4_is_crypto_q_full(u_ctx
->lldi
.ports
[0],
1844 (!(req
->base
.flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
)))) {
1848 chcr_init_hctx_per_wr(req_ctx
);
1849 error
= chcr_hash_dma_map(&u_ctx
->lldi
.pdev
->dev
, req
);
1855 get_alg_config(¶ms
.alg_prm
, crypto_ahash_digestsize(rtfm
));
1856 params
.kctx_len
= roundup(params
.alg_prm
.result_size
, 16);
1857 if (is_hmac(crypto_ahash_tfm(rtfm
))) {
1858 params
.kctx_len
*= 2;
1859 params
.opad_needed
= 1;
1861 params
.opad_needed
= 0;
1864 params
.sg_len
= chcr_hash_ent_in_wr(req
->src
, !!req_ctx
->reqlen
,
1865 HASH_SPACE_LEFT(params
.kctx_len
), 0);
1866 if (params
.sg_len
< req
->nbytes
) {
1867 if (is_hmac(crypto_ahash_tfm(rtfm
))) {
1868 params
.kctx_len
/= 2;
1869 params
.opad_needed
= 0;
1873 params
.sg_len
= rounddown(params
.sg_len
+ req_ctx
->reqlen
, bs
)
1875 params
.hash_size
= params
.alg_prm
.result_size
;
1880 params
.sg_len
= req
->nbytes
;
1881 params
.hash_size
= crypto_ahash_digestsize(rtfm
);
1882 params
.scmd1
= req_ctx
->data_len
+ req_ctx
->reqlen
+
1885 params
.bfr_len
= req_ctx
->reqlen
;
1886 req_ctx
->data_len
+= params
.bfr_len
+ params
.sg_len
;
1887 req_ctx
->hctx_wr
.result
= 1;
1888 req_ctx
->hctx_wr
.srcsg
= req
->src
;
1889 if ((req_ctx
->reqlen
+ req
->nbytes
) == 0) {
1890 create_last_hash_block(req_ctx
->reqbfr
, bs
, req_ctx
->data_len
);
1894 params
.bfr_len
= bs
;
1896 skb
= create_hash_wr(req
, ¶ms
);
1898 error
= PTR_ERR(skb
);
1901 req_ctx
->reqlen
= 0;
1902 req_ctx
->hctx_wr
.processed
+= params
.sg_len
;
1903 skb
->dev
= u_ctx
->lldi
.ports
[0];
1904 set_wr_txq(skb
, CPL_PRIORITY_DATA
, req_ctx
->txqidx
);
1906 return -EINPROGRESS
;
1908 chcr_hash_dma_unmap(&u_ctx
->lldi
.pdev
->dev
, req
);
1910 chcr_dec_wrcount(dev
);
1914 static int chcr_ahash_digest(struct ahash_request
*req
)
1916 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(req
);
1917 struct crypto_ahash
*rtfm
= crypto_ahash_reqtfm(req
);
1918 struct chcr_dev
*dev
= h_ctx(rtfm
)->dev
;
1919 struct uld_ctx
*u_ctx
= ULD_CTX(h_ctx(rtfm
));
1920 struct chcr_context
*ctx
= h_ctx(rtfm
);
1921 struct sk_buff
*skb
;
1922 struct hash_wr_param params
;
1928 req_ctx
->txqidx
= cpu
% ctx
->ntxq
;
1929 req_ctx
->rxqidx
= cpu
% ctx
->nrxq
;
1933 bs
= crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm
));
1934 error
= chcr_inc_wrcount(dev
);
1938 if (unlikely(cxgb4_is_crypto_q_full(u_ctx
->lldi
.ports
[0],
1940 (!(req
->base
.flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
)))) {
1945 chcr_init_hctx_per_wr(req_ctx
);
1946 error
= chcr_hash_dma_map(&u_ctx
->lldi
.pdev
->dev
, req
);
1952 get_alg_config(¶ms
.alg_prm
, crypto_ahash_digestsize(rtfm
));
1953 params
.kctx_len
= roundup(params
.alg_prm
.result_size
, 16);
1954 if (is_hmac(crypto_ahash_tfm(rtfm
))) {
1955 params
.kctx_len
*= 2;
1956 params
.opad_needed
= 1;
1958 params
.opad_needed
= 0;
1960 params
.sg_len
= chcr_hash_ent_in_wr(req
->src
, !!req_ctx
->reqlen
,
1961 HASH_SPACE_LEFT(params
.kctx_len
), 0);
1962 if (params
.sg_len
< req
->nbytes
) {
1963 if (is_hmac(crypto_ahash_tfm(rtfm
))) {
1964 params
.kctx_len
/= 2;
1965 params
.opad_needed
= 0;
1970 params
.sg_len
= rounddown(params
.sg_len
, bs
);
1971 params
.hash_size
= params
.alg_prm
.result_size
;
1973 params
.sg_len
= req
->nbytes
;
1974 params
.hash_size
= crypto_ahash_digestsize(rtfm
);
1977 params
.scmd1
= req
->nbytes
+ req_ctx
->data_len
;
1981 req_ctx
->hctx_wr
.result
= 1;
1982 req_ctx
->hctx_wr
.srcsg
= req
->src
;
1983 req_ctx
->data_len
+= params
.bfr_len
+ params
.sg_len
;
1985 if (req
->nbytes
== 0) {
1986 create_last_hash_block(req_ctx
->reqbfr
, bs
, 0);
1988 params
.bfr_len
= bs
;
1991 skb
= create_hash_wr(req
, ¶ms
);
1993 error
= PTR_ERR(skb
);
1996 req_ctx
->hctx_wr
.processed
+= params
.sg_len
;
1997 skb
->dev
= u_ctx
->lldi
.ports
[0];
1998 set_wr_txq(skb
, CPL_PRIORITY_DATA
, req_ctx
->txqidx
);
2000 return -EINPROGRESS
;
2002 chcr_hash_dma_unmap(&u_ctx
->lldi
.pdev
->dev
, req
);
2004 chcr_dec_wrcount(dev
);
2008 static int chcr_ahash_continue(struct ahash_request
*req
)
2010 struct chcr_ahash_req_ctx
*reqctx
= ahash_request_ctx(req
);
2011 struct chcr_hctx_per_wr
*hctx_wr
= &reqctx
->hctx_wr
;
2012 struct crypto_ahash
*rtfm
= crypto_ahash_reqtfm(req
);
2013 struct chcr_context
*ctx
= h_ctx(rtfm
);
2014 struct uld_ctx
*u_ctx
= ULD_CTX(ctx
);
2015 struct sk_buff
*skb
;
2016 struct hash_wr_param params
;
2022 reqctx
->txqidx
= cpu
% ctx
->ntxq
;
2023 reqctx
->rxqidx
= cpu
% ctx
->nrxq
;
2026 bs
= crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm
));
2027 get_alg_config(¶ms
.alg_prm
, crypto_ahash_digestsize(rtfm
));
2028 params
.kctx_len
= roundup(params
.alg_prm
.result_size
, 16);
2029 if (is_hmac(crypto_ahash_tfm(rtfm
))) {
2030 params
.kctx_len
*= 2;
2031 params
.opad_needed
= 1;
2033 params
.opad_needed
= 0;
2035 params
.sg_len
= chcr_hash_ent_in_wr(hctx_wr
->srcsg
, 0,
2036 HASH_SPACE_LEFT(params
.kctx_len
),
2038 if ((params
.sg_len
+ hctx_wr
->processed
) > req
->nbytes
)
2039 params
.sg_len
= req
->nbytes
- hctx_wr
->processed
;
2040 if (!hctx_wr
->result
||
2041 ((params
.sg_len
+ hctx_wr
->processed
) < req
->nbytes
)) {
2042 if (is_hmac(crypto_ahash_tfm(rtfm
))) {
2043 params
.kctx_len
/= 2;
2044 params
.opad_needed
= 0;
2048 params
.sg_len
= rounddown(params
.sg_len
, bs
);
2049 params
.hash_size
= params
.alg_prm
.result_size
;
2054 params
.hash_size
= crypto_ahash_digestsize(rtfm
);
2055 params
.scmd1
= reqctx
->data_len
+ params
.sg_len
;
2058 reqctx
->data_len
+= params
.sg_len
;
2059 skb
= create_hash_wr(req
, ¶ms
);
2061 error
= PTR_ERR(skb
);
2064 hctx_wr
->processed
+= params
.sg_len
;
2065 skb
->dev
= u_ctx
->lldi
.ports
[0];
2066 set_wr_txq(skb
, CPL_PRIORITY_DATA
, reqctx
->txqidx
);
2073 static inline void chcr_handle_ahash_resp(struct ahash_request
*req
,
2074 unsigned char *input
,
2077 struct chcr_ahash_req_ctx
*reqctx
= ahash_request_ctx(req
);
2078 struct chcr_hctx_per_wr
*hctx_wr
= &reqctx
->hctx_wr
;
2079 int digestsize
, updated_digestsize
;
2080 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
2081 struct uld_ctx
*u_ctx
= ULD_CTX(h_ctx(tfm
));
2082 struct chcr_dev
*dev
= h_ctx(tfm
)->dev
;
2086 digestsize
= crypto_ahash_digestsize(crypto_ahash_reqtfm(req
));
2087 updated_digestsize
= digestsize
;
2088 if (digestsize
== SHA224_DIGEST_SIZE
)
2089 updated_digestsize
= SHA256_DIGEST_SIZE
;
2090 else if (digestsize
== SHA384_DIGEST_SIZE
)
2091 updated_digestsize
= SHA512_DIGEST_SIZE
;
2093 if (hctx_wr
->dma_addr
) {
2094 dma_unmap_single(&u_ctx
->lldi
.pdev
->dev
, hctx_wr
->dma_addr
,
2095 hctx_wr
->dma_len
, DMA_TO_DEVICE
);
2096 hctx_wr
->dma_addr
= 0;
2098 if (hctx_wr
->isfinal
|| ((hctx_wr
->processed
+ reqctx
->reqlen
) ==
2100 if (hctx_wr
->result
== 1) {
2101 hctx_wr
->result
= 0;
2102 memcpy(req
->result
, input
+ sizeof(struct cpl_fw6_pld
),
2105 memcpy(reqctx
->partial_hash
,
2106 input
+ sizeof(struct cpl_fw6_pld
),
2107 updated_digestsize
);
2112 memcpy(reqctx
->partial_hash
, input
+ sizeof(struct cpl_fw6_pld
),
2113 updated_digestsize
);
2115 err
= chcr_ahash_continue(req
);
2120 if (hctx_wr
->is_sg_map
)
2121 chcr_hash_dma_unmap(&u_ctx
->lldi
.pdev
->dev
, req
);
2125 chcr_dec_wrcount(dev
);
2126 req
->base
.complete(&req
->base
, err
);
2130 * chcr_handle_resp - Unmap the DMA buffers associated with the request
2131 * @req: crypto request
2133 int chcr_handle_resp(struct crypto_async_request
*req
, unsigned char *input
,
2136 struct crypto_tfm
*tfm
= req
->tfm
;
2137 struct chcr_context
*ctx
= crypto_tfm_ctx(tfm
);
2138 struct adapter
*adap
= padap(ctx
->dev
);
2140 switch (tfm
->__crt_alg
->cra_flags
& CRYPTO_ALG_TYPE_MASK
) {
2141 case CRYPTO_ALG_TYPE_AEAD
:
2142 err
= chcr_handle_aead_resp(aead_request_cast(req
), input
, err
);
2145 case CRYPTO_ALG_TYPE_SKCIPHER
:
2146 chcr_handle_cipher_resp(skcipher_request_cast(req
),
2149 case CRYPTO_ALG_TYPE_AHASH
:
2150 chcr_handle_ahash_resp(ahash_request_cast(req
), input
, err
);
2152 atomic_inc(&adap
->chcr_stats
.complete
);
2155 static int chcr_ahash_export(struct ahash_request
*areq
, void *out
)
2157 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
2158 struct chcr_ahash_req_ctx
*state
= out
;
2160 state
->reqlen
= req_ctx
->reqlen
;
2161 state
->data_len
= req_ctx
->data_len
;
2162 memcpy(state
->bfr1
, req_ctx
->reqbfr
, req_ctx
->reqlen
);
2163 memcpy(state
->partial_hash
, req_ctx
->partial_hash
,
2164 CHCR_HASH_MAX_DIGEST_SIZE
);
2165 chcr_init_hctx_per_wr(state
);
2169 static int chcr_ahash_import(struct ahash_request
*areq
, const void *in
)
2171 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
2172 struct chcr_ahash_req_ctx
*state
= (struct chcr_ahash_req_ctx
*)in
;
2174 req_ctx
->reqlen
= state
->reqlen
;
2175 req_ctx
->data_len
= state
->data_len
;
2176 req_ctx
->reqbfr
= req_ctx
->bfr1
;
2177 req_ctx
->skbfr
= req_ctx
->bfr2
;
2178 memcpy(req_ctx
->bfr1
, state
->bfr1
, CHCR_HASH_MAX_BLOCK_SIZE_128
);
2179 memcpy(req_ctx
->partial_hash
, state
->partial_hash
,
2180 CHCR_HASH_MAX_DIGEST_SIZE
);
2181 chcr_init_hctx_per_wr(req_ctx
);
2185 static int chcr_ahash_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
2186 unsigned int keylen
)
2188 struct hmac_ctx
*hmacctx
= HMAC_CTX(h_ctx(tfm
));
2189 unsigned int digestsize
= crypto_ahash_digestsize(tfm
);
2190 unsigned int bs
= crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
2191 unsigned int i
, err
= 0, updated_digestsize
;
2193 SHASH_DESC_ON_STACK(shash
, hmacctx
->base_hash
);
2195 /* use the key to calculate the ipad and opad. ipad will sent with the
2196 * first request's data. opad will be sent with the final hash result
2197 * ipad in hmacctx->ipad and opad in hmacctx->opad location
2199 shash
->tfm
= hmacctx
->base_hash
;
2201 err
= crypto_shash_digest(shash
, key
, keylen
,
2205 keylen
= digestsize
;
2207 memcpy(hmacctx
->ipad
, key
, keylen
);
2209 memset(hmacctx
->ipad
+ keylen
, 0, bs
- keylen
);
2210 memcpy(hmacctx
->opad
, hmacctx
->ipad
, bs
);
2212 for (i
= 0; i
< bs
/ sizeof(int); i
++) {
2213 *((unsigned int *)(&hmacctx
->ipad
) + i
) ^= IPAD_DATA
;
2214 *((unsigned int *)(&hmacctx
->opad
) + i
) ^= OPAD_DATA
;
2217 updated_digestsize
= digestsize
;
2218 if (digestsize
== SHA224_DIGEST_SIZE
)
2219 updated_digestsize
= SHA256_DIGEST_SIZE
;
2220 else if (digestsize
== SHA384_DIGEST_SIZE
)
2221 updated_digestsize
= SHA512_DIGEST_SIZE
;
2222 err
= chcr_compute_partial_hash(shash
, hmacctx
->ipad
,
2223 hmacctx
->ipad
, digestsize
);
2226 chcr_change_order(hmacctx
->ipad
, updated_digestsize
);
2228 err
= chcr_compute_partial_hash(shash
, hmacctx
->opad
,
2229 hmacctx
->opad
, digestsize
);
2232 chcr_change_order(hmacctx
->opad
, updated_digestsize
);
2237 static int chcr_aes_xts_setkey(struct crypto_skcipher
*cipher
, const u8
*key
,
2238 unsigned int key_len
)
2240 struct ablk_ctx
*ablkctx
= ABLK_CTX(c_ctx(cipher
));
2241 unsigned short context_size
= 0;
2244 err
= chcr_cipher_fallback_setkey(cipher
, key
, key_len
);
2248 memcpy(ablkctx
->key
, key
, key_len
);
2249 ablkctx
->enckey_len
= key_len
;
2250 get_aes_decrypt_key(ablkctx
->rrkey
, ablkctx
->key
, key_len
<< 2);
2251 context_size
= (KEY_CONTEXT_HDR_SALT_AND_PAD
+ key_len
) >> 4;
2252 ablkctx
->key_ctx_hdr
=
2253 FILL_KEY_CTX_HDR((key_len
== AES_KEYSIZE_256
) ?
2254 CHCR_KEYCTX_CIPHER_KEY_SIZE_128
:
2255 CHCR_KEYCTX_CIPHER_KEY_SIZE_256
,
2256 CHCR_KEYCTX_NO_KEY
, 1,
2258 ablkctx
->ciph_mode
= CHCR_SCMD_CIPHER_MODE_AES_XTS
;
2261 ablkctx
->enckey_len
= 0;
2266 static int chcr_sha_init(struct ahash_request
*areq
)
2268 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
2269 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
2270 int digestsize
= crypto_ahash_digestsize(tfm
);
2272 req_ctx
->data_len
= 0;
2273 req_ctx
->reqlen
= 0;
2274 req_ctx
->reqbfr
= req_ctx
->bfr1
;
2275 req_ctx
->skbfr
= req_ctx
->bfr2
;
2276 copy_hash_init_values(req_ctx
->partial_hash
, digestsize
);
2281 static int chcr_sha_cra_init(struct crypto_tfm
*tfm
)
2283 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
2284 sizeof(struct chcr_ahash_req_ctx
));
2285 return chcr_device_init(crypto_tfm_ctx(tfm
));
2288 static int chcr_hmac_init(struct ahash_request
*areq
)
2290 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
2291 struct crypto_ahash
*rtfm
= crypto_ahash_reqtfm(areq
);
2292 struct hmac_ctx
*hmacctx
= HMAC_CTX(h_ctx(rtfm
));
2293 unsigned int digestsize
= crypto_ahash_digestsize(rtfm
);
2294 unsigned int bs
= crypto_tfm_alg_blocksize(crypto_ahash_tfm(rtfm
));
2296 chcr_sha_init(areq
);
2297 req_ctx
->data_len
= bs
;
2298 if (is_hmac(crypto_ahash_tfm(rtfm
))) {
2299 if (digestsize
== SHA224_DIGEST_SIZE
)
2300 memcpy(req_ctx
->partial_hash
, hmacctx
->ipad
,
2301 SHA256_DIGEST_SIZE
);
2302 else if (digestsize
== SHA384_DIGEST_SIZE
)
2303 memcpy(req_ctx
->partial_hash
, hmacctx
->ipad
,
2304 SHA512_DIGEST_SIZE
);
2306 memcpy(req_ctx
->partial_hash
, hmacctx
->ipad
,
2312 static int chcr_hmac_cra_init(struct crypto_tfm
*tfm
)
2314 struct chcr_context
*ctx
= crypto_tfm_ctx(tfm
);
2315 struct hmac_ctx
*hmacctx
= HMAC_CTX(ctx
);
2316 unsigned int digestsize
=
2317 crypto_ahash_digestsize(__crypto_ahash_cast(tfm
));
2319 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
2320 sizeof(struct chcr_ahash_req_ctx
));
2321 hmacctx
->base_hash
= chcr_alloc_shash(digestsize
);
2322 if (IS_ERR(hmacctx
->base_hash
))
2323 return PTR_ERR(hmacctx
->base_hash
);
2324 return chcr_device_init(crypto_tfm_ctx(tfm
));
2327 static void chcr_hmac_cra_exit(struct crypto_tfm
*tfm
)
2329 struct chcr_context
*ctx
= crypto_tfm_ctx(tfm
);
2330 struct hmac_ctx
*hmacctx
= HMAC_CTX(ctx
);
2332 if (hmacctx
->base_hash
) {
2333 chcr_free_shash(hmacctx
->base_hash
);
2334 hmacctx
->base_hash
= NULL
;
2338 inline void chcr_aead_common_exit(struct aead_request
*req
)
2340 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2341 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
2342 struct uld_ctx
*u_ctx
= ULD_CTX(a_ctx(tfm
));
2344 chcr_aead_dma_unmap(&u_ctx
->lldi
.pdev
->dev
, req
, reqctx
->op
);
2347 static int chcr_aead_common_init(struct aead_request
*req
)
2349 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
2350 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
2351 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2352 unsigned int authsize
= crypto_aead_authsize(tfm
);
2353 int error
= -EINVAL
;
2355 /* validate key size */
2356 if (aeadctx
->enckey_len
== 0)
2358 if (reqctx
->op
&& req
->cryptlen
< authsize
)
2361 reqctx
->scratch_pad
= reqctx
->iv
+ IV
;
2363 reqctx
->scratch_pad
= NULL
;
2365 error
= chcr_aead_dma_map(&ULD_CTX(a_ctx(tfm
))->lldi
.pdev
->dev
, req
,
2377 static int chcr_aead_need_fallback(struct aead_request
*req
, int dst_nents
,
2378 int aadmax
, int wrlen
,
2379 unsigned short op_type
)
2381 unsigned int authsize
= crypto_aead_authsize(crypto_aead_reqtfm(req
));
2383 if (((req
->cryptlen
- (op_type
? authsize
: 0)) == 0) ||
2384 dst_nents
> MAX_DSGL_ENT
||
2385 (req
->assoclen
> aadmax
) ||
2386 (wrlen
> SGE_MAX_WR_LEN
))
2391 static int chcr_aead_fallback(struct aead_request
*req
, unsigned short op_type
)
2393 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
2394 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
2395 struct aead_request
*subreq
= aead_request_ctx(req
);
2397 aead_request_set_tfm(subreq
, aeadctx
->sw_cipher
);
2398 aead_request_set_callback(subreq
, req
->base
.flags
,
2399 req
->base
.complete
, req
->base
.data
);
2400 aead_request_set_crypt(subreq
, req
->src
, req
->dst
, req
->cryptlen
,
2402 aead_request_set_ad(subreq
, req
->assoclen
);
2403 return op_type
? crypto_aead_decrypt(subreq
) :
2404 crypto_aead_encrypt(subreq
);
2407 static struct sk_buff
*create_authenc_wr(struct aead_request
*req
,
2411 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
2412 struct chcr_context
*ctx
= a_ctx(tfm
);
2413 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(ctx
);
2414 struct chcr_authenc_ctx
*actx
= AUTHENC_CTX(aeadctx
);
2415 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2416 struct sk_buff
*skb
= NULL
;
2417 struct chcr_wr
*chcr_req
;
2418 struct cpl_rx_phys_dsgl
*phys_cpl
;
2419 struct ulptx_sgl
*ulptx
;
2420 unsigned int transhdr_len
;
2421 unsigned int dst_size
= 0, temp
, subtype
= get_aead_subtype(tfm
);
2422 unsigned int kctx_len
= 0, dnents
, snents
;
2423 unsigned int authsize
= crypto_aead_authsize(tfm
);
2424 int error
= -EINVAL
;
2427 gfp_t flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
? GFP_KERNEL
:
2429 struct adapter
*adap
= padap(ctx
->dev
);
2430 unsigned int rx_channel_id
= reqctx
->rxqidx
/ ctx
->rxq_perchan
;
2432 if (req
->cryptlen
== 0)
2436 error
= chcr_aead_common_init(req
);
2438 return ERR_PTR(error
);
2440 if (subtype
== CRYPTO_ALG_SUB_TYPE_CBC_NULL
||
2441 subtype
== CRYPTO_ALG_SUB_TYPE_CTR_NULL
) {
2444 dnents
= sg_nents_xlen(req
->dst
, req
->assoclen
+ req
->cryptlen
+
2445 (reqctx
->op
? -authsize
: authsize
), CHCR_DST_SG_SIZE
, 0);
2446 dnents
+= MIN_AUTH_SG
; // For IV
2447 snents
= sg_nents_xlen(req
->src
, req
->assoclen
+ req
->cryptlen
,
2448 CHCR_SRC_SG_SIZE
, 0);
2449 dst_size
= get_space_for_phys_dsgl(dnents
);
2450 kctx_len
= (KEY_CONTEXT_CTX_LEN_G(ntohl(aeadctx
->key_ctx_hdr
)) << 4)
2451 - sizeof(chcr_req
->key_ctx
);
2452 transhdr_len
= CIPHER_TRANSHDR_SIZE(kctx_len
, dst_size
);
2453 reqctx
->imm
= (transhdr_len
+ req
->assoclen
+ req
->cryptlen
) <
2455 temp
= reqctx
->imm
? roundup(req
->assoclen
+ req
->cryptlen
, 16)
2456 : (sgl_len(snents
) * 8);
2457 transhdr_len
+= temp
;
2458 transhdr_len
= roundup(transhdr_len
, 16);
2460 if (chcr_aead_need_fallback(req
, dnents
, T6_MAX_AAD_SIZE
,
2461 transhdr_len
, reqctx
->op
)) {
2462 atomic_inc(&adap
->chcr_stats
.fallback
);
2463 chcr_aead_common_exit(req
);
2464 return ERR_PTR(chcr_aead_fallback(req
, reqctx
->op
));
2466 skb
= alloc_skb(transhdr_len
, flags
);
2472 chcr_req
= __skb_put_zero(skb
, transhdr_len
);
2474 temp
= (reqctx
->op
== CHCR_ENCRYPT_OP
) ? 0 : authsize
;
2477 * Input order is AAD,IV and Payload. where IV should be included as
2478 * the part of authdata. All other fields should be filled according
2479 * to the hardware spec
2481 chcr_req
->sec_cpl
.op_ivinsrtofst
=
2482 FILL_SEC_CPL_OP_IVINSR(rx_channel_id
, 2, 1);
2483 chcr_req
->sec_cpl
.pldlen
= htonl(req
->assoclen
+ IV
+ req
->cryptlen
);
2484 chcr_req
->sec_cpl
.aadstart_cipherstop_hi
= FILL_SEC_CPL_CIPHERSTOP_HI(
2486 null
? 0 : IV
+ req
->assoclen
,
2487 req
->assoclen
+ IV
+ 1,
2488 (temp
& 0x1F0) >> 4);
2489 chcr_req
->sec_cpl
.cipherstop_lo_authinsert
= FILL_SEC_CPL_AUTHINSERT(
2491 null
? 0 : req
->assoclen
+ IV
+ 1,
2493 if (subtype
== CRYPTO_ALG_SUB_TYPE_CTR_NULL
||
2494 subtype
== CRYPTO_ALG_SUB_TYPE_CTR_SHA
)
2495 temp
= CHCR_SCMD_CIPHER_MODE_AES_CTR
;
2497 temp
= CHCR_SCMD_CIPHER_MODE_AES_CBC
;
2498 chcr_req
->sec_cpl
.seqno_numivs
= FILL_SEC_CPL_SCMD0_SEQNO(reqctx
->op
,
2499 (reqctx
->op
== CHCR_ENCRYPT_OP
) ? 1 : 0,
2501 actx
->auth_mode
, aeadctx
->hmac_ctrl
,
2503 chcr_req
->sec_cpl
.ivgen_hdrlen
= FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
2506 chcr_req
->key_ctx
.ctx_hdr
= aeadctx
->key_ctx_hdr
;
2507 if (reqctx
->op
== CHCR_ENCRYPT_OP
||
2508 subtype
== CRYPTO_ALG_SUB_TYPE_CTR_SHA
||
2509 subtype
== CRYPTO_ALG_SUB_TYPE_CTR_NULL
)
2510 memcpy(chcr_req
->key_ctx
.key
, aeadctx
->key
,
2511 aeadctx
->enckey_len
);
2513 memcpy(chcr_req
->key_ctx
.key
, actx
->dec_rrkey
,
2514 aeadctx
->enckey_len
);
2516 memcpy(chcr_req
->key_ctx
.key
+ roundup(aeadctx
->enckey_len
, 16),
2517 actx
->h_iopad
, kctx_len
- roundup(aeadctx
->enckey_len
, 16));
2518 phys_cpl
= (struct cpl_rx_phys_dsgl
*)((u8
*)(chcr_req
+ 1) + kctx_len
);
2519 ivptr
= (u8
*)(phys_cpl
+ 1) + dst_size
;
2520 ulptx
= (struct ulptx_sgl
*)(ivptr
+ IV
);
2521 if (subtype
== CRYPTO_ALG_SUB_TYPE_CTR_SHA
||
2522 subtype
== CRYPTO_ALG_SUB_TYPE_CTR_NULL
) {
2523 memcpy(ivptr
, aeadctx
->nonce
, CTR_RFC3686_NONCE_SIZE
);
2524 memcpy(ivptr
+ CTR_RFC3686_NONCE_SIZE
, req
->iv
,
2525 CTR_RFC3686_IV_SIZE
);
2526 *(__be32
*)(ivptr
+ CTR_RFC3686_NONCE_SIZE
+
2527 CTR_RFC3686_IV_SIZE
) = cpu_to_be32(1);
2529 memcpy(ivptr
, req
->iv
, IV
);
2531 chcr_add_aead_dst_ent(req
, phys_cpl
, qid
);
2532 chcr_add_aead_src_ent(req
, ulptx
);
2533 atomic_inc(&adap
->chcr_stats
.cipher_rqst
);
2534 temp
= sizeof(struct cpl_rx_phys_dsgl
) + dst_size
+ IV
+
2535 kctx_len
+ (reqctx
->imm
? (req
->assoclen
+ req
->cryptlen
) : 0);
2536 create_wreq(a_ctx(tfm
), chcr_req
, &req
->base
, reqctx
->imm
, size
,
2537 transhdr_len
, temp
, 0);
2542 chcr_aead_common_exit(req
);
2544 return ERR_PTR(error
);
2547 int chcr_aead_dma_map(struct device
*dev
,
2548 struct aead_request
*req
,
2549 unsigned short op_type
)
2552 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2553 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
2554 unsigned int authsize
= crypto_aead_authsize(tfm
);
2557 dst_size
= req
->assoclen
+ req
->cryptlen
+ (op_type
?
2558 -authsize
: authsize
);
2559 if (!req
->cryptlen
|| !dst_size
)
2561 reqctx
->iv_dma
= dma_map_single(dev
, reqctx
->iv
, (IV
+ reqctx
->b0_len
),
2563 if (dma_mapping_error(dev
, reqctx
->iv_dma
))
2566 reqctx
->b0_dma
= reqctx
->iv_dma
+ IV
;
2569 if (req
->src
== req
->dst
) {
2570 error
= dma_map_sg(dev
, req
->src
,
2571 sg_nents_for_len(req
->src
, dst_size
),
2576 error
= dma_map_sg(dev
, req
->src
, sg_nents(req
->src
),
2580 error
= dma_map_sg(dev
, req
->dst
, sg_nents(req
->dst
),
2583 dma_unmap_sg(dev
, req
->src
, sg_nents(req
->src
),
2591 dma_unmap_single(dev
, reqctx
->iv_dma
, IV
, DMA_BIDIRECTIONAL
);
2595 void chcr_aead_dma_unmap(struct device
*dev
,
2596 struct aead_request
*req
,
2597 unsigned short op_type
)
2599 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2600 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
2601 unsigned int authsize
= crypto_aead_authsize(tfm
);
2604 dst_size
= req
->assoclen
+ req
->cryptlen
+ (op_type
?
2605 -authsize
: authsize
);
2606 if (!req
->cryptlen
|| !dst_size
)
2609 dma_unmap_single(dev
, reqctx
->iv_dma
, (IV
+ reqctx
->b0_len
),
2611 if (req
->src
== req
->dst
) {
2612 dma_unmap_sg(dev
, req
->src
, sg_nents(req
->src
),
2615 dma_unmap_sg(dev
, req
->src
, sg_nents(req
->src
),
2617 dma_unmap_sg(dev
, req
->dst
, sg_nents(req
->dst
),
2622 void chcr_add_aead_src_ent(struct aead_request
*req
,
2623 struct ulptx_sgl
*ulptx
)
2625 struct ulptx_walk ulp_walk
;
2626 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2629 u8
*buf
= (u8
*)ulptx
;
2631 if (reqctx
->b0_len
) {
2632 memcpy(buf
, reqctx
->scratch_pad
, reqctx
->b0_len
);
2633 buf
+= reqctx
->b0_len
;
2635 sg_pcopy_to_buffer(req
->src
, sg_nents(req
->src
),
2636 buf
, req
->cryptlen
+ req
->assoclen
, 0);
2638 ulptx_walk_init(&ulp_walk
, ulptx
);
2640 ulptx_walk_add_page(&ulp_walk
, reqctx
->b0_len
,
2642 ulptx_walk_add_sg(&ulp_walk
, req
->src
, req
->cryptlen
+
2644 ulptx_walk_end(&ulp_walk
);
2648 void chcr_add_aead_dst_ent(struct aead_request
*req
,
2649 struct cpl_rx_phys_dsgl
*phys_cpl
,
2652 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2653 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
2654 struct dsgl_walk dsgl_walk
;
2655 unsigned int authsize
= crypto_aead_authsize(tfm
);
2656 struct chcr_context
*ctx
= a_ctx(tfm
);
2658 unsigned int rx_channel_id
= reqctx
->rxqidx
/ ctx
->rxq_perchan
;
2660 dsgl_walk_init(&dsgl_walk
, phys_cpl
);
2661 dsgl_walk_add_page(&dsgl_walk
, IV
+ reqctx
->b0_len
, reqctx
->iv_dma
);
2662 temp
= req
->assoclen
+ req
->cryptlen
+
2663 (reqctx
->op
? -authsize
: authsize
);
2664 dsgl_walk_add_sg(&dsgl_walk
, req
->dst
, temp
, 0);
2665 dsgl_walk_end(&dsgl_walk
, qid
, rx_channel_id
);
2668 void chcr_add_cipher_src_ent(struct skcipher_request
*req
,
2670 struct cipher_wr_param
*wrparam
)
2672 struct ulptx_walk ulp_walk
;
2673 struct chcr_skcipher_req_ctx
*reqctx
= skcipher_request_ctx(req
);
2676 memcpy(buf
, reqctx
->iv
, IV
);
2679 sg_pcopy_to_buffer(req
->src
, sg_nents(req
->src
),
2680 buf
, wrparam
->bytes
, reqctx
->processed
);
2682 ulptx_walk_init(&ulp_walk
, (struct ulptx_sgl
*)buf
);
2683 ulptx_walk_add_sg(&ulp_walk
, reqctx
->srcsg
, wrparam
->bytes
,
2685 reqctx
->srcsg
= ulp_walk
.last_sg
;
2686 reqctx
->src_ofst
= ulp_walk
.last_sg_len
;
2687 ulptx_walk_end(&ulp_walk
);
2691 void chcr_add_cipher_dst_ent(struct skcipher_request
*req
,
2692 struct cpl_rx_phys_dsgl
*phys_cpl
,
2693 struct cipher_wr_param
*wrparam
,
2696 struct chcr_skcipher_req_ctx
*reqctx
= skcipher_request_ctx(req
);
2697 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(wrparam
->req
);
2698 struct chcr_context
*ctx
= c_ctx(tfm
);
2699 struct dsgl_walk dsgl_walk
;
2700 unsigned int rx_channel_id
= reqctx
->rxqidx
/ ctx
->rxq_perchan
;
2702 dsgl_walk_init(&dsgl_walk
, phys_cpl
);
2703 dsgl_walk_add_sg(&dsgl_walk
, reqctx
->dstsg
, wrparam
->bytes
,
2705 reqctx
->dstsg
= dsgl_walk
.last_sg
;
2706 reqctx
->dst_ofst
= dsgl_walk
.last_sg_len
;
2707 dsgl_walk_end(&dsgl_walk
, qid
, rx_channel_id
);
2710 void chcr_add_hash_src_ent(struct ahash_request
*req
,
2711 struct ulptx_sgl
*ulptx
,
2712 struct hash_wr_param
*param
)
2714 struct ulptx_walk ulp_walk
;
2715 struct chcr_ahash_req_ctx
*reqctx
= ahash_request_ctx(req
);
2717 if (reqctx
->hctx_wr
.imm
) {
2718 u8
*buf
= (u8
*)ulptx
;
2720 if (param
->bfr_len
) {
2721 memcpy(buf
, reqctx
->reqbfr
, param
->bfr_len
);
2722 buf
+= param
->bfr_len
;
2725 sg_pcopy_to_buffer(reqctx
->hctx_wr
.srcsg
,
2726 sg_nents(reqctx
->hctx_wr
.srcsg
), buf
,
2729 ulptx_walk_init(&ulp_walk
, ulptx
);
2731 ulptx_walk_add_page(&ulp_walk
, param
->bfr_len
,
2732 reqctx
->hctx_wr
.dma_addr
);
2733 ulptx_walk_add_sg(&ulp_walk
, reqctx
->hctx_wr
.srcsg
,
2734 param
->sg_len
, reqctx
->hctx_wr
.src_ofst
);
2735 reqctx
->hctx_wr
.srcsg
= ulp_walk
.last_sg
;
2736 reqctx
->hctx_wr
.src_ofst
= ulp_walk
.last_sg_len
;
2737 ulptx_walk_end(&ulp_walk
);
2741 int chcr_hash_dma_map(struct device
*dev
,
2742 struct ahash_request
*req
)
2744 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(req
);
2749 error
= dma_map_sg(dev
, req
->src
, sg_nents(req
->src
),
2753 req_ctx
->hctx_wr
.is_sg_map
= 1;
2757 void chcr_hash_dma_unmap(struct device
*dev
,
2758 struct ahash_request
*req
)
2760 struct chcr_ahash_req_ctx
*req_ctx
= ahash_request_ctx(req
);
2765 dma_unmap_sg(dev
, req
->src
, sg_nents(req
->src
),
2767 req_ctx
->hctx_wr
.is_sg_map
= 0;
2771 int chcr_cipher_dma_map(struct device
*dev
,
2772 struct skcipher_request
*req
)
2776 if (req
->src
== req
->dst
) {
2777 error
= dma_map_sg(dev
, req
->src
, sg_nents(req
->src
),
2782 error
= dma_map_sg(dev
, req
->src
, sg_nents(req
->src
),
2786 error
= dma_map_sg(dev
, req
->dst
, sg_nents(req
->dst
),
2789 dma_unmap_sg(dev
, req
->src
, sg_nents(req
->src
),
2800 void chcr_cipher_dma_unmap(struct device
*dev
,
2801 struct skcipher_request
*req
)
2803 if (req
->src
== req
->dst
) {
2804 dma_unmap_sg(dev
, req
->src
, sg_nents(req
->src
),
2807 dma_unmap_sg(dev
, req
->src
, sg_nents(req
->src
),
2809 dma_unmap_sg(dev
, req
->dst
, sg_nents(req
->dst
),
2814 static int set_msg_len(u8
*block
, unsigned int msglen
, int csize
)
2818 memset(block
, 0, csize
);
2823 else if (msglen
> (unsigned int)(1 << (8 * csize
)))
2826 data
= cpu_to_be32(msglen
);
2827 memcpy(block
- csize
, (u8
*)&data
+ 4 - csize
, csize
);
2832 static int generate_b0(struct aead_request
*req
, u8
*ivptr
,
2833 unsigned short op_type
)
2835 unsigned int l
, lp
, m
;
2837 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
2838 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2839 u8
*b0
= reqctx
->scratch_pad
;
2841 m
= crypto_aead_authsize(aead
);
2843 memcpy(b0
, ivptr
, 16);
2848 /* set m, bits 3-5 */
2849 *b0
|= (8 * ((m
- 2) / 2));
2851 /* set adata, bit 6, if associated data is used */
2854 rc
= set_msg_len(b0
+ 16 - l
,
2855 (op_type
== CHCR_DECRYPT_OP
) ?
2856 req
->cryptlen
- m
: req
->cryptlen
, l
);
2861 static inline int crypto_ccm_check_iv(const u8
*iv
)
2863 /* 2 <= L <= 8, so 1 <= L' <= 7. */
2864 if (iv
[0] < 1 || iv
[0] > 7)
2870 static int ccm_format_packet(struct aead_request
*req
,
2872 unsigned int sub_type
,
2873 unsigned short op_type
,
2874 unsigned int assoclen
)
2876 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2877 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
2878 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
2881 if (sub_type
== CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309
) {
2883 memcpy(ivptr
+ 1, &aeadctx
->salt
[0], 3);
2884 memcpy(ivptr
+ 4, req
->iv
, 8);
2885 memset(ivptr
+ 12, 0, 4);
2887 memcpy(ivptr
, req
->iv
, 16);
2890 *((unsigned short *)(reqctx
->scratch_pad
+ 16)) =
2893 rc
= generate_b0(req
, ivptr
, op_type
);
2894 /* zero the ctr value */
2895 memset(ivptr
+ 15 - ivptr
[0], 0, ivptr
[0] + 1);
2899 static void fill_sec_cpl_for_aead(struct cpl_tx_sec_pdu
*sec_cpl
,
2900 unsigned int dst_size
,
2901 struct aead_request
*req
,
2902 unsigned short op_type
)
2904 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
2905 struct chcr_context
*ctx
= a_ctx(tfm
);
2906 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(ctx
);
2907 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2908 unsigned int cipher_mode
= CHCR_SCMD_CIPHER_MODE_AES_CCM
;
2909 unsigned int mac_mode
= CHCR_SCMD_AUTH_MODE_CBCMAC
;
2910 unsigned int rx_channel_id
= reqctx
->rxqidx
/ ctx
->rxq_perchan
;
2911 unsigned int ccm_xtra
;
2912 unsigned char tag_offset
= 0, auth_offset
= 0;
2913 unsigned int assoclen
;
2915 if (get_aead_subtype(tfm
) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309
)
2916 assoclen
= req
->assoclen
- 8;
2918 assoclen
= req
->assoclen
;
2919 ccm_xtra
= CCM_B0_SIZE
+
2920 ((assoclen
) ? CCM_AAD_FIELD_SIZE
: 0);
2922 auth_offset
= req
->cryptlen
?
2923 (req
->assoclen
+ IV
+ 1 + ccm_xtra
) : 0;
2924 if (op_type
== CHCR_DECRYPT_OP
) {
2925 if (crypto_aead_authsize(tfm
) != req
->cryptlen
)
2926 tag_offset
= crypto_aead_authsize(tfm
);
2931 sec_cpl
->op_ivinsrtofst
= FILL_SEC_CPL_OP_IVINSR(rx_channel_id
, 2, 1);
2933 htonl(req
->assoclen
+ IV
+ req
->cryptlen
+ ccm_xtra
);
2934 /* For CCM there wil be b0 always. So AAD start will be 1 always */
2935 sec_cpl
->aadstart_cipherstop_hi
= FILL_SEC_CPL_CIPHERSTOP_HI(
2936 1 + IV
, IV
+ assoclen
+ ccm_xtra
,
2937 req
->assoclen
+ IV
+ 1 + ccm_xtra
, 0);
2939 sec_cpl
->cipherstop_lo_authinsert
= FILL_SEC_CPL_AUTHINSERT(0,
2940 auth_offset
, tag_offset
,
2941 (op_type
== CHCR_ENCRYPT_OP
) ? 0 :
2942 crypto_aead_authsize(tfm
));
2943 sec_cpl
->seqno_numivs
= FILL_SEC_CPL_SCMD0_SEQNO(op_type
,
2944 (op_type
== CHCR_ENCRYPT_OP
) ? 0 : 1,
2945 cipher_mode
, mac_mode
,
2946 aeadctx
->hmac_ctrl
, IV
>> 1);
2948 sec_cpl
->ivgen_hdrlen
= FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1, 0,
2952 static int aead_ccm_validate_input(unsigned short op_type
,
2953 struct aead_request
*req
,
2954 struct chcr_aead_ctx
*aeadctx
,
2955 unsigned int sub_type
)
2957 if (sub_type
!= CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309
) {
2958 if (crypto_ccm_check_iv(req
->iv
)) {
2959 pr_err("CCM: IV check fails\n");
2963 if (req
->assoclen
!= 16 && req
->assoclen
!= 20) {
2964 pr_err("RFC4309: Invalid AAD length %d\n",
2972 static struct sk_buff
*create_aead_ccm_wr(struct aead_request
*req
,
2976 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
2977 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
2978 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
2979 struct sk_buff
*skb
= NULL
;
2980 struct chcr_wr
*chcr_req
;
2981 struct cpl_rx_phys_dsgl
*phys_cpl
;
2982 struct ulptx_sgl
*ulptx
;
2983 unsigned int transhdr_len
;
2984 unsigned int dst_size
= 0, kctx_len
, dnents
, temp
, snents
;
2985 unsigned int sub_type
, assoclen
= req
->assoclen
;
2986 unsigned int authsize
= crypto_aead_authsize(tfm
);
2987 int error
= -EINVAL
;
2989 gfp_t flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
? GFP_KERNEL
:
2991 struct adapter
*adap
= padap(a_ctx(tfm
)->dev
);
2993 sub_type
= get_aead_subtype(tfm
);
2994 if (sub_type
== CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309
)
2996 reqctx
->b0_len
= CCM_B0_SIZE
+ (assoclen
? CCM_AAD_FIELD_SIZE
: 0);
2997 error
= chcr_aead_common_init(req
);
2999 return ERR_PTR(error
);
3001 error
= aead_ccm_validate_input(reqctx
->op
, req
, aeadctx
, sub_type
);
3004 dnents
= sg_nents_xlen(req
->dst
, req
->assoclen
+ req
->cryptlen
3005 + (reqctx
->op
? -authsize
: authsize
),
3006 CHCR_DST_SG_SIZE
, 0);
3007 dnents
+= MIN_CCM_SG
; // For IV and B0
3008 dst_size
= get_space_for_phys_dsgl(dnents
);
3009 snents
= sg_nents_xlen(req
->src
, req
->assoclen
+ req
->cryptlen
,
3010 CHCR_SRC_SG_SIZE
, 0);
3011 snents
+= MIN_CCM_SG
; //For B0
3012 kctx_len
= roundup(aeadctx
->enckey_len
, 16) * 2;
3013 transhdr_len
= CIPHER_TRANSHDR_SIZE(kctx_len
, dst_size
);
3014 reqctx
->imm
= (transhdr_len
+ req
->assoclen
+ req
->cryptlen
+
3015 reqctx
->b0_len
) <= SGE_MAX_WR_LEN
;
3016 temp
= reqctx
->imm
? roundup(req
->assoclen
+ req
->cryptlen
+
3017 reqctx
->b0_len
, 16) :
3018 (sgl_len(snents
) * 8);
3019 transhdr_len
+= temp
;
3020 transhdr_len
= roundup(transhdr_len
, 16);
3022 if (chcr_aead_need_fallback(req
, dnents
, T6_MAX_AAD_SIZE
-
3023 reqctx
->b0_len
, transhdr_len
, reqctx
->op
)) {
3024 atomic_inc(&adap
->chcr_stats
.fallback
);
3025 chcr_aead_common_exit(req
);
3026 return ERR_PTR(chcr_aead_fallback(req
, reqctx
->op
));
3028 skb
= alloc_skb(transhdr_len
, flags
);
3035 chcr_req
= __skb_put_zero(skb
, transhdr_len
);
3037 fill_sec_cpl_for_aead(&chcr_req
->sec_cpl
, dst_size
, req
, reqctx
->op
);
3039 chcr_req
->key_ctx
.ctx_hdr
= aeadctx
->key_ctx_hdr
;
3040 memcpy(chcr_req
->key_ctx
.key
, aeadctx
->key
, aeadctx
->enckey_len
);
3041 memcpy(chcr_req
->key_ctx
.key
+ roundup(aeadctx
->enckey_len
, 16),
3042 aeadctx
->key
, aeadctx
->enckey_len
);
3044 phys_cpl
= (struct cpl_rx_phys_dsgl
*)((u8
*)(chcr_req
+ 1) + kctx_len
);
3045 ivptr
= (u8
*)(phys_cpl
+ 1) + dst_size
;
3046 ulptx
= (struct ulptx_sgl
*)(ivptr
+ IV
);
3047 error
= ccm_format_packet(req
, ivptr
, sub_type
, reqctx
->op
, assoclen
);
3050 chcr_add_aead_dst_ent(req
, phys_cpl
, qid
);
3051 chcr_add_aead_src_ent(req
, ulptx
);
3053 atomic_inc(&adap
->chcr_stats
.aead_rqst
);
3054 temp
= sizeof(struct cpl_rx_phys_dsgl
) + dst_size
+ IV
+
3055 kctx_len
+ (reqctx
->imm
? (req
->assoclen
+ req
->cryptlen
+
3056 reqctx
->b0_len
) : 0);
3057 create_wreq(a_ctx(tfm
), chcr_req
, &req
->base
, reqctx
->imm
, 0,
3058 transhdr_len
, temp
, 0);
3065 chcr_aead_common_exit(req
);
3066 return ERR_PTR(error
);
3069 static struct sk_buff
*create_gcm_wr(struct aead_request
*req
,
3073 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
3074 struct chcr_context
*ctx
= a_ctx(tfm
);
3075 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(ctx
);
3076 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
3077 struct sk_buff
*skb
= NULL
;
3078 struct chcr_wr
*chcr_req
;
3079 struct cpl_rx_phys_dsgl
*phys_cpl
;
3080 struct ulptx_sgl
*ulptx
;
3081 unsigned int transhdr_len
, dnents
= 0, snents
;
3082 unsigned int dst_size
= 0, temp
= 0, kctx_len
, assoclen
= req
->assoclen
;
3083 unsigned int authsize
= crypto_aead_authsize(tfm
);
3084 int error
= -EINVAL
;
3086 gfp_t flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
? GFP_KERNEL
:
3088 struct adapter
*adap
= padap(ctx
->dev
);
3089 unsigned int rx_channel_id
= reqctx
->rxqidx
/ ctx
->rxq_perchan
;
3091 if (get_aead_subtype(tfm
) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106
)
3092 assoclen
= req
->assoclen
- 8;
3095 error
= chcr_aead_common_init(req
);
3097 return ERR_PTR(error
);
3098 dnents
= sg_nents_xlen(req
->dst
, req
->assoclen
+ req
->cryptlen
+
3099 (reqctx
->op
? -authsize
: authsize
),
3100 CHCR_DST_SG_SIZE
, 0);
3101 snents
= sg_nents_xlen(req
->src
, req
->assoclen
+ req
->cryptlen
,
3102 CHCR_SRC_SG_SIZE
, 0);
3103 dnents
+= MIN_GCM_SG
; // For IV
3104 dst_size
= get_space_for_phys_dsgl(dnents
);
3105 kctx_len
= roundup(aeadctx
->enckey_len
, 16) + AEAD_H_SIZE
;
3106 transhdr_len
= CIPHER_TRANSHDR_SIZE(kctx_len
, dst_size
);
3107 reqctx
->imm
= (transhdr_len
+ req
->assoclen
+ req
->cryptlen
) <=
3109 temp
= reqctx
->imm
? roundup(req
->assoclen
+ req
->cryptlen
, 16) :
3110 (sgl_len(snents
) * 8);
3111 transhdr_len
+= temp
;
3112 transhdr_len
= roundup(transhdr_len
, 16);
3113 if (chcr_aead_need_fallback(req
, dnents
, T6_MAX_AAD_SIZE
,
3114 transhdr_len
, reqctx
->op
)) {
3116 atomic_inc(&adap
->chcr_stats
.fallback
);
3117 chcr_aead_common_exit(req
);
3118 return ERR_PTR(chcr_aead_fallback(req
, reqctx
->op
));
3120 skb
= alloc_skb(transhdr_len
, flags
);
3126 chcr_req
= __skb_put_zero(skb
, transhdr_len
);
3128 //Offset of tag from end
3129 temp
= (reqctx
->op
== CHCR_ENCRYPT_OP
) ? 0 : authsize
;
3130 chcr_req
->sec_cpl
.op_ivinsrtofst
= FILL_SEC_CPL_OP_IVINSR(
3131 rx_channel_id
, 2, 1);
3132 chcr_req
->sec_cpl
.pldlen
=
3133 htonl(req
->assoclen
+ IV
+ req
->cryptlen
);
3134 chcr_req
->sec_cpl
.aadstart_cipherstop_hi
= FILL_SEC_CPL_CIPHERSTOP_HI(
3135 assoclen
? 1 + IV
: 0,
3136 assoclen
? IV
+ assoclen
: 0,
3137 req
->assoclen
+ IV
+ 1, 0);
3138 chcr_req
->sec_cpl
.cipherstop_lo_authinsert
=
3139 FILL_SEC_CPL_AUTHINSERT(0, req
->assoclen
+ IV
+ 1,
3141 chcr_req
->sec_cpl
.seqno_numivs
=
3142 FILL_SEC_CPL_SCMD0_SEQNO(reqctx
->op
, (reqctx
->op
==
3143 CHCR_ENCRYPT_OP
) ? 1 : 0,
3144 CHCR_SCMD_CIPHER_MODE_AES_GCM
,
3145 CHCR_SCMD_AUTH_MODE_GHASH
,
3146 aeadctx
->hmac_ctrl
, IV
>> 1);
3147 chcr_req
->sec_cpl
.ivgen_hdrlen
= FILL_SEC_CPL_IVGEN_HDRLEN(0, 0, 1,
3149 chcr_req
->key_ctx
.ctx_hdr
= aeadctx
->key_ctx_hdr
;
3150 memcpy(chcr_req
->key_ctx
.key
, aeadctx
->key
, aeadctx
->enckey_len
);
3151 memcpy(chcr_req
->key_ctx
.key
+ roundup(aeadctx
->enckey_len
, 16),
3152 GCM_CTX(aeadctx
)->ghash_h
, AEAD_H_SIZE
);
3154 phys_cpl
= (struct cpl_rx_phys_dsgl
*)((u8
*)(chcr_req
+ 1) + kctx_len
);
3155 ivptr
= (u8
*)(phys_cpl
+ 1) + dst_size
;
3156 /* prepare a 16 byte iv */
3157 /* S A L T | IV | 0x00000001 */
3158 if (get_aead_subtype(tfm
) ==
3159 CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106
) {
3160 memcpy(ivptr
, aeadctx
->salt
, 4);
3161 memcpy(ivptr
+ 4, req
->iv
, GCM_RFC4106_IV_SIZE
);
3163 memcpy(ivptr
, req
->iv
, GCM_AES_IV_SIZE
);
3165 *((unsigned int *)(ivptr
+ 12)) = htonl(0x01);
3167 ulptx
= (struct ulptx_sgl
*)(ivptr
+ 16);
3169 chcr_add_aead_dst_ent(req
, phys_cpl
, qid
);
3170 chcr_add_aead_src_ent(req
, ulptx
);
3171 atomic_inc(&adap
->chcr_stats
.aead_rqst
);
3172 temp
= sizeof(struct cpl_rx_phys_dsgl
) + dst_size
+ IV
+
3173 kctx_len
+ (reqctx
->imm
? (req
->assoclen
+ req
->cryptlen
) : 0);
3174 create_wreq(a_ctx(tfm
), chcr_req
, &req
->base
, reqctx
->imm
, size
,
3175 transhdr_len
, temp
, reqctx
->verify
);
3180 chcr_aead_common_exit(req
);
3181 return ERR_PTR(error
);
3186 static int chcr_aead_cra_init(struct crypto_aead
*tfm
)
3188 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
3189 struct aead_alg
*alg
= crypto_aead_alg(tfm
);
3191 aeadctx
->sw_cipher
= crypto_alloc_aead(alg
->base
.cra_name
, 0,
3192 CRYPTO_ALG_NEED_FALLBACK
|
3194 if (IS_ERR(aeadctx
->sw_cipher
))
3195 return PTR_ERR(aeadctx
->sw_cipher
);
3196 crypto_aead_set_reqsize(tfm
, max(sizeof(struct chcr_aead_reqctx
),
3197 sizeof(struct aead_request
) +
3198 crypto_aead_reqsize(aeadctx
->sw_cipher
)));
3199 return chcr_device_init(a_ctx(tfm
));
3202 static void chcr_aead_cra_exit(struct crypto_aead
*tfm
)
3204 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
3206 crypto_free_aead(aeadctx
->sw_cipher
);
3209 static int chcr_authenc_null_setauthsize(struct crypto_aead
*tfm
,
3210 unsigned int authsize
)
3212 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
3214 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_NOP
;
3215 aeadctx
->mayverify
= VERIFY_HW
;
3216 return crypto_aead_setauthsize(aeadctx
->sw_cipher
, authsize
);
3218 static int chcr_authenc_setauthsize(struct crypto_aead
*tfm
,
3219 unsigned int authsize
)
3221 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
3222 u32 maxauth
= crypto_aead_maxauthsize(tfm
);
3224 /*SHA1 authsize in ipsec is 12 instead of 10 i.e maxauthsize / 2 is not
3225 * true for sha1. authsize == 12 condition should be before
3226 * authsize == (maxauth >> 1)
3228 if (authsize
== ICV_4
) {
3229 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_PL1
;
3230 aeadctx
->mayverify
= VERIFY_HW
;
3231 } else if (authsize
== ICV_6
) {
3232 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_PL2
;
3233 aeadctx
->mayverify
= VERIFY_HW
;
3234 } else if (authsize
== ICV_10
) {
3235 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366
;
3236 aeadctx
->mayverify
= VERIFY_HW
;
3237 } else if (authsize
== ICV_12
) {
3238 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT
;
3239 aeadctx
->mayverify
= VERIFY_HW
;
3240 } else if (authsize
== ICV_14
) {
3241 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_PL3
;
3242 aeadctx
->mayverify
= VERIFY_HW
;
3243 } else if (authsize
== (maxauth
>> 1)) {
3244 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_DIV2
;
3245 aeadctx
->mayverify
= VERIFY_HW
;
3246 } else if (authsize
== maxauth
) {
3247 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_NO_TRUNC
;
3248 aeadctx
->mayverify
= VERIFY_HW
;
3250 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_NO_TRUNC
;
3251 aeadctx
->mayverify
= VERIFY_SW
;
3253 return crypto_aead_setauthsize(aeadctx
->sw_cipher
, authsize
);
3257 static int chcr_gcm_setauthsize(struct crypto_aead
*tfm
, unsigned int authsize
)
3259 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
3263 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_PL1
;
3264 aeadctx
->mayverify
= VERIFY_HW
;
3267 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_DIV2
;
3268 aeadctx
->mayverify
= VERIFY_HW
;
3271 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT
;
3272 aeadctx
->mayverify
= VERIFY_HW
;
3275 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_PL3
;
3276 aeadctx
->mayverify
= VERIFY_HW
;
3279 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_NO_TRUNC
;
3280 aeadctx
->mayverify
= VERIFY_HW
;
3284 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_NO_TRUNC
;
3285 aeadctx
->mayverify
= VERIFY_SW
;
3290 return crypto_aead_setauthsize(aeadctx
->sw_cipher
, authsize
);
3293 static int chcr_4106_4309_setauthsize(struct crypto_aead
*tfm
,
3294 unsigned int authsize
)
3296 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
3300 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_DIV2
;
3301 aeadctx
->mayverify
= VERIFY_HW
;
3304 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT
;
3305 aeadctx
->mayverify
= VERIFY_HW
;
3308 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_NO_TRUNC
;
3309 aeadctx
->mayverify
= VERIFY_HW
;
3314 return crypto_aead_setauthsize(aeadctx
->sw_cipher
, authsize
);
3317 static int chcr_ccm_setauthsize(struct crypto_aead
*tfm
,
3318 unsigned int authsize
)
3320 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(tfm
));
3324 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_PL1
;
3325 aeadctx
->mayverify
= VERIFY_HW
;
3328 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_PL2
;
3329 aeadctx
->mayverify
= VERIFY_HW
;
3332 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_DIV2
;
3333 aeadctx
->mayverify
= VERIFY_HW
;
3336 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_TRUNC_RFC4366
;
3337 aeadctx
->mayverify
= VERIFY_HW
;
3340 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_IPSEC_96BIT
;
3341 aeadctx
->mayverify
= VERIFY_HW
;
3344 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_PL3
;
3345 aeadctx
->mayverify
= VERIFY_HW
;
3348 aeadctx
->hmac_ctrl
= CHCR_SCMD_HMAC_CTRL_NO_TRUNC
;
3349 aeadctx
->mayverify
= VERIFY_HW
;
3354 return crypto_aead_setauthsize(aeadctx
->sw_cipher
, authsize
);
3357 static int chcr_ccm_common_setkey(struct crypto_aead
*aead
,
3359 unsigned int keylen
)
3361 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(aead
));
3362 unsigned char ck_size
, mk_size
;
3363 int key_ctx_size
= 0;
3365 key_ctx_size
= sizeof(struct _key_ctx
) + roundup(keylen
, 16) * 2;
3366 if (keylen
== AES_KEYSIZE_128
) {
3367 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_128
;
3368 mk_size
= CHCR_KEYCTX_MAC_KEY_SIZE_128
;
3369 } else if (keylen
== AES_KEYSIZE_192
) {
3370 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_192
;
3371 mk_size
= CHCR_KEYCTX_MAC_KEY_SIZE_192
;
3372 } else if (keylen
== AES_KEYSIZE_256
) {
3373 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_256
;
3374 mk_size
= CHCR_KEYCTX_MAC_KEY_SIZE_256
;
3376 aeadctx
->enckey_len
= 0;
3379 aeadctx
->key_ctx_hdr
= FILL_KEY_CTX_HDR(ck_size
, mk_size
, 0, 0,
3381 memcpy(aeadctx
->key
, key
, keylen
);
3382 aeadctx
->enckey_len
= keylen
;
3387 static int chcr_aead_ccm_setkey(struct crypto_aead
*aead
,
3389 unsigned int keylen
)
3391 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(aead
));
3394 crypto_aead_clear_flags(aeadctx
->sw_cipher
, CRYPTO_TFM_REQ_MASK
);
3395 crypto_aead_set_flags(aeadctx
->sw_cipher
, crypto_aead_get_flags(aead
) &
3396 CRYPTO_TFM_REQ_MASK
);
3397 error
= crypto_aead_setkey(aeadctx
->sw_cipher
, key
, keylen
);
3400 return chcr_ccm_common_setkey(aead
, key
, keylen
);
3403 static int chcr_aead_rfc4309_setkey(struct crypto_aead
*aead
, const u8
*key
,
3404 unsigned int keylen
)
3406 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(aead
));
3410 aeadctx
->enckey_len
= 0;
3413 crypto_aead_clear_flags(aeadctx
->sw_cipher
, CRYPTO_TFM_REQ_MASK
);
3414 crypto_aead_set_flags(aeadctx
->sw_cipher
, crypto_aead_get_flags(aead
) &
3415 CRYPTO_TFM_REQ_MASK
);
3416 error
= crypto_aead_setkey(aeadctx
->sw_cipher
, key
, keylen
);
3420 memcpy(aeadctx
->salt
, key
+ keylen
, 3);
3421 return chcr_ccm_common_setkey(aead
, key
, keylen
);
3424 static int chcr_gcm_setkey(struct crypto_aead
*aead
, const u8
*key
,
3425 unsigned int keylen
)
3427 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(aead
));
3428 struct chcr_gcm_ctx
*gctx
= GCM_CTX(aeadctx
);
3429 unsigned int ck_size
;
3430 int ret
= 0, key_ctx_size
= 0;
3431 struct crypto_aes_ctx aes
;
3433 aeadctx
->enckey_len
= 0;
3434 crypto_aead_clear_flags(aeadctx
->sw_cipher
, CRYPTO_TFM_REQ_MASK
);
3435 crypto_aead_set_flags(aeadctx
->sw_cipher
, crypto_aead_get_flags(aead
)
3436 & CRYPTO_TFM_REQ_MASK
);
3437 ret
= crypto_aead_setkey(aeadctx
->sw_cipher
, key
, keylen
);
3441 if (get_aead_subtype(aead
) == CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106
&&
3443 keylen
-= 4; /* nonce/salt is present in the last 4 bytes */
3444 memcpy(aeadctx
->salt
, key
+ keylen
, 4);
3446 if (keylen
== AES_KEYSIZE_128
) {
3447 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_128
;
3448 } else if (keylen
== AES_KEYSIZE_192
) {
3449 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_192
;
3450 } else if (keylen
== AES_KEYSIZE_256
) {
3451 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_256
;
3453 pr_err("GCM: Invalid key length %d\n", keylen
);
3458 memcpy(aeadctx
->key
, key
, keylen
);
3459 aeadctx
->enckey_len
= keylen
;
3460 key_ctx_size
= sizeof(struct _key_ctx
) + roundup(keylen
, 16) +
3462 aeadctx
->key_ctx_hdr
= FILL_KEY_CTX_HDR(ck_size
,
3463 CHCR_KEYCTX_MAC_KEY_SIZE_128
,
3466 /* Calculate the H = CIPH(K, 0 repeated 16 times).
3467 * It will go in key context
3469 ret
= aes_expandkey(&aes
, key
, keylen
);
3471 aeadctx
->enckey_len
= 0;
3474 memset(gctx
->ghash_h
, 0, AEAD_H_SIZE
);
3475 aes_encrypt(&aes
, gctx
->ghash_h
, gctx
->ghash_h
);
3476 memzero_explicit(&aes
, sizeof(aes
));
3482 static int chcr_authenc_setkey(struct crypto_aead
*authenc
, const u8
*key
,
3483 unsigned int keylen
)
3485 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(authenc
));
3486 struct chcr_authenc_ctx
*actx
= AUTHENC_CTX(aeadctx
);
3487 /* it contains auth and cipher key both*/
3488 struct crypto_authenc_keys keys
;
3489 unsigned int bs
, subtype
;
3490 unsigned int max_authsize
= crypto_aead_alg(authenc
)->maxauthsize
;
3491 int err
= 0, i
, key_ctx_len
= 0;
3492 unsigned char ck_size
= 0;
3493 unsigned char pad
[CHCR_HASH_MAX_BLOCK_SIZE_128
] = { 0 };
3494 struct crypto_shash
*base_hash
= ERR_PTR(-EINVAL
);
3495 struct algo_param param
;
3499 crypto_aead_clear_flags(aeadctx
->sw_cipher
, CRYPTO_TFM_REQ_MASK
);
3500 crypto_aead_set_flags(aeadctx
->sw_cipher
, crypto_aead_get_flags(authenc
)
3501 & CRYPTO_TFM_REQ_MASK
);
3502 err
= crypto_aead_setkey(aeadctx
->sw_cipher
, key
, keylen
);
3506 if (crypto_authenc_extractkeys(&keys
, key
, keylen
) != 0)
3509 if (get_alg_config(¶m
, max_authsize
)) {
3510 pr_err("chcr : Unsupported digest size\n");
3513 subtype
= get_aead_subtype(authenc
);
3514 if (subtype
== CRYPTO_ALG_SUB_TYPE_CTR_SHA
||
3515 subtype
== CRYPTO_ALG_SUB_TYPE_CTR_NULL
) {
3516 if (keys
.enckeylen
< CTR_RFC3686_NONCE_SIZE
)
3518 memcpy(aeadctx
->nonce
, keys
.enckey
+ (keys
.enckeylen
3519 - CTR_RFC3686_NONCE_SIZE
), CTR_RFC3686_NONCE_SIZE
);
3520 keys
.enckeylen
-= CTR_RFC3686_NONCE_SIZE
;
3522 if (keys
.enckeylen
== AES_KEYSIZE_128
) {
3523 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_128
;
3524 } else if (keys
.enckeylen
== AES_KEYSIZE_192
) {
3525 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_192
;
3526 } else if (keys
.enckeylen
== AES_KEYSIZE_256
) {
3527 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_256
;
3529 pr_err("chcr : Unsupported cipher key\n");
3533 /* Copy only encryption key. We use authkey to generate h(ipad) and
3534 * h(opad) so authkey is not needed again. authkeylen size have the
3535 * size of the hash digest size.
3537 memcpy(aeadctx
->key
, keys
.enckey
, keys
.enckeylen
);
3538 aeadctx
->enckey_len
= keys
.enckeylen
;
3539 if (subtype
== CRYPTO_ALG_SUB_TYPE_CBC_SHA
||
3540 subtype
== CRYPTO_ALG_SUB_TYPE_CBC_NULL
) {
3542 get_aes_decrypt_key(actx
->dec_rrkey
, aeadctx
->key
,
3543 aeadctx
->enckey_len
<< 3);
3545 base_hash
= chcr_alloc_shash(max_authsize
);
3546 if (IS_ERR(base_hash
)) {
3547 pr_err("chcr : Base driver cannot be loaded\n");
3548 aeadctx
->enckey_len
= 0;
3549 memzero_explicit(&keys
, sizeof(keys
));
3553 SHASH_DESC_ON_STACK(shash
, base_hash
);
3555 shash
->tfm
= base_hash
;
3556 bs
= crypto_shash_blocksize(base_hash
);
3557 align
= KEYCTX_ALIGN_PAD(max_authsize
);
3558 o_ptr
= actx
->h_iopad
+ param
.result_size
+ align
;
3560 if (keys
.authkeylen
> bs
) {
3561 err
= crypto_shash_digest(shash
, keys
.authkey
,
3565 pr_err("chcr : Base driver cannot be loaded\n");
3568 keys
.authkeylen
= max_authsize
;
3570 memcpy(o_ptr
, keys
.authkey
, keys
.authkeylen
);
3572 /* Compute the ipad-digest*/
3573 memset(pad
+ keys
.authkeylen
, 0, bs
- keys
.authkeylen
);
3574 memcpy(pad
, o_ptr
, keys
.authkeylen
);
3575 for (i
= 0; i
< bs
>> 2; i
++)
3576 *((unsigned int *)pad
+ i
) ^= IPAD_DATA
;
3578 if (chcr_compute_partial_hash(shash
, pad
, actx
->h_iopad
,
3581 /* Compute the opad-digest */
3582 memset(pad
+ keys
.authkeylen
, 0, bs
- keys
.authkeylen
);
3583 memcpy(pad
, o_ptr
, keys
.authkeylen
);
3584 for (i
= 0; i
< bs
>> 2; i
++)
3585 *((unsigned int *)pad
+ i
) ^= OPAD_DATA
;
3587 if (chcr_compute_partial_hash(shash
, pad
, o_ptr
, max_authsize
))
3590 /* convert the ipad and opad digest to network order */
3591 chcr_change_order(actx
->h_iopad
, param
.result_size
);
3592 chcr_change_order(o_ptr
, param
.result_size
);
3593 key_ctx_len
= sizeof(struct _key_ctx
) +
3594 roundup(keys
.enckeylen
, 16) +
3595 (param
.result_size
+ align
) * 2;
3596 aeadctx
->key_ctx_hdr
= FILL_KEY_CTX_HDR(ck_size
, param
.mk_size
,
3597 0, 1, key_ctx_len
>> 4);
3598 actx
->auth_mode
= param
.auth_mode
;
3599 chcr_free_shash(base_hash
);
3601 memzero_explicit(&keys
, sizeof(keys
));
3605 aeadctx
->enckey_len
= 0;
3606 memzero_explicit(&keys
, sizeof(keys
));
3607 if (!IS_ERR(base_hash
))
3608 chcr_free_shash(base_hash
);
3612 static int chcr_aead_digest_null_setkey(struct crypto_aead
*authenc
,
3613 const u8
*key
, unsigned int keylen
)
3615 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(a_ctx(authenc
));
3616 struct chcr_authenc_ctx
*actx
= AUTHENC_CTX(aeadctx
);
3617 struct crypto_authenc_keys keys
;
3619 /* it contains auth and cipher key both*/
3620 unsigned int subtype
;
3621 int key_ctx_len
= 0;
3622 unsigned char ck_size
= 0;
3624 crypto_aead_clear_flags(aeadctx
->sw_cipher
, CRYPTO_TFM_REQ_MASK
);
3625 crypto_aead_set_flags(aeadctx
->sw_cipher
, crypto_aead_get_flags(authenc
)
3626 & CRYPTO_TFM_REQ_MASK
);
3627 err
= crypto_aead_setkey(aeadctx
->sw_cipher
, key
, keylen
);
3631 if (crypto_authenc_extractkeys(&keys
, key
, keylen
) != 0)
3634 subtype
= get_aead_subtype(authenc
);
3635 if (subtype
== CRYPTO_ALG_SUB_TYPE_CTR_SHA
||
3636 subtype
== CRYPTO_ALG_SUB_TYPE_CTR_NULL
) {
3637 if (keys
.enckeylen
< CTR_RFC3686_NONCE_SIZE
)
3639 memcpy(aeadctx
->nonce
, keys
.enckey
+ (keys
.enckeylen
3640 - CTR_RFC3686_NONCE_SIZE
), CTR_RFC3686_NONCE_SIZE
);
3641 keys
.enckeylen
-= CTR_RFC3686_NONCE_SIZE
;
3643 if (keys
.enckeylen
== AES_KEYSIZE_128
) {
3644 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_128
;
3645 } else if (keys
.enckeylen
== AES_KEYSIZE_192
) {
3646 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_192
;
3647 } else if (keys
.enckeylen
== AES_KEYSIZE_256
) {
3648 ck_size
= CHCR_KEYCTX_CIPHER_KEY_SIZE_256
;
3650 pr_err("chcr : Unsupported cipher key %d\n", keys
.enckeylen
);
3653 memcpy(aeadctx
->key
, keys
.enckey
, keys
.enckeylen
);
3654 aeadctx
->enckey_len
= keys
.enckeylen
;
3655 if (subtype
== CRYPTO_ALG_SUB_TYPE_CBC_SHA
||
3656 subtype
== CRYPTO_ALG_SUB_TYPE_CBC_NULL
) {
3657 get_aes_decrypt_key(actx
->dec_rrkey
, aeadctx
->key
,
3658 aeadctx
->enckey_len
<< 3);
3660 key_ctx_len
= sizeof(struct _key_ctx
) + roundup(keys
.enckeylen
, 16);
3662 aeadctx
->key_ctx_hdr
= FILL_KEY_CTX_HDR(ck_size
, CHCR_KEYCTX_NO_KEY
, 0,
3663 0, key_ctx_len
>> 4);
3664 actx
->auth_mode
= CHCR_SCMD_AUTH_MODE_NOP
;
3665 memzero_explicit(&keys
, sizeof(keys
));
3668 aeadctx
->enckey_len
= 0;
3669 memzero_explicit(&keys
, sizeof(keys
));
3673 static int chcr_aead_op(struct aead_request
*req
,
3675 create_wr_t create_wr_fn
)
3677 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
3678 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
3679 struct chcr_context
*ctx
= a_ctx(tfm
);
3680 struct uld_ctx
*u_ctx
= ULD_CTX(ctx
);
3681 struct sk_buff
*skb
;
3682 struct chcr_dev
*cdev
;
3684 cdev
= a_ctx(tfm
)->dev
;
3686 pr_err("chcr : %s : No crypto device.\n", __func__
);
3690 if (chcr_inc_wrcount(cdev
)) {
3691 /* Detach state for CHCR means lldi or padap is freed.
3692 * We cannot increment fallback here.
3694 return chcr_aead_fallback(req
, reqctx
->op
);
3697 if (cxgb4_is_crypto_q_full(u_ctx
->lldi
.ports
[0],
3699 (!(req
->base
.flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
))) {
3700 chcr_dec_wrcount(cdev
);
3704 /* Form a WR from req */
3705 skb
= create_wr_fn(req
, u_ctx
->lldi
.rxq_ids
[reqctx
->rxqidx
], size
);
3707 if (IS_ERR_OR_NULL(skb
)) {
3708 chcr_dec_wrcount(cdev
);
3709 return PTR_ERR_OR_ZERO(skb
);
3712 skb
->dev
= u_ctx
->lldi
.ports
[0];
3713 set_wr_txq(skb
, CPL_PRIORITY_DATA
, reqctx
->txqidx
);
3715 return -EINPROGRESS
;
3718 static int chcr_aead_encrypt(struct aead_request
*req
)
3720 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
3721 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
3722 struct chcr_context
*ctx
= a_ctx(tfm
);
3726 reqctx
->txqidx
= cpu
% ctx
->ntxq
;
3727 reqctx
->rxqidx
= cpu
% ctx
->nrxq
;
3730 reqctx
->verify
= VERIFY_HW
;
3731 reqctx
->op
= CHCR_ENCRYPT_OP
;
3733 switch (get_aead_subtype(tfm
)) {
3734 case CRYPTO_ALG_SUB_TYPE_CTR_SHA
:
3735 case CRYPTO_ALG_SUB_TYPE_CBC_SHA
:
3736 case CRYPTO_ALG_SUB_TYPE_CBC_NULL
:
3737 case CRYPTO_ALG_SUB_TYPE_CTR_NULL
:
3738 return chcr_aead_op(req
, 0, create_authenc_wr
);
3739 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM
:
3740 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309
:
3741 return chcr_aead_op(req
, 0, create_aead_ccm_wr
);
3743 return chcr_aead_op(req
, 0, create_gcm_wr
);
3747 static int chcr_aead_decrypt(struct aead_request
*req
)
3749 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
3750 struct chcr_context
*ctx
= a_ctx(tfm
);
3751 struct chcr_aead_ctx
*aeadctx
= AEAD_CTX(ctx
);
3752 struct chcr_aead_reqctx
*reqctx
= aead_request_ctx(req
);
3757 reqctx
->txqidx
= cpu
% ctx
->ntxq
;
3758 reqctx
->rxqidx
= cpu
% ctx
->nrxq
;
3761 if (aeadctx
->mayverify
== VERIFY_SW
) {
3762 size
= crypto_aead_maxauthsize(tfm
);
3763 reqctx
->verify
= VERIFY_SW
;
3766 reqctx
->verify
= VERIFY_HW
;
3768 reqctx
->op
= CHCR_DECRYPT_OP
;
3769 switch (get_aead_subtype(tfm
)) {
3770 case CRYPTO_ALG_SUB_TYPE_CBC_SHA
:
3771 case CRYPTO_ALG_SUB_TYPE_CTR_SHA
:
3772 case CRYPTO_ALG_SUB_TYPE_CBC_NULL
:
3773 case CRYPTO_ALG_SUB_TYPE_CTR_NULL
:
3774 return chcr_aead_op(req
, size
, create_authenc_wr
);
3775 case CRYPTO_ALG_SUB_TYPE_AEAD_CCM
:
3776 case CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309
:
3777 return chcr_aead_op(req
, size
, create_aead_ccm_wr
);
3779 return chcr_aead_op(req
, size
, create_gcm_wr
);
3783 static struct chcr_alg_template driver_algs
[] = {
3786 .type
= CRYPTO_ALG_TYPE_SKCIPHER
| CRYPTO_ALG_SUB_TYPE_CBC
,
3789 .base
.cra_name
= "cbc(aes)",
3790 .base
.cra_driver_name
= "cbc-aes-chcr",
3791 .base
.cra_blocksize
= AES_BLOCK_SIZE
,
3793 .init
= chcr_init_tfm
,
3794 .exit
= chcr_exit_tfm
,
3795 .min_keysize
= AES_MIN_KEY_SIZE
,
3796 .max_keysize
= AES_MAX_KEY_SIZE
,
3797 .ivsize
= AES_BLOCK_SIZE
,
3798 .setkey
= chcr_aes_cbc_setkey
,
3799 .encrypt
= chcr_aes_encrypt
,
3800 .decrypt
= chcr_aes_decrypt
,
3804 .type
= CRYPTO_ALG_TYPE_SKCIPHER
| CRYPTO_ALG_SUB_TYPE_XTS
,
3807 .base
.cra_name
= "xts(aes)",
3808 .base
.cra_driver_name
= "xts-aes-chcr",
3809 .base
.cra_blocksize
= AES_BLOCK_SIZE
,
3811 .init
= chcr_init_tfm
,
3812 .exit
= chcr_exit_tfm
,
3813 .min_keysize
= 2 * AES_MIN_KEY_SIZE
,
3814 .max_keysize
= 2 * AES_MAX_KEY_SIZE
,
3815 .ivsize
= AES_BLOCK_SIZE
,
3816 .setkey
= chcr_aes_xts_setkey
,
3817 .encrypt
= chcr_aes_encrypt
,
3818 .decrypt
= chcr_aes_decrypt
,
3822 .type
= CRYPTO_ALG_TYPE_SKCIPHER
| CRYPTO_ALG_SUB_TYPE_CTR
,
3825 .base
.cra_name
= "ctr(aes)",
3826 .base
.cra_driver_name
= "ctr-aes-chcr",
3827 .base
.cra_blocksize
= 1,
3829 .init
= chcr_init_tfm
,
3830 .exit
= chcr_exit_tfm
,
3831 .min_keysize
= AES_MIN_KEY_SIZE
,
3832 .max_keysize
= AES_MAX_KEY_SIZE
,
3833 .ivsize
= AES_BLOCK_SIZE
,
3834 .setkey
= chcr_aes_ctr_setkey
,
3835 .encrypt
= chcr_aes_encrypt
,
3836 .decrypt
= chcr_aes_decrypt
,
3840 .type
= CRYPTO_ALG_TYPE_SKCIPHER
|
3841 CRYPTO_ALG_SUB_TYPE_CTR_RFC3686
,
3844 .base
.cra_name
= "rfc3686(ctr(aes))",
3845 .base
.cra_driver_name
= "rfc3686-ctr-aes-chcr",
3846 .base
.cra_blocksize
= 1,
3848 .init
= chcr_rfc3686_init
,
3849 .exit
= chcr_exit_tfm
,
3850 .min_keysize
= AES_MIN_KEY_SIZE
+ CTR_RFC3686_NONCE_SIZE
,
3851 .max_keysize
= AES_MAX_KEY_SIZE
+ CTR_RFC3686_NONCE_SIZE
,
3852 .ivsize
= CTR_RFC3686_IV_SIZE
,
3853 .setkey
= chcr_aes_rfc3686_setkey
,
3854 .encrypt
= chcr_aes_encrypt
,
3855 .decrypt
= chcr_aes_decrypt
,
3860 .type
= CRYPTO_ALG_TYPE_AHASH
,
3863 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
3866 .cra_driver_name
= "sha1-chcr",
3867 .cra_blocksize
= SHA1_BLOCK_SIZE
,
3872 .type
= CRYPTO_ALG_TYPE_AHASH
,
3875 .halg
.digestsize
= SHA256_DIGEST_SIZE
,
3877 .cra_name
= "sha256",
3878 .cra_driver_name
= "sha256-chcr",
3879 .cra_blocksize
= SHA256_BLOCK_SIZE
,
3884 .type
= CRYPTO_ALG_TYPE_AHASH
,
3887 .halg
.digestsize
= SHA224_DIGEST_SIZE
,
3889 .cra_name
= "sha224",
3890 .cra_driver_name
= "sha224-chcr",
3891 .cra_blocksize
= SHA224_BLOCK_SIZE
,
3896 .type
= CRYPTO_ALG_TYPE_AHASH
,
3899 .halg
.digestsize
= SHA384_DIGEST_SIZE
,
3901 .cra_name
= "sha384",
3902 .cra_driver_name
= "sha384-chcr",
3903 .cra_blocksize
= SHA384_BLOCK_SIZE
,
3908 .type
= CRYPTO_ALG_TYPE_AHASH
,
3911 .halg
.digestsize
= SHA512_DIGEST_SIZE
,
3913 .cra_name
= "sha512",
3914 .cra_driver_name
= "sha512-chcr",
3915 .cra_blocksize
= SHA512_BLOCK_SIZE
,
3921 .type
= CRYPTO_ALG_TYPE_HMAC
,
3924 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
3926 .cra_name
= "hmac(sha1)",
3927 .cra_driver_name
= "hmac-sha1-chcr",
3928 .cra_blocksize
= SHA1_BLOCK_SIZE
,
3933 .type
= CRYPTO_ALG_TYPE_HMAC
,
3936 .halg
.digestsize
= SHA224_DIGEST_SIZE
,
3938 .cra_name
= "hmac(sha224)",
3939 .cra_driver_name
= "hmac-sha224-chcr",
3940 .cra_blocksize
= SHA224_BLOCK_SIZE
,
3945 .type
= CRYPTO_ALG_TYPE_HMAC
,
3948 .halg
.digestsize
= SHA256_DIGEST_SIZE
,
3950 .cra_name
= "hmac(sha256)",
3951 .cra_driver_name
= "hmac-sha256-chcr",
3952 .cra_blocksize
= SHA256_BLOCK_SIZE
,
3957 .type
= CRYPTO_ALG_TYPE_HMAC
,
3960 .halg
.digestsize
= SHA384_DIGEST_SIZE
,
3962 .cra_name
= "hmac(sha384)",
3963 .cra_driver_name
= "hmac-sha384-chcr",
3964 .cra_blocksize
= SHA384_BLOCK_SIZE
,
3969 .type
= CRYPTO_ALG_TYPE_HMAC
,
3972 .halg
.digestsize
= SHA512_DIGEST_SIZE
,
3974 .cra_name
= "hmac(sha512)",
3975 .cra_driver_name
= "hmac-sha512-chcr",
3976 .cra_blocksize
= SHA512_BLOCK_SIZE
,
3980 /* Add AEAD Algorithms */
3982 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_AEAD_GCM
,
3986 .cra_name
= "gcm(aes)",
3987 .cra_driver_name
= "gcm-aes-chcr",
3989 .cra_priority
= CHCR_AEAD_PRIORITY
,
3990 .cra_ctxsize
= sizeof(struct chcr_context
) +
3991 sizeof(struct chcr_aead_ctx
) +
3992 sizeof(struct chcr_gcm_ctx
),
3994 .ivsize
= GCM_AES_IV_SIZE
,
3995 .maxauthsize
= GHASH_DIGEST_SIZE
,
3996 .setkey
= chcr_gcm_setkey
,
3997 .setauthsize
= chcr_gcm_setauthsize
,
4001 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_AEAD_RFC4106
,
4005 .cra_name
= "rfc4106(gcm(aes))",
4006 .cra_driver_name
= "rfc4106-gcm-aes-chcr",
4008 .cra_priority
= CHCR_AEAD_PRIORITY
+ 1,
4009 .cra_ctxsize
= sizeof(struct chcr_context
) +
4010 sizeof(struct chcr_aead_ctx
) +
4011 sizeof(struct chcr_gcm_ctx
),
4014 .ivsize
= GCM_RFC4106_IV_SIZE
,
4015 .maxauthsize
= GHASH_DIGEST_SIZE
,
4016 .setkey
= chcr_gcm_setkey
,
4017 .setauthsize
= chcr_4106_4309_setauthsize
,
4021 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_AEAD_CCM
,
4025 .cra_name
= "ccm(aes)",
4026 .cra_driver_name
= "ccm-aes-chcr",
4028 .cra_priority
= CHCR_AEAD_PRIORITY
,
4029 .cra_ctxsize
= sizeof(struct chcr_context
) +
4030 sizeof(struct chcr_aead_ctx
),
4033 .ivsize
= AES_BLOCK_SIZE
,
4034 .maxauthsize
= GHASH_DIGEST_SIZE
,
4035 .setkey
= chcr_aead_ccm_setkey
,
4036 .setauthsize
= chcr_ccm_setauthsize
,
4040 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_AEAD_RFC4309
,
4044 .cra_name
= "rfc4309(ccm(aes))",
4045 .cra_driver_name
= "rfc4309-ccm-aes-chcr",
4047 .cra_priority
= CHCR_AEAD_PRIORITY
+ 1,
4048 .cra_ctxsize
= sizeof(struct chcr_context
) +
4049 sizeof(struct chcr_aead_ctx
),
4053 .maxauthsize
= GHASH_DIGEST_SIZE
,
4054 .setkey
= chcr_aead_rfc4309_setkey
,
4055 .setauthsize
= chcr_4106_4309_setauthsize
,
4059 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CBC_SHA
,
4063 .cra_name
= "authenc(hmac(sha1),cbc(aes))",
4065 "authenc-hmac-sha1-cbc-aes-chcr",
4066 .cra_blocksize
= AES_BLOCK_SIZE
,
4067 .cra_priority
= CHCR_AEAD_PRIORITY
,
4068 .cra_ctxsize
= sizeof(struct chcr_context
) +
4069 sizeof(struct chcr_aead_ctx
) +
4070 sizeof(struct chcr_authenc_ctx
),
4073 .ivsize
= AES_BLOCK_SIZE
,
4074 .maxauthsize
= SHA1_DIGEST_SIZE
,
4075 .setkey
= chcr_authenc_setkey
,
4076 .setauthsize
= chcr_authenc_setauthsize
,
4080 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CBC_SHA
,
4085 .cra_name
= "authenc(hmac(sha256),cbc(aes))",
4087 "authenc-hmac-sha256-cbc-aes-chcr",
4088 .cra_blocksize
= AES_BLOCK_SIZE
,
4089 .cra_priority
= CHCR_AEAD_PRIORITY
,
4090 .cra_ctxsize
= sizeof(struct chcr_context
) +
4091 sizeof(struct chcr_aead_ctx
) +
4092 sizeof(struct chcr_authenc_ctx
),
4095 .ivsize
= AES_BLOCK_SIZE
,
4096 .maxauthsize
= SHA256_DIGEST_SIZE
,
4097 .setkey
= chcr_authenc_setkey
,
4098 .setauthsize
= chcr_authenc_setauthsize
,
4102 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CBC_SHA
,
4106 .cra_name
= "authenc(hmac(sha224),cbc(aes))",
4108 "authenc-hmac-sha224-cbc-aes-chcr",
4109 .cra_blocksize
= AES_BLOCK_SIZE
,
4110 .cra_priority
= CHCR_AEAD_PRIORITY
,
4111 .cra_ctxsize
= sizeof(struct chcr_context
) +
4112 sizeof(struct chcr_aead_ctx
) +
4113 sizeof(struct chcr_authenc_ctx
),
4115 .ivsize
= AES_BLOCK_SIZE
,
4116 .maxauthsize
= SHA224_DIGEST_SIZE
,
4117 .setkey
= chcr_authenc_setkey
,
4118 .setauthsize
= chcr_authenc_setauthsize
,
4122 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CBC_SHA
,
4126 .cra_name
= "authenc(hmac(sha384),cbc(aes))",
4128 "authenc-hmac-sha384-cbc-aes-chcr",
4129 .cra_blocksize
= AES_BLOCK_SIZE
,
4130 .cra_priority
= CHCR_AEAD_PRIORITY
,
4131 .cra_ctxsize
= sizeof(struct chcr_context
) +
4132 sizeof(struct chcr_aead_ctx
) +
4133 sizeof(struct chcr_authenc_ctx
),
4136 .ivsize
= AES_BLOCK_SIZE
,
4137 .maxauthsize
= SHA384_DIGEST_SIZE
,
4138 .setkey
= chcr_authenc_setkey
,
4139 .setauthsize
= chcr_authenc_setauthsize
,
4143 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CBC_SHA
,
4147 .cra_name
= "authenc(hmac(sha512),cbc(aes))",
4149 "authenc-hmac-sha512-cbc-aes-chcr",
4150 .cra_blocksize
= AES_BLOCK_SIZE
,
4151 .cra_priority
= CHCR_AEAD_PRIORITY
,
4152 .cra_ctxsize
= sizeof(struct chcr_context
) +
4153 sizeof(struct chcr_aead_ctx
) +
4154 sizeof(struct chcr_authenc_ctx
),
4157 .ivsize
= AES_BLOCK_SIZE
,
4158 .maxauthsize
= SHA512_DIGEST_SIZE
,
4159 .setkey
= chcr_authenc_setkey
,
4160 .setauthsize
= chcr_authenc_setauthsize
,
4164 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CBC_NULL
,
4168 .cra_name
= "authenc(digest_null,cbc(aes))",
4170 "authenc-digest_null-cbc-aes-chcr",
4171 .cra_blocksize
= AES_BLOCK_SIZE
,
4172 .cra_priority
= CHCR_AEAD_PRIORITY
,
4173 .cra_ctxsize
= sizeof(struct chcr_context
) +
4174 sizeof(struct chcr_aead_ctx
) +
4175 sizeof(struct chcr_authenc_ctx
),
4178 .ivsize
= AES_BLOCK_SIZE
,
4180 .setkey
= chcr_aead_digest_null_setkey
,
4181 .setauthsize
= chcr_authenc_null_setauthsize
,
4185 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CTR_SHA
,
4189 .cra_name
= "authenc(hmac(sha1),rfc3686(ctr(aes)))",
4191 "authenc-hmac-sha1-rfc3686-ctr-aes-chcr",
4193 .cra_priority
= CHCR_AEAD_PRIORITY
,
4194 .cra_ctxsize
= sizeof(struct chcr_context
) +
4195 sizeof(struct chcr_aead_ctx
) +
4196 sizeof(struct chcr_authenc_ctx
),
4199 .ivsize
= CTR_RFC3686_IV_SIZE
,
4200 .maxauthsize
= SHA1_DIGEST_SIZE
,
4201 .setkey
= chcr_authenc_setkey
,
4202 .setauthsize
= chcr_authenc_setauthsize
,
4206 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CTR_SHA
,
4211 .cra_name
= "authenc(hmac(sha256),rfc3686(ctr(aes)))",
4213 "authenc-hmac-sha256-rfc3686-ctr-aes-chcr",
4215 .cra_priority
= CHCR_AEAD_PRIORITY
,
4216 .cra_ctxsize
= sizeof(struct chcr_context
) +
4217 sizeof(struct chcr_aead_ctx
) +
4218 sizeof(struct chcr_authenc_ctx
),
4221 .ivsize
= CTR_RFC3686_IV_SIZE
,
4222 .maxauthsize
= SHA256_DIGEST_SIZE
,
4223 .setkey
= chcr_authenc_setkey
,
4224 .setauthsize
= chcr_authenc_setauthsize
,
4228 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CTR_SHA
,
4232 .cra_name
= "authenc(hmac(sha224),rfc3686(ctr(aes)))",
4234 "authenc-hmac-sha224-rfc3686-ctr-aes-chcr",
4236 .cra_priority
= CHCR_AEAD_PRIORITY
,
4237 .cra_ctxsize
= sizeof(struct chcr_context
) +
4238 sizeof(struct chcr_aead_ctx
) +
4239 sizeof(struct chcr_authenc_ctx
),
4241 .ivsize
= CTR_RFC3686_IV_SIZE
,
4242 .maxauthsize
= SHA224_DIGEST_SIZE
,
4243 .setkey
= chcr_authenc_setkey
,
4244 .setauthsize
= chcr_authenc_setauthsize
,
4248 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CTR_SHA
,
4252 .cra_name
= "authenc(hmac(sha384),rfc3686(ctr(aes)))",
4254 "authenc-hmac-sha384-rfc3686-ctr-aes-chcr",
4256 .cra_priority
= CHCR_AEAD_PRIORITY
,
4257 .cra_ctxsize
= sizeof(struct chcr_context
) +
4258 sizeof(struct chcr_aead_ctx
) +
4259 sizeof(struct chcr_authenc_ctx
),
4262 .ivsize
= CTR_RFC3686_IV_SIZE
,
4263 .maxauthsize
= SHA384_DIGEST_SIZE
,
4264 .setkey
= chcr_authenc_setkey
,
4265 .setauthsize
= chcr_authenc_setauthsize
,
4269 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CTR_SHA
,
4273 .cra_name
= "authenc(hmac(sha512),rfc3686(ctr(aes)))",
4275 "authenc-hmac-sha512-rfc3686-ctr-aes-chcr",
4277 .cra_priority
= CHCR_AEAD_PRIORITY
,
4278 .cra_ctxsize
= sizeof(struct chcr_context
) +
4279 sizeof(struct chcr_aead_ctx
) +
4280 sizeof(struct chcr_authenc_ctx
),
4283 .ivsize
= CTR_RFC3686_IV_SIZE
,
4284 .maxauthsize
= SHA512_DIGEST_SIZE
,
4285 .setkey
= chcr_authenc_setkey
,
4286 .setauthsize
= chcr_authenc_setauthsize
,
4290 .type
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_SUB_TYPE_CTR_NULL
,
4294 .cra_name
= "authenc(digest_null,rfc3686(ctr(aes)))",
4296 "authenc-digest_null-rfc3686-ctr-aes-chcr",
4298 .cra_priority
= CHCR_AEAD_PRIORITY
,
4299 .cra_ctxsize
= sizeof(struct chcr_context
) +
4300 sizeof(struct chcr_aead_ctx
) +
4301 sizeof(struct chcr_authenc_ctx
),
4304 .ivsize
= CTR_RFC3686_IV_SIZE
,
4306 .setkey
= chcr_aead_digest_null_setkey
,
4307 .setauthsize
= chcr_authenc_null_setauthsize
,
4313 * chcr_unregister_alg - Deregister crypto algorithms with
4316 static int chcr_unregister_alg(void)
4320 for (i
= 0; i
< ARRAY_SIZE(driver_algs
); i
++) {
4321 switch (driver_algs
[i
].type
& CRYPTO_ALG_TYPE_MASK
) {
4322 case CRYPTO_ALG_TYPE_SKCIPHER
:
4323 if (driver_algs
[i
].is_registered
)
4324 crypto_unregister_skcipher(
4325 &driver_algs
[i
].alg
.skcipher
);
4327 case CRYPTO_ALG_TYPE_AEAD
:
4328 if (driver_algs
[i
].is_registered
)
4329 crypto_unregister_aead(
4330 &driver_algs
[i
].alg
.aead
);
4332 case CRYPTO_ALG_TYPE_AHASH
:
4333 if (driver_algs
[i
].is_registered
)
4334 crypto_unregister_ahash(
4335 &driver_algs
[i
].alg
.hash
);
4338 driver_algs
[i
].is_registered
= 0;
4343 #define SZ_AHASH_CTX sizeof(struct chcr_context)
4344 #define SZ_AHASH_H_CTX (sizeof(struct chcr_context) + sizeof(struct hmac_ctx))
4345 #define SZ_AHASH_REQ_CTX sizeof(struct chcr_ahash_req_ctx)
4348 * chcr_register_alg - Register crypto algorithms with kernel framework.
4350 static int chcr_register_alg(void)
4352 struct crypto_alg ai
;
4353 struct ahash_alg
*a_hash
;
4357 for (i
= 0; i
< ARRAY_SIZE(driver_algs
); i
++) {
4358 if (driver_algs
[i
].is_registered
)
4360 switch (driver_algs
[i
].type
& CRYPTO_ALG_TYPE_MASK
) {
4361 case CRYPTO_ALG_TYPE_SKCIPHER
:
4362 driver_algs
[i
].alg
.skcipher
.base
.cra_priority
=
4364 driver_algs
[i
].alg
.skcipher
.base
.cra_module
= THIS_MODULE
;
4365 driver_algs
[i
].alg
.skcipher
.base
.cra_flags
=
4366 CRYPTO_ALG_TYPE_SKCIPHER
| CRYPTO_ALG_ASYNC
|
4367 CRYPTO_ALG_NEED_FALLBACK
;
4368 driver_algs
[i
].alg
.skcipher
.base
.cra_ctxsize
=
4369 sizeof(struct chcr_context
) +
4370 sizeof(struct ablk_ctx
);
4371 driver_algs
[i
].alg
.skcipher
.base
.cra_alignmask
= 0;
4373 err
= crypto_register_skcipher(&driver_algs
[i
].alg
.skcipher
);
4374 name
= driver_algs
[i
].alg
.skcipher
.base
.cra_driver_name
;
4376 case CRYPTO_ALG_TYPE_AEAD
:
4377 driver_algs
[i
].alg
.aead
.base
.cra_flags
=
4378 CRYPTO_ALG_ASYNC
| CRYPTO_ALG_NEED_FALLBACK
;
4379 driver_algs
[i
].alg
.aead
.encrypt
= chcr_aead_encrypt
;
4380 driver_algs
[i
].alg
.aead
.decrypt
= chcr_aead_decrypt
;
4381 driver_algs
[i
].alg
.aead
.init
= chcr_aead_cra_init
;
4382 driver_algs
[i
].alg
.aead
.exit
= chcr_aead_cra_exit
;
4383 driver_algs
[i
].alg
.aead
.base
.cra_module
= THIS_MODULE
;
4384 err
= crypto_register_aead(&driver_algs
[i
].alg
.aead
);
4385 name
= driver_algs
[i
].alg
.aead
.base
.cra_driver_name
;
4387 case CRYPTO_ALG_TYPE_AHASH
:
4388 a_hash
= &driver_algs
[i
].alg
.hash
;
4389 a_hash
->update
= chcr_ahash_update
;
4390 a_hash
->final
= chcr_ahash_final
;
4391 a_hash
->finup
= chcr_ahash_finup
;
4392 a_hash
->digest
= chcr_ahash_digest
;
4393 a_hash
->export
= chcr_ahash_export
;
4394 a_hash
->import
= chcr_ahash_import
;
4395 a_hash
->halg
.statesize
= SZ_AHASH_REQ_CTX
;
4396 a_hash
->halg
.base
.cra_priority
= CHCR_CRA_PRIORITY
;
4397 a_hash
->halg
.base
.cra_module
= THIS_MODULE
;
4398 a_hash
->halg
.base
.cra_flags
= CRYPTO_ALG_ASYNC
;
4399 a_hash
->halg
.base
.cra_alignmask
= 0;
4400 a_hash
->halg
.base
.cra_exit
= NULL
;
4402 if (driver_algs
[i
].type
== CRYPTO_ALG_TYPE_HMAC
) {
4403 a_hash
->halg
.base
.cra_init
= chcr_hmac_cra_init
;
4404 a_hash
->halg
.base
.cra_exit
= chcr_hmac_cra_exit
;
4405 a_hash
->init
= chcr_hmac_init
;
4406 a_hash
->setkey
= chcr_ahash_setkey
;
4407 a_hash
->halg
.base
.cra_ctxsize
= SZ_AHASH_H_CTX
;
4409 a_hash
->init
= chcr_sha_init
;
4410 a_hash
->halg
.base
.cra_ctxsize
= SZ_AHASH_CTX
;
4411 a_hash
->halg
.base
.cra_init
= chcr_sha_cra_init
;
4413 err
= crypto_register_ahash(&driver_algs
[i
].alg
.hash
);
4414 ai
= driver_algs
[i
].alg
.hash
.halg
.base
;
4415 name
= ai
.cra_driver_name
;
4419 pr_err("chcr : %s : Algorithm registration failed\n",
4423 driver_algs
[i
].is_registered
= 1;
4429 chcr_unregister_alg();
4434 * start_crypto - Register the crypto algorithms.
4435 * This should called once when the first device comesup. After this
4436 * kernel will start calling driver APIs for crypto operations.
4438 int start_crypto(void)
4440 return chcr_register_alg();
4444 * stop_crypto - Deregister all the crypto algorithms with kernel.
4445 * This should be called once when the last device goes down. After this
4446 * kernel will not call the driver API for crypto operations.
4448 int stop_crypto(void)
4450 chcr_unregister_alg();