1 // SPDX-License-Identifier: GPL-2.0-only
3 * AMD Cryptographic Coprocessor (CCP) SHA crypto API support
5 * Copyright (C) 2013,2018 Advanced Micro Devices, Inc.
7 * Author: Tom Lendacky <thomas.lendacky@amd.com>
8 * Author: Gary R Hook <gary.hook@amd.com>
11 #include <linux/module.h>
12 #include <linux/sched.h>
13 #include <linux/delay.h>
14 #include <linux/scatterlist.h>
15 #include <linux/crypto.h>
16 #include <crypto/algapi.h>
17 #include <crypto/hash.h>
18 #include <crypto/hmac.h>
19 #include <crypto/internal/hash.h>
20 #include <crypto/sha.h>
21 #include <crypto/scatterwalk.h>
22 #include <linux/string.h>
24 #include "ccp-crypto.h"
26 static int ccp_sha_complete(struct crypto_async_request
*async_req
, int ret
)
28 struct ahash_request
*req
= ahash_request_cast(async_req
);
29 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
30 struct ccp_sha_req_ctx
*rctx
= ahash_request_ctx(req
);
31 unsigned int digest_size
= crypto_ahash_digestsize(tfm
);
37 /* Save remaining data to buffer */
38 unsigned int offset
= rctx
->nbytes
- rctx
->hash_rem
;
40 scatterwalk_map_and_copy(rctx
->buf
, rctx
->src
,
41 offset
, rctx
->hash_rem
, 0);
42 rctx
->buf_count
= rctx
->hash_rem
;
47 /* Update result area if supplied */
48 if (req
->result
&& rctx
->final
)
49 memcpy(req
->result
, rctx
->ctx
, digest_size
);
52 sg_free_table(&rctx
->data_sg
);
57 static int ccp_do_sha_update(struct ahash_request
*req
, unsigned int nbytes
,
60 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
61 struct ccp_ctx
*ctx
= crypto_ahash_ctx(tfm
);
62 struct ccp_sha_req_ctx
*rctx
= ahash_request_ctx(req
);
63 struct scatterlist
*sg
;
64 unsigned int block_size
=
65 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
66 unsigned int sg_count
;
71 len
= (u64
)rctx
->buf_count
+ (u64
)nbytes
;
73 if (!final
&& (len
<= block_size
)) {
74 scatterwalk_map_and_copy(rctx
->buf
+ rctx
->buf_count
, req
->src
,
76 rctx
->buf_count
+= nbytes
;
82 rctx
->nbytes
= nbytes
;
85 rctx
->hash_rem
= final
? 0 : len
& (block_size
- 1);
86 rctx
->hash_cnt
= len
- rctx
->hash_rem
;
87 if (!final
&& !rctx
->hash_rem
) {
88 /* CCP can't do zero length final, so keep some data around */
89 rctx
->hash_cnt
-= block_size
;
90 rctx
->hash_rem
= block_size
;
93 /* Initialize the context scatterlist */
94 sg_init_one(&rctx
->ctx_sg
, rctx
->ctx
, sizeof(rctx
->ctx
));
97 if (rctx
->buf_count
&& nbytes
) {
98 /* Build the data scatterlist table - allocate enough entries
99 * for both data pieces (buffer and input data)
101 gfp
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
?
102 GFP_KERNEL
: GFP_ATOMIC
;
103 sg_count
= sg_nents(req
->src
) + 1;
104 ret
= sg_alloc_table(&rctx
->data_sg
, sg_count
, gfp
);
108 sg_init_one(&rctx
->buf_sg
, rctx
->buf
, rctx
->buf_count
);
109 sg
= ccp_crypto_sg_table_add(&rctx
->data_sg
, &rctx
->buf_sg
);
114 sg
= ccp_crypto_sg_table_add(&rctx
->data_sg
, req
->src
);
121 sg
= rctx
->data_sg
.sgl
;
122 } else if (rctx
->buf_count
) {
123 sg_init_one(&rctx
->buf_sg
, rctx
->buf
, rctx
->buf_count
);
130 rctx
->msg_bits
+= (rctx
->hash_cnt
<< 3); /* Total in bits */
132 memset(&rctx
->cmd
, 0, sizeof(rctx
->cmd
));
133 INIT_LIST_HEAD(&rctx
->cmd
.entry
);
134 rctx
->cmd
.engine
= CCP_ENGINE_SHA
;
135 rctx
->cmd
.u
.sha
.type
= rctx
->type
;
136 rctx
->cmd
.u
.sha
.ctx
= &rctx
->ctx_sg
;
138 switch (rctx
->type
) {
140 rctx
->cmd
.u
.sha
.ctx_len
= SHA1_DIGEST_SIZE
;
142 case CCP_SHA_TYPE_224
:
143 rctx
->cmd
.u
.sha
.ctx_len
= SHA224_DIGEST_SIZE
;
145 case CCP_SHA_TYPE_256
:
146 rctx
->cmd
.u
.sha
.ctx_len
= SHA256_DIGEST_SIZE
;
148 case CCP_SHA_TYPE_384
:
149 rctx
->cmd
.u
.sha
.ctx_len
= SHA384_DIGEST_SIZE
;
151 case CCP_SHA_TYPE_512
:
152 rctx
->cmd
.u
.sha
.ctx_len
= SHA512_DIGEST_SIZE
;
155 /* Should never get here */
159 rctx
->cmd
.u
.sha
.src
= sg
;
160 rctx
->cmd
.u
.sha
.src_len
= rctx
->hash_cnt
;
161 rctx
->cmd
.u
.sha
.opad
= ctx
->u
.sha
.key_len
?
162 &ctx
->u
.sha
.opad_sg
: NULL
;
163 rctx
->cmd
.u
.sha
.opad_len
= ctx
->u
.sha
.key_len
?
164 ctx
->u
.sha
.opad_count
: 0;
165 rctx
->cmd
.u
.sha
.first
= rctx
->first
;
166 rctx
->cmd
.u
.sha
.final
= rctx
->final
;
167 rctx
->cmd
.u
.sha
.msg_bits
= rctx
->msg_bits
;
171 ret
= ccp_crypto_enqueue_request(&req
->base
, &rctx
->cmd
);
176 sg_free_table(&rctx
->data_sg
);
181 static int ccp_sha_init(struct ahash_request
*req
)
183 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
184 struct ccp_ctx
*ctx
= crypto_ahash_ctx(tfm
);
185 struct ccp_sha_req_ctx
*rctx
= ahash_request_ctx(req
);
186 struct ccp_crypto_ahash_alg
*alg
=
187 ccp_crypto_ahash_alg(crypto_ahash_tfm(tfm
));
188 unsigned int block_size
=
189 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
191 memset(rctx
, 0, sizeof(*rctx
));
193 rctx
->type
= alg
->type
;
196 if (ctx
->u
.sha
.key_len
) {
197 /* Buffer the HMAC key for first update */
198 memcpy(rctx
->buf
, ctx
->u
.sha
.ipad
, block_size
);
199 rctx
->buf_count
= block_size
;
205 static int ccp_sha_update(struct ahash_request
*req
)
207 return ccp_do_sha_update(req
, req
->nbytes
, 0);
210 static int ccp_sha_final(struct ahash_request
*req
)
212 return ccp_do_sha_update(req
, 0, 1);
215 static int ccp_sha_finup(struct ahash_request
*req
)
217 return ccp_do_sha_update(req
, req
->nbytes
, 1);
220 static int ccp_sha_digest(struct ahash_request
*req
)
224 ret
= ccp_sha_init(req
);
228 return ccp_sha_finup(req
);
231 static int ccp_sha_export(struct ahash_request
*req
, void *out
)
233 struct ccp_sha_req_ctx
*rctx
= ahash_request_ctx(req
);
234 struct ccp_sha_exp_ctx state
;
236 /* Don't let anything leak to 'out' */
237 memset(&state
, 0, sizeof(state
));
239 state
.type
= rctx
->type
;
240 state
.msg_bits
= rctx
->msg_bits
;
241 state
.first
= rctx
->first
;
242 memcpy(state
.ctx
, rctx
->ctx
, sizeof(state
.ctx
));
243 state
.buf_count
= rctx
->buf_count
;
244 memcpy(state
.buf
, rctx
->buf
, sizeof(state
.buf
));
246 /* 'out' may not be aligned so memcpy from local variable */
247 memcpy(out
, &state
, sizeof(state
));
252 static int ccp_sha_import(struct ahash_request
*req
, const void *in
)
254 struct ccp_sha_req_ctx
*rctx
= ahash_request_ctx(req
);
255 struct ccp_sha_exp_ctx state
;
257 /* 'in' may not be aligned so memcpy to local variable */
258 memcpy(&state
, in
, sizeof(state
));
260 memset(rctx
, 0, sizeof(*rctx
));
261 rctx
->type
= state
.type
;
262 rctx
->msg_bits
= state
.msg_bits
;
263 rctx
->first
= state
.first
;
264 memcpy(rctx
->ctx
, state
.ctx
, sizeof(rctx
->ctx
));
265 rctx
->buf_count
= state
.buf_count
;
266 memcpy(rctx
->buf
, state
.buf
, sizeof(rctx
->buf
));
271 static int ccp_sha_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
272 unsigned int key_len
)
274 struct ccp_ctx
*ctx
= crypto_tfm_ctx(crypto_ahash_tfm(tfm
));
275 struct crypto_shash
*shash
= ctx
->u
.sha
.hmac_tfm
;
276 unsigned int block_size
= crypto_shash_blocksize(shash
);
277 unsigned int digest_size
= crypto_shash_digestsize(shash
);
280 /* Set to zero until complete */
281 ctx
->u
.sha
.key_len
= 0;
283 /* Clear key area to provide zero padding for keys smaller
284 * than the block size
286 memset(ctx
->u
.sha
.key
, 0, sizeof(ctx
->u
.sha
.key
));
288 if (key_len
> block_size
) {
289 /* Must hash the input key */
290 ret
= crypto_shash_tfm_digest(shash
, key
, key_len
,
295 key_len
= digest_size
;
297 memcpy(ctx
->u
.sha
.key
, key
, key_len
);
300 for (i
= 0; i
< block_size
; i
++) {
301 ctx
->u
.sha
.ipad
[i
] = ctx
->u
.sha
.key
[i
] ^ HMAC_IPAD_VALUE
;
302 ctx
->u
.sha
.opad
[i
] = ctx
->u
.sha
.key
[i
] ^ HMAC_OPAD_VALUE
;
305 sg_init_one(&ctx
->u
.sha
.opad_sg
, ctx
->u
.sha
.opad
, block_size
);
306 ctx
->u
.sha
.opad_count
= block_size
;
308 ctx
->u
.sha
.key_len
= key_len
;
313 static int ccp_sha_cra_init(struct crypto_tfm
*tfm
)
315 struct ccp_ctx
*ctx
= crypto_tfm_ctx(tfm
);
316 struct crypto_ahash
*ahash
= __crypto_ahash_cast(tfm
);
318 ctx
->complete
= ccp_sha_complete
;
319 ctx
->u
.sha
.key_len
= 0;
321 crypto_ahash_set_reqsize(ahash
, sizeof(struct ccp_sha_req_ctx
));
326 static void ccp_sha_cra_exit(struct crypto_tfm
*tfm
)
330 static int ccp_hmac_sha_cra_init(struct crypto_tfm
*tfm
)
332 struct ccp_ctx
*ctx
= crypto_tfm_ctx(tfm
);
333 struct ccp_crypto_ahash_alg
*alg
= ccp_crypto_ahash_alg(tfm
);
334 struct crypto_shash
*hmac_tfm
;
336 hmac_tfm
= crypto_alloc_shash(alg
->child_alg
, 0, 0);
337 if (IS_ERR(hmac_tfm
)) {
338 pr_warn("could not load driver %s need for HMAC support\n",
340 return PTR_ERR(hmac_tfm
);
343 ctx
->u
.sha
.hmac_tfm
= hmac_tfm
;
345 return ccp_sha_cra_init(tfm
);
348 static void ccp_hmac_sha_cra_exit(struct crypto_tfm
*tfm
)
350 struct ccp_ctx
*ctx
= crypto_tfm_ctx(tfm
);
352 if (ctx
->u
.sha
.hmac_tfm
)
353 crypto_free_shash(ctx
->u
.sha
.hmac_tfm
);
355 ccp_sha_cra_exit(tfm
);
359 unsigned int version
;
361 const char *drv_name
;
362 enum ccp_sha_type type
;
367 static struct ccp_sha_def sha_algs
[] = {
369 .version
= CCP_VERSION(3, 0),
371 .drv_name
= "sha1-ccp",
372 .type
= CCP_SHA_TYPE_1
,
373 .digest_size
= SHA1_DIGEST_SIZE
,
374 .block_size
= SHA1_BLOCK_SIZE
,
377 .version
= CCP_VERSION(3, 0),
379 .drv_name
= "sha224-ccp",
380 .type
= CCP_SHA_TYPE_224
,
381 .digest_size
= SHA224_DIGEST_SIZE
,
382 .block_size
= SHA224_BLOCK_SIZE
,
385 .version
= CCP_VERSION(3, 0),
387 .drv_name
= "sha256-ccp",
388 .type
= CCP_SHA_TYPE_256
,
389 .digest_size
= SHA256_DIGEST_SIZE
,
390 .block_size
= SHA256_BLOCK_SIZE
,
393 .version
= CCP_VERSION(5, 0),
395 .drv_name
= "sha384-ccp",
396 .type
= CCP_SHA_TYPE_384
,
397 .digest_size
= SHA384_DIGEST_SIZE
,
398 .block_size
= SHA384_BLOCK_SIZE
,
401 .version
= CCP_VERSION(5, 0),
403 .drv_name
= "sha512-ccp",
404 .type
= CCP_SHA_TYPE_512
,
405 .digest_size
= SHA512_DIGEST_SIZE
,
406 .block_size
= SHA512_BLOCK_SIZE
,
410 static int ccp_register_hmac_alg(struct list_head
*head
,
411 const struct ccp_sha_def
*def
,
412 const struct ccp_crypto_ahash_alg
*base_alg
)
414 struct ccp_crypto_ahash_alg
*ccp_alg
;
415 struct ahash_alg
*alg
;
416 struct hash_alg_common
*halg
;
417 struct crypto_alg
*base
;
420 ccp_alg
= kzalloc(sizeof(*ccp_alg
), GFP_KERNEL
);
424 /* Copy the base algorithm and only change what's necessary */
425 *ccp_alg
= *base_alg
;
426 INIT_LIST_HEAD(&ccp_alg
->entry
);
428 strscpy(ccp_alg
->child_alg
, def
->name
, CRYPTO_MAX_ALG_NAME
);
431 alg
->setkey
= ccp_sha_setkey
;
436 snprintf(base
->cra_name
, CRYPTO_MAX_ALG_NAME
, "hmac(%s)", def
->name
);
437 snprintf(base
->cra_driver_name
, CRYPTO_MAX_ALG_NAME
, "hmac-%s",
439 base
->cra_init
= ccp_hmac_sha_cra_init
;
440 base
->cra_exit
= ccp_hmac_sha_cra_exit
;
442 ret
= crypto_register_ahash(alg
);
444 pr_err("%s ahash algorithm registration error (%d)\n",
445 base
->cra_name
, ret
);
450 list_add(&ccp_alg
->entry
, head
);
455 static int ccp_register_sha_alg(struct list_head
*head
,
456 const struct ccp_sha_def
*def
)
458 struct ccp_crypto_ahash_alg
*ccp_alg
;
459 struct ahash_alg
*alg
;
460 struct hash_alg_common
*halg
;
461 struct crypto_alg
*base
;
464 ccp_alg
= kzalloc(sizeof(*ccp_alg
), GFP_KERNEL
);
468 INIT_LIST_HEAD(&ccp_alg
->entry
);
470 ccp_alg
->type
= def
->type
;
473 alg
->init
= ccp_sha_init
;
474 alg
->update
= ccp_sha_update
;
475 alg
->final
= ccp_sha_final
;
476 alg
->finup
= ccp_sha_finup
;
477 alg
->digest
= ccp_sha_digest
;
478 alg
->export
= ccp_sha_export
;
479 alg
->import
= ccp_sha_import
;
482 halg
->digestsize
= def
->digest_size
;
483 halg
->statesize
= sizeof(struct ccp_sha_exp_ctx
);
486 snprintf(base
->cra_name
, CRYPTO_MAX_ALG_NAME
, "%s", def
->name
);
487 snprintf(base
->cra_driver_name
, CRYPTO_MAX_ALG_NAME
, "%s",
489 base
->cra_flags
= CRYPTO_ALG_ASYNC
|
490 CRYPTO_ALG_ALLOCATES_MEMORY
|
491 CRYPTO_ALG_KERN_DRIVER_ONLY
|
492 CRYPTO_ALG_NEED_FALLBACK
;
493 base
->cra_blocksize
= def
->block_size
;
494 base
->cra_ctxsize
= sizeof(struct ccp_ctx
);
495 base
->cra_priority
= CCP_CRA_PRIORITY
;
496 base
->cra_init
= ccp_sha_cra_init
;
497 base
->cra_exit
= ccp_sha_cra_exit
;
498 base
->cra_module
= THIS_MODULE
;
500 ret
= crypto_register_ahash(alg
);
502 pr_err("%s ahash algorithm registration error (%d)\n",
503 base
->cra_name
, ret
);
508 list_add(&ccp_alg
->entry
, head
);
510 ret
= ccp_register_hmac_alg(head
, def
, ccp_alg
);
515 int ccp_register_sha_algs(struct list_head
*head
)
518 unsigned int ccpversion
= ccp_version();
520 for (i
= 0; i
< ARRAY_SIZE(sha_algs
); i
++) {
521 if (sha_algs
[i
].version
> ccpversion
)
523 ret
= ccp_register_sha_alg(head
, &sha_algs
[i
]);