1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Asynchronous Cryptographic Hash operations.
5 * This is the asynchronous version of hash.c with notification of
6 * completion via a callback.
8 * Copyright (c) 2008 Loc Ho <lho@amcc.com>
11 #include <crypto/scatterwalk.h>
12 #include <linux/cryptouser.h>
13 #include <linux/err.h>
14 #include <linux/kernel.h>
15 #include <linux/module.h>
16 #include <linux/sched.h>
17 #include <linux/slab.h>
18 #include <linux/seq_file.h>
19 #include <linux/string.h>
20 #include <net/netlink.h>
24 static const struct crypto_type crypto_ahash_type
;
26 struct ahash_request_priv
{
27 crypto_completion_t complete
;
31 void *ubuf
[] CRYPTO_MINALIGN_ATTR
;
34 static int hash_walk_next(struct crypto_hash_walk
*walk
)
36 unsigned int alignmask
= walk
->alignmask
;
37 unsigned int offset
= walk
->offset
;
38 unsigned int nbytes
= min(walk
->entrylen
,
39 ((unsigned int)(PAGE_SIZE
)) - offset
);
41 walk
->data
= kmap_local_page(walk
->pg
);
44 if (offset
& alignmask
) {
45 unsigned int unaligned
= alignmask
+ 1 - (offset
& alignmask
);
47 if (nbytes
> unaligned
)
51 walk
->entrylen
-= nbytes
;
55 static int hash_walk_new_entry(struct crypto_hash_walk
*walk
)
57 struct scatterlist
*sg
;
60 walk
->offset
= sg
->offset
;
61 walk
->pg
= sg_page(walk
->sg
) + (walk
->offset
>> PAGE_SHIFT
);
62 walk
->offset
= offset_in_page(walk
->offset
);
63 walk
->entrylen
= sg
->length
;
65 if (walk
->entrylen
> walk
->total
)
66 walk
->entrylen
= walk
->total
;
67 walk
->total
-= walk
->entrylen
;
69 return hash_walk_next(walk
);
72 int crypto_hash_walk_done(struct crypto_hash_walk
*walk
, int err
)
74 unsigned int alignmask
= walk
->alignmask
;
76 walk
->data
-= walk
->offset
;
78 if (walk
->entrylen
&& (walk
->offset
& alignmask
) && !err
) {
81 walk
->offset
= ALIGN(walk
->offset
, alignmask
+ 1);
82 nbytes
= min(walk
->entrylen
,
83 (unsigned int)(PAGE_SIZE
- walk
->offset
));
85 walk
->entrylen
-= nbytes
;
86 walk
->data
+= walk
->offset
;
91 kunmap_local(walk
->data
);
92 crypto_yield(walk
->flags
);
100 return hash_walk_next(walk
);
106 walk
->sg
= sg_next(walk
->sg
);
108 return hash_walk_new_entry(walk
);
110 EXPORT_SYMBOL_GPL(crypto_hash_walk_done
);
112 int crypto_hash_walk_first(struct ahash_request
*req
,
113 struct crypto_hash_walk
*walk
)
115 walk
->total
= req
->nbytes
;
122 walk
->alignmask
= crypto_ahash_alignmask(crypto_ahash_reqtfm(req
));
124 walk
->flags
= req
->base
.flags
;
126 return hash_walk_new_entry(walk
);
128 EXPORT_SYMBOL_GPL(crypto_hash_walk_first
);
130 static int ahash_setkey_unaligned(struct crypto_ahash
*tfm
, const u8
*key
,
133 unsigned long alignmask
= crypto_ahash_alignmask(tfm
);
135 u8
*buffer
, *alignbuffer
;
136 unsigned long absize
;
138 absize
= keylen
+ alignmask
;
139 buffer
= kmalloc(absize
, GFP_KERNEL
);
143 alignbuffer
= (u8
*)ALIGN((unsigned long)buffer
, alignmask
+ 1);
144 memcpy(alignbuffer
, key
, keylen
);
145 ret
= tfm
->setkey(tfm
, alignbuffer
, keylen
);
146 kfree_sensitive(buffer
);
150 static int ahash_nosetkey(struct crypto_ahash
*tfm
, const u8
*key
,
156 static void ahash_set_needkey(struct crypto_ahash
*tfm
)
158 const struct hash_alg_common
*alg
= crypto_hash_alg_common(tfm
);
160 if (tfm
->setkey
!= ahash_nosetkey
&&
161 !(alg
->base
.cra_flags
& CRYPTO_ALG_OPTIONAL_KEY
))
162 crypto_ahash_set_flags(tfm
, CRYPTO_TFM_NEED_KEY
);
165 int crypto_ahash_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
168 unsigned long alignmask
= crypto_ahash_alignmask(tfm
);
171 if ((unsigned long)key
& alignmask
)
172 err
= ahash_setkey_unaligned(tfm
, key
, keylen
);
174 err
= tfm
->setkey(tfm
, key
, keylen
);
177 ahash_set_needkey(tfm
);
181 crypto_ahash_clear_flags(tfm
, CRYPTO_TFM_NEED_KEY
);
184 EXPORT_SYMBOL_GPL(crypto_ahash_setkey
);
186 static int ahash_save_req(struct ahash_request
*req
, crypto_completion_t cplt
,
189 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
190 unsigned long alignmask
= crypto_ahash_alignmask(tfm
);
191 unsigned int ds
= crypto_ahash_digestsize(tfm
);
192 struct ahash_request
*subreq
;
193 unsigned int subreq_size
;
194 unsigned int reqsize
;
199 subreq_size
= sizeof(*subreq
);
200 reqsize
= crypto_ahash_reqsize(tfm
);
201 reqsize
= ALIGN(reqsize
, crypto_tfm_ctx_alignment());
202 subreq_size
+= reqsize
;
204 subreq_size
+= alignmask
& ~(crypto_tfm_ctx_alignment() - 1);
206 flags
= ahash_request_flags(req
);
207 gfp
= (flags
& CRYPTO_TFM_REQ_MAY_SLEEP
) ? GFP_KERNEL
: GFP_ATOMIC
;
208 subreq
= kmalloc(subreq_size
, gfp
);
212 ahash_request_set_tfm(subreq
, tfm
);
213 ahash_request_set_callback(subreq
, flags
, cplt
, req
);
215 result
= (u8
*)(subreq
+ 1) + reqsize
;
216 result
= PTR_ALIGN(result
, alignmask
+ 1);
218 ahash_request_set_crypt(subreq
, req
->src
, result
, req
->nbytes
);
223 state
= kmalloc(crypto_ahash_statesize(tfm
), gfp
);
229 crypto_ahash_export(req
, state
);
230 crypto_ahash_import(subreq
, state
);
231 kfree_sensitive(state
);
239 static void ahash_restore_req(struct ahash_request
*req
, int err
)
241 struct ahash_request
*subreq
= req
->priv
;
244 memcpy(req
->result
, subreq
->result
,
245 crypto_ahash_digestsize(crypto_ahash_reqtfm(req
)));
249 kfree_sensitive(subreq
);
252 static void ahash_op_unaligned_done(void *data
, int err
)
254 struct ahash_request
*areq
= data
;
256 if (err
== -EINPROGRESS
)
259 /* First copy req->result into req->priv.result */
260 ahash_restore_req(areq
, err
);
263 /* Complete the ORIGINAL request. */
264 ahash_request_complete(areq
, err
);
267 static int ahash_op_unaligned(struct ahash_request
*req
,
268 int (*op
)(struct ahash_request
*),
273 err
= ahash_save_req(req
, ahash_op_unaligned_done
, has_state
);
278 if (err
== -EINPROGRESS
|| err
== -EBUSY
)
281 ahash_restore_req(req
, err
);
286 static int crypto_ahash_op(struct ahash_request
*req
,
287 int (*op
)(struct ahash_request
*),
290 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
291 unsigned long alignmask
= crypto_ahash_alignmask(tfm
);
294 if ((unsigned long)req
->result
& alignmask
)
295 err
= ahash_op_unaligned(req
, op
, has_state
);
299 return crypto_hash_errstat(crypto_hash_alg_common(tfm
), err
);
302 int crypto_ahash_final(struct ahash_request
*req
)
304 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
305 struct hash_alg_common
*alg
= crypto_hash_alg_common(tfm
);
307 if (IS_ENABLED(CONFIG_CRYPTO_STATS
))
308 atomic64_inc(&hash_get_stat(alg
)->hash_cnt
);
310 return crypto_ahash_op(req
, tfm
->final
, true);
312 EXPORT_SYMBOL_GPL(crypto_ahash_final
);
314 int crypto_ahash_finup(struct ahash_request
*req
)
316 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
317 struct hash_alg_common
*alg
= crypto_hash_alg_common(tfm
);
319 if (IS_ENABLED(CONFIG_CRYPTO_STATS
)) {
320 struct crypto_istat_hash
*istat
= hash_get_stat(alg
);
322 atomic64_inc(&istat
->hash_cnt
);
323 atomic64_add(req
->nbytes
, &istat
->hash_tlen
);
326 return crypto_ahash_op(req
, tfm
->finup
, true);
328 EXPORT_SYMBOL_GPL(crypto_ahash_finup
);
330 int crypto_ahash_digest(struct ahash_request
*req
)
332 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
333 struct hash_alg_common
*alg
= crypto_hash_alg_common(tfm
);
335 if (IS_ENABLED(CONFIG_CRYPTO_STATS
)) {
336 struct crypto_istat_hash
*istat
= hash_get_stat(alg
);
338 atomic64_inc(&istat
->hash_cnt
);
339 atomic64_add(req
->nbytes
, &istat
->hash_tlen
);
342 if (crypto_ahash_get_flags(tfm
) & CRYPTO_TFM_NEED_KEY
)
343 return crypto_hash_errstat(alg
, -ENOKEY
);
345 return crypto_ahash_op(req
, tfm
->digest
, false);
347 EXPORT_SYMBOL_GPL(crypto_ahash_digest
);
349 static void ahash_def_finup_done2(void *data
, int err
)
351 struct ahash_request
*areq
= data
;
353 if (err
== -EINPROGRESS
)
356 ahash_restore_req(areq
, err
);
358 ahash_request_complete(areq
, err
);
361 static int ahash_def_finup_finish1(struct ahash_request
*req
, int err
)
363 struct ahash_request
*subreq
= req
->priv
;
368 subreq
->base
.complete
= ahash_def_finup_done2
;
370 err
= crypto_ahash_reqtfm(req
)->final(subreq
);
371 if (err
== -EINPROGRESS
|| err
== -EBUSY
)
375 ahash_restore_req(req
, err
);
379 static void ahash_def_finup_done1(void *data
, int err
)
381 struct ahash_request
*areq
= data
;
382 struct ahash_request
*subreq
;
384 if (err
== -EINPROGRESS
)
388 subreq
->base
.flags
&= CRYPTO_TFM_REQ_MAY_BACKLOG
;
390 err
= ahash_def_finup_finish1(areq
, err
);
391 if (err
== -EINPROGRESS
|| err
== -EBUSY
)
395 ahash_request_complete(areq
, err
);
398 static int ahash_def_finup(struct ahash_request
*req
)
400 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
403 err
= ahash_save_req(req
, ahash_def_finup_done1
, true);
407 err
= tfm
->update(req
->priv
);
408 if (err
== -EINPROGRESS
|| err
== -EBUSY
)
411 return ahash_def_finup_finish1(req
, err
);
414 static void crypto_ahash_exit_tfm(struct crypto_tfm
*tfm
)
416 struct crypto_ahash
*hash
= __crypto_ahash_cast(tfm
);
417 struct ahash_alg
*alg
= crypto_ahash_alg(hash
);
422 static int crypto_ahash_init_tfm(struct crypto_tfm
*tfm
)
424 struct crypto_ahash
*hash
= __crypto_ahash_cast(tfm
);
425 struct ahash_alg
*alg
= crypto_ahash_alg(hash
);
427 hash
->setkey
= ahash_nosetkey
;
429 crypto_ahash_set_statesize(hash
, alg
->halg
.statesize
);
431 if (tfm
->__crt_alg
->cra_type
!= &crypto_ahash_type
)
432 return crypto_init_shash_ops_async(tfm
);
434 hash
->init
= alg
->init
;
435 hash
->update
= alg
->update
;
436 hash
->final
= alg
->final
;
437 hash
->finup
= alg
->finup
?: ahash_def_finup
;
438 hash
->digest
= alg
->digest
;
439 hash
->export
= alg
->export
;
440 hash
->import
= alg
->import
;
443 hash
->setkey
= alg
->setkey
;
444 ahash_set_needkey(hash
);
448 tfm
->exit
= crypto_ahash_exit_tfm
;
450 return alg
->init_tfm
? alg
->init_tfm(hash
) : 0;
453 static unsigned int crypto_ahash_extsize(struct crypto_alg
*alg
)
455 if (alg
->cra_type
!= &crypto_ahash_type
)
456 return sizeof(struct crypto_shash
*);
458 return crypto_alg_extsize(alg
);
461 static void crypto_ahash_free_instance(struct crypto_instance
*inst
)
463 struct ahash_instance
*ahash
= ahash_instance(inst
);
468 static int __maybe_unused
crypto_ahash_report(
469 struct sk_buff
*skb
, struct crypto_alg
*alg
)
471 struct crypto_report_hash rhash
;
473 memset(&rhash
, 0, sizeof(rhash
));
475 strscpy(rhash
.type
, "ahash", sizeof(rhash
.type
));
477 rhash
.blocksize
= alg
->cra_blocksize
;
478 rhash
.digestsize
= __crypto_hash_alg_common(alg
)->digestsize
;
480 return nla_put(skb
, CRYPTOCFGA_REPORT_HASH
, sizeof(rhash
), &rhash
);
483 static void crypto_ahash_show(struct seq_file
*m
, struct crypto_alg
*alg
)
485 static void crypto_ahash_show(struct seq_file
*m
, struct crypto_alg
*alg
)
487 seq_printf(m
, "type : ahash\n");
488 seq_printf(m
, "async : %s\n", alg
->cra_flags
& CRYPTO_ALG_ASYNC
?
490 seq_printf(m
, "blocksize : %u\n", alg
->cra_blocksize
);
491 seq_printf(m
, "digestsize : %u\n",
492 __crypto_hash_alg_common(alg
)->digestsize
);
495 static int __maybe_unused
crypto_ahash_report_stat(
496 struct sk_buff
*skb
, struct crypto_alg
*alg
)
498 return crypto_hash_report_stat(skb
, alg
, "ahash");
501 static const struct crypto_type crypto_ahash_type
= {
502 .extsize
= crypto_ahash_extsize
,
503 .init_tfm
= crypto_ahash_init_tfm
,
504 .free
= crypto_ahash_free_instance
,
505 #ifdef CONFIG_PROC_FS
506 .show
= crypto_ahash_show
,
508 #if IS_ENABLED(CONFIG_CRYPTO_USER)
509 .report
= crypto_ahash_report
,
511 #ifdef CONFIG_CRYPTO_STATS
512 .report_stat
= crypto_ahash_report_stat
,
514 .maskclear
= ~CRYPTO_ALG_TYPE_MASK
,
515 .maskset
= CRYPTO_ALG_TYPE_AHASH_MASK
,
516 .type
= CRYPTO_ALG_TYPE_AHASH
,
517 .tfmsize
= offsetof(struct crypto_ahash
, base
),
520 int crypto_grab_ahash(struct crypto_ahash_spawn
*spawn
,
521 struct crypto_instance
*inst
,
522 const char *name
, u32 type
, u32 mask
)
524 spawn
->base
.frontend
= &crypto_ahash_type
;
525 return crypto_grab_spawn(&spawn
->base
, inst
, name
, type
, mask
);
527 EXPORT_SYMBOL_GPL(crypto_grab_ahash
);
529 struct crypto_ahash
*crypto_alloc_ahash(const char *alg_name
, u32 type
,
532 return crypto_alloc_tfm(alg_name
, &crypto_ahash_type
, type
, mask
);
534 EXPORT_SYMBOL_GPL(crypto_alloc_ahash
);
536 int crypto_has_ahash(const char *alg_name
, u32 type
, u32 mask
)
538 return crypto_type_has_alg(alg_name
, &crypto_ahash_type
, type
, mask
);
540 EXPORT_SYMBOL_GPL(crypto_has_ahash
);
542 struct crypto_ahash
*crypto_clone_ahash(struct crypto_ahash
*hash
)
544 struct hash_alg_common
*halg
= crypto_hash_alg_common(hash
);
545 struct crypto_tfm
*tfm
= crypto_ahash_tfm(hash
);
546 struct crypto_ahash
*nhash
;
547 struct ahash_alg
*alg
;
550 if (!crypto_hash_alg_has_setkey(halg
)) {
551 tfm
= crypto_tfm_get(tfm
);
553 return ERR_CAST(tfm
);
558 nhash
= crypto_clone_tfm(&crypto_ahash_type
, tfm
);
563 nhash
->init
= hash
->init
;
564 nhash
->update
= hash
->update
;
565 nhash
->final
= hash
->final
;
566 nhash
->finup
= hash
->finup
;
567 nhash
->digest
= hash
->digest
;
568 nhash
->export
= hash
->export
;
569 nhash
->import
= hash
->import
;
570 nhash
->setkey
= hash
->setkey
;
571 nhash
->reqsize
= hash
->reqsize
;
572 nhash
->statesize
= hash
->statesize
;
574 if (tfm
->__crt_alg
->cra_type
!= &crypto_ahash_type
)
575 return crypto_clone_shash_ops_async(nhash
, hash
);
578 alg
= crypto_ahash_alg(hash
);
582 err
= alg
->clone_tfm(nhash
, hash
);
589 crypto_free_ahash(nhash
);
592 EXPORT_SYMBOL_GPL(crypto_clone_ahash
);
594 static int ahash_prepare_alg(struct ahash_alg
*alg
)
596 struct crypto_alg
*base
= &alg
->halg
.base
;
599 if (alg
->halg
.statesize
== 0)
602 err
= hash_prepare_alg(&alg
->halg
);
606 base
->cra_type
= &crypto_ahash_type
;
607 base
->cra_flags
|= CRYPTO_ALG_TYPE_AHASH
;
612 int crypto_register_ahash(struct ahash_alg
*alg
)
614 struct crypto_alg
*base
= &alg
->halg
.base
;
617 err
= ahash_prepare_alg(alg
);
621 return crypto_register_alg(base
);
623 EXPORT_SYMBOL_GPL(crypto_register_ahash
);
625 void crypto_unregister_ahash(struct ahash_alg
*alg
)
627 crypto_unregister_alg(&alg
->halg
.base
);
629 EXPORT_SYMBOL_GPL(crypto_unregister_ahash
);
631 int crypto_register_ahashes(struct ahash_alg
*algs
, int count
)
635 for (i
= 0; i
< count
; i
++) {
636 ret
= crypto_register_ahash(&algs
[i
]);
644 for (--i
; i
>= 0; --i
)
645 crypto_unregister_ahash(&algs
[i
]);
649 EXPORT_SYMBOL_GPL(crypto_register_ahashes
);
651 void crypto_unregister_ahashes(struct ahash_alg
*algs
, int count
)
655 for (i
= count
- 1; i
>= 0; --i
)
656 crypto_unregister_ahash(&algs
[i
]);
658 EXPORT_SYMBOL_GPL(crypto_unregister_ahashes
);
660 int ahash_register_instance(struct crypto_template
*tmpl
,
661 struct ahash_instance
*inst
)
665 if (WARN_ON(!inst
->free
))
668 err
= ahash_prepare_alg(&inst
->alg
);
672 return crypto_register_instance(tmpl
, ahash_crypto_instance(inst
));
674 EXPORT_SYMBOL_GPL(ahash_register_instance
);
676 bool crypto_hash_alg_has_setkey(struct hash_alg_common
*halg
)
678 struct crypto_alg
*alg
= &halg
->base
;
680 if (alg
->cra_type
!= &crypto_ahash_type
)
681 return crypto_shash_alg_has_setkey(__crypto_shash_alg(alg
));
683 return __crypto_ahash_alg(alg
)->setkey
!= NULL
;
685 EXPORT_SYMBOL_GPL(crypto_hash_alg_has_setkey
);
687 MODULE_LICENSE("GPL");
688 MODULE_DESCRIPTION("Asynchronous cryptographic hash type");