1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Software async crypto daemon.
5 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
7 * Added AEAD support to cryptd.
8 * Authors: Tadeusz Struk (tadeusz.struk@intel.com)
9 * Adrian Hoban <adrian.hoban@intel.com>
10 * Gabriele Paoloni <gabriele.paoloni@intel.com>
11 * Aidan O'Mahony (aidan.o.mahony@intel.com)
12 * Copyright (c) 2010, Intel Corporation.
15 #include <crypto/internal/hash.h>
16 #include <crypto/internal/aead.h>
17 #include <crypto/internal/skcipher.h>
18 #include <crypto/cryptd.h>
19 #include <crypto/crypto_wq.h>
20 #include <linux/atomic.h>
21 #include <linux/err.h>
22 #include <linux/init.h>
23 #include <linux/kernel.h>
24 #include <linux/list.h>
25 #include <linux/module.h>
26 #include <linux/scatterlist.h>
27 #include <linux/sched.h>
28 #include <linux/slab.h>
30 static unsigned int cryptd_max_cpu_qlen
= 1000;
31 module_param(cryptd_max_cpu_qlen
, uint
, 0);
32 MODULE_PARM_DESC(cryptd_max_cpu_qlen
, "Set cryptd Max queue depth");
34 struct cryptd_cpu_queue
{
35 struct crypto_queue queue
;
36 struct work_struct work
;
40 struct cryptd_cpu_queue __percpu
*cpu_queue
;
43 struct cryptd_instance_ctx
{
44 struct crypto_spawn spawn
;
45 struct cryptd_queue
*queue
;
48 struct skcipherd_instance_ctx
{
49 struct crypto_skcipher_spawn spawn
;
50 struct cryptd_queue
*queue
;
53 struct hashd_instance_ctx
{
54 struct crypto_shash_spawn spawn
;
55 struct cryptd_queue
*queue
;
58 struct aead_instance_ctx
{
59 struct crypto_aead_spawn aead_spawn
;
60 struct cryptd_queue
*queue
;
63 struct cryptd_skcipher_ctx
{
65 struct crypto_sync_skcipher
*child
;
68 struct cryptd_skcipher_request_ctx
{
69 crypto_completion_t complete
;
72 struct cryptd_hash_ctx
{
74 struct crypto_shash
*child
;
77 struct cryptd_hash_request_ctx
{
78 crypto_completion_t complete
;
79 struct shash_desc desc
;
82 struct cryptd_aead_ctx
{
84 struct crypto_aead
*child
;
87 struct cryptd_aead_request_ctx
{
88 crypto_completion_t complete
;
91 static void cryptd_queue_worker(struct work_struct
*work
);
93 static int cryptd_init_queue(struct cryptd_queue
*queue
,
94 unsigned int max_cpu_qlen
)
97 struct cryptd_cpu_queue
*cpu_queue
;
99 queue
->cpu_queue
= alloc_percpu(struct cryptd_cpu_queue
);
100 if (!queue
->cpu_queue
)
102 for_each_possible_cpu(cpu
) {
103 cpu_queue
= per_cpu_ptr(queue
->cpu_queue
, cpu
);
104 crypto_init_queue(&cpu_queue
->queue
, max_cpu_qlen
);
105 INIT_WORK(&cpu_queue
->work
, cryptd_queue_worker
);
107 pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen
);
111 static void cryptd_fini_queue(struct cryptd_queue
*queue
)
114 struct cryptd_cpu_queue
*cpu_queue
;
116 for_each_possible_cpu(cpu
) {
117 cpu_queue
= per_cpu_ptr(queue
->cpu_queue
, cpu
);
118 BUG_ON(cpu_queue
->queue
.qlen
);
120 free_percpu(queue
->cpu_queue
);
123 static int cryptd_enqueue_request(struct cryptd_queue
*queue
,
124 struct crypto_async_request
*request
)
127 struct cryptd_cpu_queue
*cpu_queue
;
131 cpu_queue
= this_cpu_ptr(queue
->cpu_queue
);
132 err
= crypto_enqueue_request(&cpu_queue
->queue
, request
);
134 refcnt
= crypto_tfm_ctx(request
->tfm
);
139 queue_work_on(cpu
, kcrypto_wq
, &cpu_queue
->work
);
141 if (!atomic_read(refcnt
))
152 /* Called in workqueue context, do one real cryption work (via
153 * req->complete) and reschedule itself if there are more work to
155 static void cryptd_queue_worker(struct work_struct
*work
)
157 struct cryptd_cpu_queue
*cpu_queue
;
158 struct crypto_async_request
*req
, *backlog
;
160 cpu_queue
= container_of(work
, struct cryptd_cpu_queue
, work
);
162 * Only handle one request at a time to avoid hogging crypto workqueue.
163 * preempt_disable/enable is used to prevent being preempted by
164 * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent
165 * cryptd_enqueue_request() being accessed from software interrupts.
169 backlog
= crypto_get_backlog(&cpu_queue
->queue
);
170 req
= crypto_dequeue_request(&cpu_queue
->queue
);
178 backlog
->complete(backlog
, -EINPROGRESS
);
179 req
->complete(req
, 0);
181 if (cpu_queue
->queue
.qlen
)
182 queue_work(kcrypto_wq
, &cpu_queue
->work
);
185 static inline struct cryptd_queue
*cryptd_get_queue(struct crypto_tfm
*tfm
)
187 struct crypto_instance
*inst
= crypto_tfm_alg_instance(tfm
);
188 struct cryptd_instance_ctx
*ictx
= crypto_instance_ctx(inst
);
192 static inline void cryptd_check_internal(struct rtattr
**tb
, u32
*type
,
195 struct crypto_attr_type
*algt
;
197 algt
= crypto_get_attr_type(tb
);
201 *type
|= algt
->type
& CRYPTO_ALG_INTERNAL
;
202 *mask
|= algt
->mask
& CRYPTO_ALG_INTERNAL
;
205 static int cryptd_init_instance(struct crypto_instance
*inst
,
206 struct crypto_alg
*alg
)
208 if (snprintf(inst
->alg
.cra_driver_name
, CRYPTO_MAX_ALG_NAME
,
210 alg
->cra_driver_name
) >= CRYPTO_MAX_ALG_NAME
)
211 return -ENAMETOOLONG
;
213 memcpy(inst
->alg
.cra_name
, alg
->cra_name
, CRYPTO_MAX_ALG_NAME
);
215 inst
->alg
.cra_priority
= alg
->cra_priority
+ 50;
216 inst
->alg
.cra_blocksize
= alg
->cra_blocksize
;
217 inst
->alg
.cra_alignmask
= alg
->cra_alignmask
;
222 static void *cryptd_alloc_instance(struct crypto_alg
*alg
, unsigned int head
,
226 struct crypto_instance
*inst
;
229 p
= kzalloc(head
+ sizeof(*inst
) + tail
, GFP_KERNEL
);
231 return ERR_PTR(-ENOMEM
);
233 inst
= (void *)(p
+ head
);
235 err
= cryptd_init_instance(inst
, alg
);
248 static int cryptd_skcipher_setkey(struct crypto_skcipher
*parent
,
249 const u8
*key
, unsigned int keylen
)
251 struct cryptd_skcipher_ctx
*ctx
= crypto_skcipher_ctx(parent
);
252 struct crypto_sync_skcipher
*child
= ctx
->child
;
255 crypto_sync_skcipher_clear_flags(child
, CRYPTO_TFM_REQ_MASK
);
256 crypto_sync_skcipher_set_flags(child
,
257 crypto_skcipher_get_flags(parent
) &
258 CRYPTO_TFM_REQ_MASK
);
259 err
= crypto_sync_skcipher_setkey(child
, key
, keylen
);
260 crypto_skcipher_set_flags(parent
,
261 crypto_sync_skcipher_get_flags(child
) &
262 CRYPTO_TFM_RES_MASK
);
266 static void cryptd_skcipher_complete(struct skcipher_request
*req
, int err
)
268 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
269 struct cryptd_skcipher_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
270 struct cryptd_skcipher_request_ctx
*rctx
= skcipher_request_ctx(req
);
271 int refcnt
= atomic_read(&ctx
->refcnt
);
274 rctx
->complete(&req
->base
, err
);
277 if (err
!= -EINPROGRESS
&& refcnt
&& atomic_dec_and_test(&ctx
->refcnt
))
278 crypto_free_skcipher(tfm
);
281 static void cryptd_skcipher_encrypt(struct crypto_async_request
*base
,
284 struct skcipher_request
*req
= skcipher_request_cast(base
);
285 struct cryptd_skcipher_request_ctx
*rctx
= skcipher_request_ctx(req
);
286 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
287 struct cryptd_skcipher_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
288 struct crypto_sync_skcipher
*child
= ctx
->child
;
289 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq
, child
);
291 if (unlikely(err
== -EINPROGRESS
))
294 skcipher_request_set_sync_tfm(subreq
, child
);
295 skcipher_request_set_callback(subreq
, CRYPTO_TFM_REQ_MAY_SLEEP
,
297 skcipher_request_set_crypt(subreq
, req
->src
, req
->dst
, req
->cryptlen
,
300 err
= crypto_skcipher_encrypt(subreq
);
301 skcipher_request_zero(subreq
);
303 req
->base
.complete
= rctx
->complete
;
306 cryptd_skcipher_complete(req
, err
);
309 static void cryptd_skcipher_decrypt(struct crypto_async_request
*base
,
312 struct skcipher_request
*req
= skcipher_request_cast(base
);
313 struct cryptd_skcipher_request_ctx
*rctx
= skcipher_request_ctx(req
);
314 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
315 struct cryptd_skcipher_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
316 struct crypto_sync_skcipher
*child
= ctx
->child
;
317 SYNC_SKCIPHER_REQUEST_ON_STACK(subreq
, child
);
319 if (unlikely(err
== -EINPROGRESS
))
322 skcipher_request_set_sync_tfm(subreq
, child
);
323 skcipher_request_set_callback(subreq
, CRYPTO_TFM_REQ_MAY_SLEEP
,
325 skcipher_request_set_crypt(subreq
, req
->src
, req
->dst
, req
->cryptlen
,
328 err
= crypto_skcipher_decrypt(subreq
);
329 skcipher_request_zero(subreq
);
331 req
->base
.complete
= rctx
->complete
;
334 cryptd_skcipher_complete(req
, err
);
337 static int cryptd_skcipher_enqueue(struct skcipher_request
*req
,
338 crypto_completion_t
compl)
340 struct cryptd_skcipher_request_ctx
*rctx
= skcipher_request_ctx(req
);
341 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
342 struct cryptd_queue
*queue
;
344 queue
= cryptd_get_queue(crypto_skcipher_tfm(tfm
));
345 rctx
->complete
= req
->base
.complete
;
346 req
->base
.complete
= compl;
348 return cryptd_enqueue_request(queue
, &req
->base
);
351 static int cryptd_skcipher_encrypt_enqueue(struct skcipher_request
*req
)
353 return cryptd_skcipher_enqueue(req
, cryptd_skcipher_encrypt
);
356 static int cryptd_skcipher_decrypt_enqueue(struct skcipher_request
*req
)
358 return cryptd_skcipher_enqueue(req
, cryptd_skcipher_decrypt
);
361 static int cryptd_skcipher_init_tfm(struct crypto_skcipher
*tfm
)
363 struct skcipher_instance
*inst
= skcipher_alg_instance(tfm
);
364 struct skcipherd_instance_ctx
*ictx
= skcipher_instance_ctx(inst
);
365 struct crypto_skcipher_spawn
*spawn
= &ictx
->spawn
;
366 struct cryptd_skcipher_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
367 struct crypto_skcipher
*cipher
;
369 cipher
= crypto_spawn_skcipher(spawn
);
371 return PTR_ERR(cipher
);
373 ctx
->child
= (struct crypto_sync_skcipher
*)cipher
;
374 crypto_skcipher_set_reqsize(
375 tfm
, sizeof(struct cryptd_skcipher_request_ctx
));
379 static void cryptd_skcipher_exit_tfm(struct crypto_skcipher
*tfm
)
381 struct cryptd_skcipher_ctx
*ctx
= crypto_skcipher_ctx(tfm
);
383 crypto_free_sync_skcipher(ctx
->child
);
386 static void cryptd_skcipher_free(struct skcipher_instance
*inst
)
388 struct skcipherd_instance_ctx
*ctx
= skcipher_instance_ctx(inst
);
390 crypto_drop_skcipher(&ctx
->spawn
);
393 static int cryptd_create_skcipher(struct crypto_template
*tmpl
,
395 struct cryptd_queue
*queue
)
397 struct skcipherd_instance_ctx
*ctx
;
398 struct skcipher_instance
*inst
;
399 struct skcipher_alg
*alg
;
406 mask
= CRYPTO_ALG_ASYNC
;
408 cryptd_check_internal(tb
, &type
, &mask
);
410 name
= crypto_attr_alg_name(tb
[1]);
412 return PTR_ERR(name
);
414 inst
= kzalloc(sizeof(*inst
) + sizeof(*ctx
), GFP_KERNEL
);
418 ctx
= skcipher_instance_ctx(inst
);
421 crypto_set_skcipher_spawn(&ctx
->spawn
, skcipher_crypto_instance(inst
));
422 err
= crypto_grab_skcipher(&ctx
->spawn
, name
, type
, mask
);
426 alg
= crypto_spawn_skcipher_alg(&ctx
->spawn
);
427 err
= cryptd_init_instance(skcipher_crypto_instance(inst
), &alg
->base
);
429 goto out_drop_skcipher
;
431 inst
->alg
.base
.cra_flags
= CRYPTO_ALG_ASYNC
|
432 (alg
->base
.cra_flags
& CRYPTO_ALG_INTERNAL
);
434 inst
->alg
.ivsize
= crypto_skcipher_alg_ivsize(alg
);
435 inst
->alg
.chunksize
= crypto_skcipher_alg_chunksize(alg
);
436 inst
->alg
.min_keysize
= crypto_skcipher_alg_min_keysize(alg
);
437 inst
->alg
.max_keysize
= crypto_skcipher_alg_max_keysize(alg
);
439 inst
->alg
.base
.cra_ctxsize
= sizeof(struct cryptd_skcipher_ctx
);
441 inst
->alg
.init
= cryptd_skcipher_init_tfm
;
442 inst
->alg
.exit
= cryptd_skcipher_exit_tfm
;
444 inst
->alg
.setkey
= cryptd_skcipher_setkey
;
445 inst
->alg
.encrypt
= cryptd_skcipher_encrypt_enqueue
;
446 inst
->alg
.decrypt
= cryptd_skcipher_decrypt_enqueue
;
448 inst
->free
= cryptd_skcipher_free
;
450 err
= skcipher_register_instance(tmpl
, inst
);
453 crypto_drop_skcipher(&ctx
->spawn
);
460 static int cryptd_hash_init_tfm(struct crypto_tfm
*tfm
)
462 struct crypto_instance
*inst
= crypto_tfm_alg_instance(tfm
);
463 struct hashd_instance_ctx
*ictx
= crypto_instance_ctx(inst
);
464 struct crypto_shash_spawn
*spawn
= &ictx
->spawn
;
465 struct cryptd_hash_ctx
*ctx
= crypto_tfm_ctx(tfm
);
466 struct crypto_shash
*hash
;
468 hash
= crypto_spawn_shash(spawn
);
470 return PTR_ERR(hash
);
473 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
474 sizeof(struct cryptd_hash_request_ctx
) +
475 crypto_shash_descsize(hash
));
479 static void cryptd_hash_exit_tfm(struct crypto_tfm
*tfm
)
481 struct cryptd_hash_ctx
*ctx
= crypto_tfm_ctx(tfm
);
483 crypto_free_shash(ctx
->child
);
486 static int cryptd_hash_setkey(struct crypto_ahash
*parent
,
487 const u8
*key
, unsigned int keylen
)
489 struct cryptd_hash_ctx
*ctx
= crypto_ahash_ctx(parent
);
490 struct crypto_shash
*child
= ctx
->child
;
493 crypto_shash_clear_flags(child
, CRYPTO_TFM_REQ_MASK
);
494 crypto_shash_set_flags(child
, crypto_ahash_get_flags(parent
) &
495 CRYPTO_TFM_REQ_MASK
);
496 err
= crypto_shash_setkey(child
, key
, keylen
);
497 crypto_ahash_set_flags(parent
, crypto_shash_get_flags(child
) &
498 CRYPTO_TFM_RES_MASK
);
502 static int cryptd_hash_enqueue(struct ahash_request
*req
,
503 crypto_completion_t
compl)
505 struct cryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
506 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
507 struct cryptd_queue
*queue
=
508 cryptd_get_queue(crypto_ahash_tfm(tfm
));
510 rctx
->complete
= req
->base
.complete
;
511 req
->base
.complete
= compl;
513 return cryptd_enqueue_request(queue
, &req
->base
);
516 static void cryptd_hash_complete(struct ahash_request
*req
, int err
)
518 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
519 struct cryptd_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
520 struct cryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
521 int refcnt
= atomic_read(&ctx
->refcnt
);
524 rctx
->complete(&req
->base
, err
);
527 if (err
!= -EINPROGRESS
&& refcnt
&& atomic_dec_and_test(&ctx
->refcnt
))
528 crypto_free_ahash(tfm
);
531 static void cryptd_hash_init(struct crypto_async_request
*req_async
, int err
)
533 struct cryptd_hash_ctx
*ctx
= crypto_tfm_ctx(req_async
->tfm
);
534 struct crypto_shash
*child
= ctx
->child
;
535 struct ahash_request
*req
= ahash_request_cast(req_async
);
536 struct cryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
537 struct shash_desc
*desc
= &rctx
->desc
;
539 if (unlikely(err
== -EINPROGRESS
))
544 err
= crypto_shash_init(desc
);
546 req
->base
.complete
= rctx
->complete
;
549 cryptd_hash_complete(req
, err
);
552 static int cryptd_hash_init_enqueue(struct ahash_request
*req
)
554 return cryptd_hash_enqueue(req
, cryptd_hash_init
);
557 static void cryptd_hash_update(struct crypto_async_request
*req_async
, int err
)
559 struct ahash_request
*req
= ahash_request_cast(req_async
);
560 struct cryptd_hash_request_ctx
*rctx
;
562 rctx
= ahash_request_ctx(req
);
564 if (unlikely(err
== -EINPROGRESS
))
567 err
= shash_ahash_update(req
, &rctx
->desc
);
569 req
->base
.complete
= rctx
->complete
;
572 cryptd_hash_complete(req
, err
);
575 static int cryptd_hash_update_enqueue(struct ahash_request
*req
)
577 return cryptd_hash_enqueue(req
, cryptd_hash_update
);
580 static void cryptd_hash_final(struct crypto_async_request
*req_async
, int err
)
582 struct ahash_request
*req
= ahash_request_cast(req_async
);
583 struct cryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
585 if (unlikely(err
== -EINPROGRESS
))
588 err
= crypto_shash_final(&rctx
->desc
, req
->result
);
590 req
->base
.complete
= rctx
->complete
;
593 cryptd_hash_complete(req
, err
);
596 static int cryptd_hash_final_enqueue(struct ahash_request
*req
)
598 return cryptd_hash_enqueue(req
, cryptd_hash_final
);
601 static void cryptd_hash_finup(struct crypto_async_request
*req_async
, int err
)
603 struct ahash_request
*req
= ahash_request_cast(req_async
);
604 struct cryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
606 if (unlikely(err
== -EINPROGRESS
))
609 err
= shash_ahash_finup(req
, &rctx
->desc
);
611 req
->base
.complete
= rctx
->complete
;
614 cryptd_hash_complete(req
, err
);
617 static int cryptd_hash_finup_enqueue(struct ahash_request
*req
)
619 return cryptd_hash_enqueue(req
, cryptd_hash_finup
);
622 static void cryptd_hash_digest(struct crypto_async_request
*req_async
, int err
)
624 struct cryptd_hash_ctx
*ctx
= crypto_tfm_ctx(req_async
->tfm
);
625 struct crypto_shash
*child
= ctx
->child
;
626 struct ahash_request
*req
= ahash_request_cast(req_async
);
627 struct cryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
628 struct shash_desc
*desc
= &rctx
->desc
;
630 if (unlikely(err
== -EINPROGRESS
))
635 err
= shash_ahash_digest(req
, desc
);
637 req
->base
.complete
= rctx
->complete
;
640 cryptd_hash_complete(req
, err
);
643 static int cryptd_hash_digest_enqueue(struct ahash_request
*req
)
645 return cryptd_hash_enqueue(req
, cryptd_hash_digest
);
648 static int cryptd_hash_export(struct ahash_request
*req
, void *out
)
650 struct cryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
652 return crypto_shash_export(&rctx
->desc
, out
);
655 static int cryptd_hash_import(struct ahash_request
*req
, const void *in
)
657 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
658 struct cryptd_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
659 struct shash_desc
*desc
= cryptd_shash_desc(req
);
661 desc
->tfm
= ctx
->child
;
663 return crypto_shash_import(desc
, in
);
666 static int cryptd_create_hash(struct crypto_template
*tmpl
, struct rtattr
**tb
,
667 struct cryptd_queue
*queue
)
669 struct hashd_instance_ctx
*ctx
;
670 struct ahash_instance
*inst
;
671 struct shash_alg
*salg
;
672 struct crypto_alg
*alg
;
677 cryptd_check_internal(tb
, &type
, &mask
);
679 salg
= shash_attr_alg(tb
[1], type
, mask
);
681 return PTR_ERR(salg
);
684 inst
= cryptd_alloc_instance(alg
, ahash_instance_headroom(),
690 ctx
= ahash_instance_ctx(inst
);
693 err
= crypto_init_shash_spawn(&ctx
->spawn
, salg
,
694 ahash_crypto_instance(inst
));
698 inst
->alg
.halg
.base
.cra_flags
= CRYPTO_ALG_ASYNC
|
699 (alg
->cra_flags
& (CRYPTO_ALG_INTERNAL
|
700 CRYPTO_ALG_OPTIONAL_KEY
));
702 inst
->alg
.halg
.digestsize
= salg
->digestsize
;
703 inst
->alg
.halg
.statesize
= salg
->statesize
;
704 inst
->alg
.halg
.base
.cra_ctxsize
= sizeof(struct cryptd_hash_ctx
);
706 inst
->alg
.halg
.base
.cra_init
= cryptd_hash_init_tfm
;
707 inst
->alg
.halg
.base
.cra_exit
= cryptd_hash_exit_tfm
;
709 inst
->alg
.init
= cryptd_hash_init_enqueue
;
710 inst
->alg
.update
= cryptd_hash_update_enqueue
;
711 inst
->alg
.final
= cryptd_hash_final_enqueue
;
712 inst
->alg
.finup
= cryptd_hash_finup_enqueue
;
713 inst
->alg
.export
= cryptd_hash_export
;
714 inst
->alg
.import
= cryptd_hash_import
;
715 if (crypto_shash_alg_has_setkey(salg
))
716 inst
->alg
.setkey
= cryptd_hash_setkey
;
717 inst
->alg
.digest
= cryptd_hash_digest_enqueue
;
719 err
= ahash_register_instance(tmpl
, inst
);
721 crypto_drop_shash(&ctx
->spawn
);
731 static int cryptd_aead_setkey(struct crypto_aead
*parent
,
732 const u8
*key
, unsigned int keylen
)
734 struct cryptd_aead_ctx
*ctx
= crypto_aead_ctx(parent
);
735 struct crypto_aead
*child
= ctx
->child
;
737 return crypto_aead_setkey(child
, key
, keylen
);
740 static int cryptd_aead_setauthsize(struct crypto_aead
*parent
,
741 unsigned int authsize
)
743 struct cryptd_aead_ctx
*ctx
= crypto_aead_ctx(parent
);
744 struct crypto_aead
*child
= ctx
->child
;
746 return crypto_aead_setauthsize(child
, authsize
);
749 static void cryptd_aead_crypt(struct aead_request
*req
,
750 struct crypto_aead
*child
,
752 int (*crypt
)(struct aead_request
*req
))
754 struct cryptd_aead_request_ctx
*rctx
;
755 struct cryptd_aead_ctx
*ctx
;
756 crypto_completion_t
compl;
757 struct crypto_aead
*tfm
;
760 rctx
= aead_request_ctx(req
);
761 compl = rctx
->complete
;
763 tfm
= crypto_aead_reqtfm(req
);
765 if (unlikely(err
== -EINPROGRESS
))
767 aead_request_set_tfm(req
, child
);
771 ctx
= crypto_aead_ctx(tfm
);
772 refcnt
= atomic_read(&ctx
->refcnt
);
775 compl(&req
->base
, err
);
778 if (err
!= -EINPROGRESS
&& refcnt
&& atomic_dec_and_test(&ctx
->refcnt
))
779 crypto_free_aead(tfm
);
782 static void cryptd_aead_encrypt(struct crypto_async_request
*areq
, int err
)
784 struct cryptd_aead_ctx
*ctx
= crypto_tfm_ctx(areq
->tfm
);
785 struct crypto_aead
*child
= ctx
->child
;
786 struct aead_request
*req
;
788 req
= container_of(areq
, struct aead_request
, base
);
789 cryptd_aead_crypt(req
, child
, err
, crypto_aead_alg(child
)->encrypt
);
792 static void cryptd_aead_decrypt(struct crypto_async_request
*areq
, int err
)
794 struct cryptd_aead_ctx
*ctx
= crypto_tfm_ctx(areq
->tfm
);
795 struct crypto_aead
*child
= ctx
->child
;
796 struct aead_request
*req
;
798 req
= container_of(areq
, struct aead_request
, base
);
799 cryptd_aead_crypt(req
, child
, err
, crypto_aead_alg(child
)->decrypt
);
802 static int cryptd_aead_enqueue(struct aead_request
*req
,
803 crypto_completion_t
compl)
805 struct cryptd_aead_request_ctx
*rctx
= aead_request_ctx(req
);
806 struct crypto_aead
*tfm
= crypto_aead_reqtfm(req
);
807 struct cryptd_queue
*queue
= cryptd_get_queue(crypto_aead_tfm(tfm
));
809 rctx
->complete
= req
->base
.complete
;
810 req
->base
.complete
= compl;
811 return cryptd_enqueue_request(queue
, &req
->base
);
814 static int cryptd_aead_encrypt_enqueue(struct aead_request
*req
)
816 return cryptd_aead_enqueue(req
, cryptd_aead_encrypt
);
819 static int cryptd_aead_decrypt_enqueue(struct aead_request
*req
)
821 return cryptd_aead_enqueue(req
, cryptd_aead_decrypt
);
824 static int cryptd_aead_init_tfm(struct crypto_aead
*tfm
)
826 struct aead_instance
*inst
= aead_alg_instance(tfm
);
827 struct aead_instance_ctx
*ictx
= aead_instance_ctx(inst
);
828 struct crypto_aead_spawn
*spawn
= &ictx
->aead_spawn
;
829 struct cryptd_aead_ctx
*ctx
= crypto_aead_ctx(tfm
);
830 struct crypto_aead
*cipher
;
832 cipher
= crypto_spawn_aead(spawn
);
834 return PTR_ERR(cipher
);
837 crypto_aead_set_reqsize(
838 tfm
, max((unsigned)sizeof(struct cryptd_aead_request_ctx
),
839 crypto_aead_reqsize(cipher
)));
843 static void cryptd_aead_exit_tfm(struct crypto_aead
*tfm
)
845 struct cryptd_aead_ctx
*ctx
= crypto_aead_ctx(tfm
);
846 crypto_free_aead(ctx
->child
);
849 static int cryptd_create_aead(struct crypto_template
*tmpl
,
851 struct cryptd_queue
*queue
)
853 struct aead_instance_ctx
*ctx
;
854 struct aead_instance
*inst
;
855 struct aead_alg
*alg
;
858 u32 mask
= CRYPTO_ALG_ASYNC
;
861 cryptd_check_internal(tb
, &type
, &mask
);
863 name
= crypto_attr_alg_name(tb
[1]);
865 return PTR_ERR(name
);
867 inst
= kzalloc(sizeof(*inst
) + sizeof(*ctx
), GFP_KERNEL
);
871 ctx
= aead_instance_ctx(inst
);
874 crypto_set_aead_spawn(&ctx
->aead_spawn
, aead_crypto_instance(inst
));
875 err
= crypto_grab_aead(&ctx
->aead_spawn
, name
, type
, mask
);
879 alg
= crypto_spawn_aead_alg(&ctx
->aead_spawn
);
880 err
= cryptd_init_instance(aead_crypto_instance(inst
), &alg
->base
);
884 inst
->alg
.base
.cra_flags
= CRYPTO_ALG_ASYNC
|
885 (alg
->base
.cra_flags
& CRYPTO_ALG_INTERNAL
);
886 inst
->alg
.base
.cra_ctxsize
= sizeof(struct cryptd_aead_ctx
);
888 inst
->alg
.ivsize
= crypto_aead_alg_ivsize(alg
);
889 inst
->alg
.maxauthsize
= crypto_aead_alg_maxauthsize(alg
);
891 inst
->alg
.init
= cryptd_aead_init_tfm
;
892 inst
->alg
.exit
= cryptd_aead_exit_tfm
;
893 inst
->alg
.setkey
= cryptd_aead_setkey
;
894 inst
->alg
.setauthsize
= cryptd_aead_setauthsize
;
895 inst
->alg
.encrypt
= cryptd_aead_encrypt_enqueue
;
896 inst
->alg
.decrypt
= cryptd_aead_decrypt_enqueue
;
898 err
= aead_register_instance(tmpl
, inst
);
901 crypto_drop_aead(&ctx
->aead_spawn
);
908 static struct cryptd_queue queue
;
910 static int cryptd_create(struct crypto_template
*tmpl
, struct rtattr
**tb
)
912 struct crypto_attr_type
*algt
;
914 algt
= crypto_get_attr_type(tb
);
916 return PTR_ERR(algt
);
918 switch (algt
->type
& algt
->mask
& CRYPTO_ALG_TYPE_MASK
) {
919 case CRYPTO_ALG_TYPE_BLKCIPHER
:
920 return cryptd_create_skcipher(tmpl
, tb
, &queue
);
921 case CRYPTO_ALG_TYPE_DIGEST
:
922 return cryptd_create_hash(tmpl
, tb
, &queue
);
923 case CRYPTO_ALG_TYPE_AEAD
:
924 return cryptd_create_aead(tmpl
, tb
, &queue
);
930 static void cryptd_free(struct crypto_instance
*inst
)
932 struct cryptd_instance_ctx
*ctx
= crypto_instance_ctx(inst
);
933 struct hashd_instance_ctx
*hctx
= crypto_instance_ctx(inst
);
934 struct aead_instance_ctx
*aead_ctx
= crypto_instance_ctx(inst
);
936 switch (inst
->alg
.cra_flags
& CRYPTO_ALG_TYPE_MASK
) {
937 case CRYPTO_ALG_TYPE_AHASH
:
938 crypto_drop_shash(&hctx
->spawn
);
939 kfree(ahash_instance(inst
));
941 case CRYPTO_ALG_TYPE_AEAD
:
942 crypto_drop_aead(&aead_ctx
->aead_spawn
);
943 kfree(aead_instance(inst
));
946 crypto_drop_spawn(&ctx
->spawn
);
951 static struct crypto_template cryptd_tmpl
= {
953 .create
= cryptd_create
,
955 .module
= THIS_MODULE
,
958 struct cryptd_skcipher
*cryptd_alloc_skcipher(const char *alg_name
,
961 char cryptd_alg_name
[CRYPTO_MAX_ALG_NAME
];
962 struct cryptd_skcipher_ctx
*ctx
;
963 struct crypto_skcipher
*tfm
;
965 if (snprintf(cryptd_alg_name
, CRYPTO_MAX_ALG_NAME
,
966 "cryptd(%s)", alg_name
) >= CRYPTO_MAX_ALG_NAME
)
967 return ERR_PTR(-EINVAL
);
969 tfm
= crypto_alloc_skcipher(cryptd_alg_name
, type
, mask
);
971 return ERR_CAST(tfm
);
973 if (tfm
->base
.__crt_alg
->cra_module
!= THIS_MODULE
) {
974 crypto_free_skcipher(tfm
);
975 return ERR_PTR(-EINVAL
);
978 ctx
= crypto_skcipher_ctx(tfm
);
979 atomic_set(&ctx
->refcnt
, 1);
981 return container_of(tfm
, struct cryptd_skcipher
, base
);
983 EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher
);
985 struct crypto_skcipher
*cryptd_skcipher_child(struct cryptd_skcipher
*tfm
)
987 struct cryptd_skcipher_ctx
*ctx
= crypto_skcipher_ctx(&tfm
->base
);
989 return &ctx
->child
->base
;
991 EXPORT_SYMBOL_GPL(cryptd_skcipher_child
);
993 bool cryptd_skcipher_queued(struct cryptd_skcipher
*tfm
)
995 struct cryptd_skcipher_ctx
*ctx
= crypto_skcipher_ctx(&tfm
->base
);
997 return atomic_read(&ctx
->refcnt
) - 1;
999 EXPORT_SYMBOL_GPL(cryptd_skcipher_queued
);
1001 void cryptd_free_skcipher(struct cryptd_skcipher
*tfm
)
1003 struct cryptd_skcipher_ctx
*ctx
= crypto_skcipher_ctx(&tfm
->base
);
1005 if (atomic_dec_and_test(&ctx
->refcnt
))
1006 crypto_free_skcipher(&tfm
->base
);
1008 EXPORT_SYMBOL_GPL(cryptd_free_skcipher
);
1010 struct cryptd_ahash
*cryptd_alloc_ahash(const char *alg_name
,
1013 char cryptd_alg_name
[CRYPTO_MAX_ALG_NAME
];
1014 struct cryptd_hash_ctx
*ctx
;
1015 struct crypto_ahash
*tfm
;
1017 if (snprintf(cryptd_alg_name
, CRYPTO_MAX_ALG_NAME
,
1018 "cryptd(%s)", alg_name
) >= CRYPTO_MAX_ALG_NAME
)
1019 return ERR_PTR(-EINVAL
);
1020 tfm
= crypto_alloc_ahash(cryptd_alg_name
, type
, mask
);
1022 return ERR_CAST(tfm
);
1023 if (tfm
->base
.__crt_alg
->cra_module
!= THIS_MODULE
) {
1024 crypto_free_ahash(tfm
);
1025 return ERR_PTR(-EINVAL
);
1028 ctx
= crypto_ahash_ctx(tfm
);
1029 atomic_set(&ctx
->refcnt
, 1);
1031 return __cryptd_ahash_cast(tfm
);
1033 EXPORT_SYMBOL_GPL(cryptd_alloc_ahash
);
1035 struct crypto_shash
*cryptd_ahash_child(struct cryptd_ahash
*tfm
)
1037 struct cryptd_hash_ctx
*ctx
= crypto_ahash_ctx(&tfm
->base
);
1041 EXPORT_SYMBOL_GPL(cryptd_ahash_child
);
1043 struct shash_desc
*cryptd_shash_desc(struct ahash_request
*req
)
1045 struct cryptd_hash_request_ctx
*rctx
= ahash_request_ctx(req
);
1048 EXPORT_SYMBOL_GPL(cryptd_shash_desc
);
1050 bool cryptd_ahash_queued(struct cryptd_ahash
*tfm
)
1052 struct cryptd_hash_ctx
*ctx
= crypto_ahash_ctx(&tfm
->base
);
1054 return atomic_read(&ctx
->refcnt
) - 1;
1056 EXPORT_SYMBOL_GPL(cryptd_ahash_queued
);
1058 void cryptd_free_ahash(struct cryptd_ahash
*tfm
)
1060 struct cryptd_hash_ctx
*ctx
= crypto_ahash_ctx(&tfm
->base
);
1062 if (atomic_dec_and_test(&ctx
->refcnt
))
1063 crypto_free_ahash(&tfm
->base
);
1065 EXPORT_SYMBOL_GPL(cryptd_free_ahash
);
1067 struct cryptd_aead
*cryptd_alloc_aead(const char *alg_name
,
1070 char cryptd_alg_name
[CRYPTO_MAX_ALG_NAME
];
1071 struct cryptd_aead_ctx
*ctx
;
1072 struct crypto_aead
*tfm
;
1074 if (snprintf(cryptd_alg_name
, CRYPTO_MAX_ALG_NAME
,
1075 "cryptd(%s)", alg_name
) >= CRYPTO_MAX_ALG_NAME
)
1076 return ERR_PTR(-EINVAL
);
1077 tfm
= crypto_alloc_aead(cryptd_alg_name
, type
, mask
);
1079 return ERR_CAST(tfm
);
1080 if (tfm
->base
.__crt_alg
->cra_module
!= THIS_MODULE
) {
1081 crypto_free_aead(tfm
);
1082 return ERR_PTR(-EINVAL
);
1085 ctx
= crypto_aead_ctx(tfm
);
1086 atomic_set(&ctx
->refcnt
, 1);
1088 return __cryptd_aead_cast(tfm
);
1090 EXPORT_SYMBOL_GPL(cryptd_alloc_aead
);
1092 struct crypto_aead
*cryptd_aead_child(struct cryptd_aead
*tfm
)
1094 struct cryptd_aead_ctx
*ctx
;
1095 ctx
= crypto_aead_ctx(&tfm
->base
);
1098 EXPORT_SYMBOL_GPL(cryptd_aead_child
);
1100 bool cryptd_aead_queued(struct cryptd_aead
*tfm
)
1102 struct cryptd_aead_ctx
*ctx
= crypto_aead_ctx(&tfm
->base
);
1104 return atomic_read(&ctx
->refcnt
) - 1;
1106 EXPORT_SYMBOL_GPL(cryptd_aead_queued
);
1108 void cryptd_free_aead(struct cryptd_aead
*tfm
)
1110 struct cryptd_aead_ctx
*ctx
= crypto_aead_ctx(&tfm
->base
);
1112 if (atomic_dec_and_test(&ctx
->refcnt
))
1113 crypto_free_aead(&tfm
->base
);
1115 EXPORT_SYMBOL_GPL(cryptd_free_aead
);
1117 static int __init
cryptd_init(void)
1121 err
= cryptd_init_queue(&queue
, cryptd_max_cpu_qlen
);
1125 err
= crypto_register_template(&cryptd_tmpl
);
1127 cryptd_fini_queue(&queue
);
1132 static void __exit
cryptd_exit(void)
1134 cryptd_fini_queue(&queue
);
1135 crypto_unregister_template(&cryptd_tmpl
);
1138 subsys_initcall(cryptd_init
);
1139 module_exit(cryptd_exit
);
1141 MODULE_LICENSE("GPL");
1142 MODULE_DESCRIPTION("Software async crypto daemon");
1143 MODULE_ALIAS_CRYPTO("cryptd");