]> git.proxmox.com Git - mirror_ubuntu-artful-kernel.git/blame - crypto/cryptd.c
platform/x86: intel_pmc_ipc: Convert to use platform_device_register_full
[mirror_ubuntu-artful-kernel.git] / crypto / cryptd.c
CommitLineData
124b53d0
HX
1/*
2 * Software async crypto daemon.
3 *
4 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
5 *
298c926c
AH
6 * Added AEAD support to cryptd.
7 * Authors: Tadeusz Struk (tadeusz.struk@intel.com)
8 * Adrian Hoban <adrian.hoban@intel.com>
9 * Gabriele Paoloni <gabriele.paoloni@intel.com>
10 * Aidan O'Mahony (aidan.o.mahony@intel.com)
11 * Copyright (c) 2010, Intel Corporation.
12 *
124b53d0
HX
13 * This program is free software; you can redistribute it and/or modify it
14 * under the terms of the GNU General Public License as published by the Free
15 * Software Foundation; either version 2 of the License, or (at your option)
16 * any later version.
17 *
18 */
19
20#include <crypto/algapi.h>
18e33e6d 21#include <crypto/internal/hash.h>
298c926c 22#include <crypto/internal/aead.h>
1cac2cbc 23#include <crypto/cryptd.h>
254eff77 24#include <crypto/crypto_wq.h>
81760ea6 25#include <linux/atomic.h>
124b53d0
HX
26#include <linux/err.h>
27#include <linux/init.h>
28#include <linux/kernel.h>
124b53d0
HX
29#include <linux/list.h>
30#include <linux/module.h>
124b53d0
HX
31#include <linux/scatterlist.h>
32#include <linux/sched.h>
33#include <linux/slab.h>
124b53d0 34
81760ea6 35#define CRYPTD_MAX_CPU_QLEN 1000
124b53d0 36
254eff77 37struct cryptd_cpu_queue {
124b53d0 38 struct crypto_queue queue;
254eff77
HY
39 struct work_struct work;
40};
41
42struct cryptd_queue {
a29d8b8e 43 struct cryptd_cpu_queue __percpu *cpu_queue;
124b53d0
HX
44};
45
46struct cryptd_instance_ctx {
47 struct crypto_spawn spawn;
254eff77 48 struct cryptd_queue *queue;
124b53d0
HX
49};
50
46309d89
HX
51struct hashd_instance_ctx {
52 struct crypto_shash_spawn spawn;
53 struct cryptd_queue *queue;
54};
55
298c926c
AH
56struct aead_instance_ctx {
57 struct crypto_aead_spawn aead_spawn;
58 struct cryptd_queue *queue;
59};
60
124b53d0 61struct cryptd_blkcipher_ctx {
81760ea6 62 atomic_t refcnt;
124b53d0
HX
63 struct crypto_blkcipher *child;
64};
65
66struct cryptd_blkcipher_request_ctx {
67 crypto_completion_t complete;
68};
69
b8a28251 70struct cryptd_hash_ctx {
81760ea6 71 atomic_t refcnt;
46309d89 72 struct crypto_shash *child;
b8a28251
LH
73};
74
75struct cryptd_hash_request_ctx {
76 crypto_completion_t complete;
46309d89 77 struct shash_desc desc;
b8a28251 78};
124b53d0 79
298c926c 80struct cryptd_aead_ctx {
81760ea6 81 atomic_t refcnt;
298c926c
AH
82 struct crypto_aead *child;
83};
84
85struct cryptd_aead_request_ctx {
86 crypto_completion_t complete;
87};
88
254eff77
HY
89static void cryptd_queue_worker(struct work_struct *work);
90
91static int cryptd_init_queue(struct cryptd_queue *queue,
92 unsigned int max_cpu_qlen)
93{
94 int cpu;
95 struct cryptd_cpu_queue *cpu_queue;
96
97 queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue);
98 if (!queue->cpu_queue)
99 return -ENOMEM;
100 for_each_possible_cpu(cpu) {
101 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
102 crypto_init_queue(&cpu_queue->queue, max_cpu_qlen);
103 INIT_WORK(&cpu_queue->work, cryptd_queue_worker);
104 }
105 return 0;
106}
107
108static void cryptd_fini_queue(struct cryptd_queue *queue)
109{
110 int cpu;
111 struct cryptd_cpu_queue *cpu_queue;
112
113 for_each_possible_cpu(cpu) {
114 cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu);
115 BUG_ON(cpu_queue->queue.qlen);
116 }
117 free_percpu(queue->cpu_queue);
118}
119
120static int cryptd_enqueue_request(struct cryptd_queue *queue,
121 struct crypto_async_request *request)
122{
123 int cpu, err;
124 struct cryptd_cpu_queue *cpu_queue;
81760ea6
HX
125 struct crypto_tfm *tfm;
126 atomic_t *refcnt;
127 bool may_backlog;
254eff77
HY
128
129 cpu = get_cpu();
0b44f486 130 cpu_queue = this_cpu_ptr(queue->cpu_queue);
254eff77 131 err = crypto_enqueue_request(&cpu_queue->queue, request);
81760ea6
HX
132
133 refcnt = crypto_tfm_ctx(request->tfm);
134 may_backlog = request->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
135
136 if (err == -EBUSY && !may_backlog)
137 goto out_put_cpu;
138
254eff77 139 queue_work_on(cpu, kcrypto_wq, &cpu_queue->work);
81760ea6
HX
140
141 if (!atomic_read(refcnt))
142 goto out_put_cpu;
143
144 tfm = request->tfm;
145 atomic_inc(refcnt);
146
147out_put_cpu:
254eff77
HY
148 put_cpu();
149
150 return err;
151}
152
153/* Called in workqueue context, do one real cryption work (via
154 * req->complete) and reschedule itself if there are more work to
155 * do. */
156static void cryptd_queue_worker(struct work_struct *work)
157{
158 struct cryptd_cpu_queue *cpu_queue;
159 struct crypto_async_request *req, *backlog;
160
161 cpu_queue = container_of(work, struct cryptd_cpu_queue, work);
9efade1b
JK
162 /*
163 * Only handle one request at a time to avoid hogging crypto workqueue.
164 * preempt_disable/enable is used to prevent being preempted by
165 * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent
166 * cryptd_enqueue_request() being accessed from software interrupts.
167 */
168 local_bh_disable();
254eff77
HY
169 preempt_disable();
170 backlog = crypto_get_backlog(&cpu_queue->queue);
171 req = crypto_dequeue_request(&cpu_queue->queue);
172 preempt_enable();
9efade1b 173 local_bh_enable();
254eff77
HY
174
175 if (!req)
176 return;
177
178 if (backlog)
179 backlog->complete(backlog, -EINPROGRESS);
180 req->complete(req, 0);
181
182 if (cpu_queue->queue.qlen)
183 queue_work(kcrypto_wq, &cpu_queue->work);
184}
185
186static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm)
124b53d0
HX
187{
188 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
189 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
254eff77 190 return ictx->queue;
124b53d0
HX
191}
192
466a7b9e
SM
193static inline void cryptd_check_internal(struct rtattr **tb, u32 *type,
194 u32 *mask)
195{
196 struct crypto_attr_type *algt;
197
198 algt = crypto_get_attr_type(tb);
199 if (IS_ERR(algt))
200 return;
f6da3205 201
5e4b8c1f
HX
202 *type |= algt->type & CRYPTO_ALG_INTERNAL;
203 *mask |= algt->mask & CRYPTO_ALG_INTERNAL;
466a7b9e
SM
204}
205
124b53d0
HX
206static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent,
207 const u8 *key, unsigned int keylen)
208{
209 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent);
210 struct crypto_blkcipher *child = ctx->child;
211 int err;
212
213 crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
214 crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) &
215 CRYPTO_TFM_REQ_MASK);
216 err = crypto_blkcipher_setkey(child, key, keylen);
217 crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) &
218 CRYPTO_TFM_RES_MASK);
219 return err;
220}
221
222static void cryptd_blkcipher_crypt(struct ablkcipher_request *req,
223 struct crypto_blkcipher *child,
224 int err,
225 int (*crypt)(struct blkcipher_desc *desc,
226 struct scatterlist *dst,
227 struct scatterlist *src,
228 unsigned int len))
229{
230 struct cryptd_blkcipher_request_ctx *rctx;
81760ea6
HX
231 struct cryptd_blkcipher_ctx *ctx;
232 struct crypto_ablkcipher *tfm;
124b53d0 233 struct blkcipher_desc desc;
81760ea6 234 int refcnt;
124b53d0
HX
235
236 rctx = ablkcipher_request_ctx(req);
237
93aa7f8a
HX
238 if (unlikely(err == -EINPROGRESS))
239 goto out;
124b53d0
HX
240
241 desc.tfm = child;
242 desc.info = req->info;
243 desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
244
245 err = crypt(&desc, req->dst, req->src, req->nbytes);
246
247 req->base.complete = rctx->complete;
248
93aa7f8a 249out:
81760ea6
HX
250 tfm = crypto_ablkcipher_reqtfm(req);
251 ctx = crypto_ablkcipher_ctx(tfm);
252 refcnt = atomic_read(&ctx->refcnt);
253
124b53d0 254 local_bh_disable();
93aa7f8a 255 rctx->complete(&req->base, err);
124b53d0 256 local_bh_enable();
81760ea6
HX
257
258 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
259 crypto_free_ablkcipher(tfm);
124b53d0
HX
260}
261
262static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err)
263{
264 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
265 struct crypto_blkcipher *child = ctx->child;
266
267 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
268 crypto_blkcipher_crt(child)->encrypt);
269}
270
271static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err)
272{
273 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm);
274 struct crypto_blkcipher *child = ctx->child;
275
276 cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err,
277 crypto_blkcipher_crt(child)->decrypt);
278}
279
280static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req,
3e3dc25f 281 crypto_completion_t compl)
124b53d0
HX
282{
283 struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req);
284 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
254eff77 285 struct cryptd_queue *queue;
124b53d0 286
254eff77 287 queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm));
124b53d0 288 rctx->complete = req->base.complete;
3e3dc25f 289 req->base.complete = compl;
124b53d0 290
254eff77 291 return cryptd_enqueue_request(queue, &req->base);
124b53d0
HX
292}
293
294static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req)
295{
296 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt);
297}
298
299static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req)
300{
301 return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt);
302}
303
304static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm)
305{
306 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
307 struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst);
308 struct crypto_spawn *spawn = &ictx->spawn;
309 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
310 struct crypto_blkcipher *cipher;
311
312 cipher = crypto_spawn_blkcipher(spawn);
313 if (IS_ERR(cipher))
314 return PTR_ERR(cipher);
315
316 ctx->child = cipher;
317 tfm->crt_ablkcipher.reqsize =
318 sizeof(struct cryptd_blkcipher_request_ctx);
319 return 0;
320}
321
322static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm)
323{
324 struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm);
124b53d0
HX
325
326 crypto_free_blkcipher(ctx->child);
327}
328
9b8c456e
HX
329static int cryptd_init_instance(struct crypto_instance *inst,
330 struct crypto_alg *alg)
331{
332 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
333 "cryptd(%s)",
334 alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
335 return -ENAMETOOLONG;
336
337 memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
338
339 inst->alg.cra_priority = alg->cra_priority + 50;
340 inst->alg.cra_blocksize = alg->cra_blocksize;
341 inst->alg.cra_alignmask = alg->cra_alignmask;
342
343 return 0;
344}
345
0b535adf
HX
346static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head,
347 unsigned int tail)
124b53d0 348{
0b535adf 349 char *p;
124b53d0 350 struct crypto_instance *inst;
124b53d0
HX
351 int err;
352
0b535adf
HX
353 p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL);
354 if (!p)
355 return ERR_PTR(-ENOMEM);
356
357 inst = (void *)(p + head);
124b53d0 358
9b8c456e
HX
359 err = cryptd_init_instance(inst, alg);
360 if (err)
124b53d0
HX
361 goto out_free_inst;
362
124b53d0 363out:
0b535adf 364 return p;
124b53d0
HX
365
366out_free_inst:
0b535adf
HX
367 kfree(p);
368 p = ERR_PTR(err);
124b53d0
HX
369 goto out;
370}
371
9cd899a3
HX
372static int cryptd_create_blkcipher(struct crypto_template *tmpl,
373 struct rtattr **tb,
374 struct cryptd_queue *queue)
124b53d0 375{
46309d89 376 struct cryptd_instance_ctx *ctx;
124b53d0
HX
377 struct crypto_instance *inst;
378 struct crypto_alg *alg;
466a7b9e
SM
379 u32 type = CRYPTO_ALG_TYPE_BLKCIPHER;
380 u32 mask = CRYPTO_ALG_TYPE_MASK;
46309d89 381 int err;
124b53d0 382
466a7b9e
SM
383 cryptd_check_internal(tb, &type, &mask);
384
385 alg = crypto_get_attr_alg(tb, type, mask);
124b53d0 386 if (IS_ERR(alg))
9cd899a3 387 return PTR_ERR(alg);
124b53d0 388
0b535adf 389 inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx));
05ed8758 390 err = PTR_ERR(inst);
124b53d0
HX
391 if (IS_ERR(inst))
392 goto out_put_alg;
393
46309d89
HX
394 ctx = crypto_instance_ctx(inst);
395 ctx->queue = queue;
396
397 err = crypto_init_spawn(&ctx->spawn, alg, inst,
398 CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC);
399 if (err)
400 goto out_free_inst;
401
466a7b9e
SM
402 type = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
403 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
404 type |= CRYPTO_ALG_INTERNAL;
405 inst->alg.cra_flags = type;
124b53d0
HX
406 inst->alg.cra_type = &crypto_ablkcipher_type;
407
408 inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize;
409 inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize;
410 inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize;
411
927eead5
HX
412 inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv;
413
124b53d0
HX
414 inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx);
415
416 inst->alg.cra_init = cryptd_blkcipher_init_tfm;
417 inst->alg.cra_exit = cryptd_blkcipher_exit_tfm;
418
419 inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey;
420 inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue;
421 inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue;
422
9cd899a3
HX
423 err = crypto_register_instance(tmpl, inst);
424 if (err) {
425 crypto_drop_spawn(&ctx->spawn);
426out_free_inst:
427 kfree(inst);
428 }
429
124b53d0
HX
430out_put_alg:
431 crypto_mod_put(alg);
9cd899a3 432 return err;
124b53d0
HX
433}
434
b8a28251
LH
435static int cryptd_hash_init_tfm(struct crypto_tfm *tfm)
436{
437 struct crypto_instance *inst = crypto_tfm_alg_instance(tfm);
46309d89
HX
438 struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst);
439 struct crypto_shash_spawn *spawn = &ictx->spawn;
b8a28251 440 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
46309d89 441 struct crypto_shash *hash;
b8a28251 442
46309d89
HX
443 hash = crypto_spawn_shash(spawn);
444 if (IS_ERR(hash))
445 return PTR_ERR(hash);
b8a28251 446
46309d89 447 ctx->child = hash;
0d6669e2
HX
448 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
449 sizeof(struct cryptd_hash_request_ctx) +
450 crypto_shash_descsize(hash));
b8a28251
LH
451 return 0;
452}
453
454static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm)
455{
456 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm);
b8a28251 457
46309d89 458 crypto_free_shash(ctx->child);
b8a28251
LH
459}
460
461static int cryptd_hash_setkey(struct crypto_ahash *parent,
462 const u8 *key, unsigned int keylen)
463{
464 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent);
46309d89 465 struct crypto_shash *child = ctx->child;
b8a28251
LH
466 int err;
467
46309d89
HX
468 crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
469 crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) &
470 CRYPTO_TFM_REQ_MASK);
471 err = crypto_shash_setkey(child, key, keylen);
472 crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) &
473 CRYPTO_TFM_RES_MASK);
b8a28251
LH
474 return err;
475}
476
477static int cryptd_hash_enqueue(struct ahash_request *req,
3e3dc25f 478 crypto_completion_t compl)
b8a28251
LH
479{
480 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
481 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
254eff77
HY
482 struct cryptd_queue *queue =
483 cryptd_get_queue(crypto_ahash_tfm(tfm));
b8a28251
LH
484
485 rctx->complete = req->base.complete;
3e3dc25f 486 req->base.complete = compl;
b8a28251 487
254eff77 488 return cryptd_enqueue_request(queue, &req->base);
b8a28251
LH
489}
490
81760ea6
HX
491static void cryptd_hash_complete(struct ahash_request *req, int err)
492{
493 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
494 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm);
495 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
496 int refcnt = atomic_read(&ctx->refcnt);
497
498 local_bh_disable();
499 rctx->complete(&req->base, err);
500 local_bh_enable();
501
502 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
503 crypto_free_ahash(tfm);
504}
505
b8a28251
LH
506static void cryptd_hash_init(struct crypto_async_request *req_async, int err)
507{
46309d89
HX
508 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
509 struct crypto_shash *child = ctx->child;
510 struct ahash_request *req = ahash_request_cast(req_async);
511 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
512 struct shash_desc *desc = &rctx->desc;
b8a28251
LH
513
514 if (unlikely(err == -EINPROGRESS))
515 goto out;
516
46309d89
HX
517 desc->tfm = child;
518 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
b8a28251 519
46309d89 520 err = crypto_shash_init(desc);
b8a28251
LH
521
522 req->base.complete = rctx->complete;
523
524out:
81760ea6 525 cryptd_hash_complete(req, err);
b8a28251
LH
526}
527
528static int cryptd_hash_init_enqueue(struct ahash_request *req)
529{
530 return cryptd_hash_enqueue(req, cryptd_hash_init);
531}
532
533static void cryptd_hash_update(struct crypto_async_request *req_async, int err)
534{
46309d89 535 struct ahash_request *req = ahash_request_cast(req_async);
b8a28251 536 struct cryptd_hash_request_ctx *rctx;
b8a28251
LH
537
538 rctx = ahash_request_ctx(req);
539
540 if (unlikely(err == -EINPROGRESS))
541 goto out;
542
46309d89 543 err = shash_ahash_update(req, &rctx->desc);
b8a28251
LH
544
545 req->base.complete = rctx->complete;
546
547out:
81760ea6 548 cryptd_hash_complete(req, err);
b8a28251
LH
549}
550
551static int cryptd_hash_update_enqueue(struct ahash_request *req)
552{
553 return cryptd_hash_enqueue(req, cryptd_hash_update);
554}
555
556static void cryptd_hash_final(struct crypto_async_request *req_async, int err)
557{
46309d89
HX
558 struct ahash_request *req = ahash_request_cast(req_async);
559 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
b8a28251
LH
560
561 if (unlikely(err == -EINPROGRESS))
562 goto out;
563
46309d89 564 err = crypto_shash_final(&rctx->desc, req->result);
b8a28251
LH
565
566 req->base.complete = rctx->complete;
567
568out:
81760ea6 569 cryptd_hash_complete(req, err);
b8a28251
LH
570}
571
572static int cryptd_hash_final_enqueue(struct ahash_request *req)
573{
574 return cryptd_hash_enqueue(req, cryptd_hash_final);
575}
576
6fba00d1
HX
577static void cryptd_hash_finup(struct crypto_async_request *req_async, int err)
578{
579 struct ahash_request *req = ahash_request_cast(req_async);
580 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
581
582 if (unlikely(err == -EINPROGRESS))
583 goto out;
584
585 err = shash_ahash_finup(req, &rctx->desc);
586
587 req->base.complete = rctx->complete;
588
589out:
81760ea6 590 cryptd_hash_complete(req, err);
6fba00d1
HX
591}
592
593static int cryptd_hash_finup_enqueue(struct ahash_request *req)
594{
595 return cryptd_hash_enqueue(req, cryptd_hash_finup);
596}
597
b8a28251
LH
598static void cryptd_hash_digest(struct crypto_async_request *req_async, int err)
599{
46309d89
HX
600 struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm);
601 struct crypto_shash *child = ctx->child;
602 struct ahash_request *req = ahash_request_cast(req_async);
603 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
604 struct shash_desc *desc = &rctx->desc;
b8a28251
LH
605
606 if (unlikely(err == -EINPROGRESS))
607 goto out;
608
46309d89
HX
609 desc->tfm = child;
610 desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
b8a28251 611
46309d89 612 err = shash_ahash_digest(req, desc);
b8a28251
LH
613
614 req->base.complete = rctx->complete;
615
616out:
81760ea6 617 cryptd_hash_complete(req, err);
b8a28251
LH
618}
619
620static int cryptd_hash_digest_enqueue(struct ahash_request *req)
621{
622 return cryptd_hash_enqueue(req, cryptd_hash_digest);
623}
624
6fba00d1
HX
625static int cryptd_hash_export(struct ahash_request *req, void *out)
626{
627 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
628
629 return crypto_shash_export(&rctx->desc, out);
630}
631
632static int cryptd_hash_import(struct ahash_request *req, const void *in)
633{
634 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
635
636 return crypto_shash_import(&rctx->desc, in);
637}
638
9cd899a3
HX
639static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb,
640 struct cryptd_queue *queue)
b8a28251 641{
46309d89 642 struct hashd_instance_ctx *ctx;
0b535adf 643 struct ahash_instance *inst;
46309d89 644 struct shash_alg *salg;
b8a28251 645 struct crypto_alg *alg;
466a7b9e
SM
646 u32 type = 0;
647 u32 mask = 0;
46309d89 648 int err;
b8a28251 649
466a7b9e
SM
650 cryptd_check_internal(tb, &type, &mask);
651
652 salg = shash_attr_alg(tb[1], type, mask);
46309d89 653 if (IS_ERR(salg))
9cd899a3 654 return PTR_ERR(salg);
b8a28251 655
46309d89 656 alg = &salg->base;
0b535adf
HX
657 inst = cryptd_alloc_instance(alg, ahash_instance_headroom(),
658 sizeof(*ctx));
05ed8758 659 err = PTR_ERR(inst);
b8a28251
LH
660 if (IS_ERR(inst))
661 goto out_put_alg;
662
0b535adf 663 ctx = ahash_instance_ctx(inst);
46309d89
HX
664 ctx->queue = queue;
665
0b535adf
HX
666 err = crypto_init_shash_spawn(&ctx->spawn, salg,
667 ahash_crypto_instance(inst));
46309d89
HX
668 if (err)
669 goto out_free_inst;
670
466a7b9e
SM
671 type = CRYPTO_ALG_ASYNC;
672 if (alg->cra_flags & CRYPTO_ALG_INTERNAL)
673 type |= CRYPTO_ALG_INTERNAL;
674 inst->alg.halg.base.cra_flags = type;
b8a28251 675
0b535adf 676 inst->alg.halg.digestsize = salg->digestsize;
1a078340 677 inst->alg.halg.statesize = salg->statesize;
0b535adf 678 inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx);
b8a28251 679
0b535adf
HX
680 inst->alg.halg.base.cra_init = cryptd_hash_init_tfm;
681 inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm;
b8a28251 682
0b535adf
HX
683 inst->alg.init = cryptd_hash_init_enqueue;
684 inst->alg.update = cryptd_hash_update_enqueue;
685 inst->alg.final = cryptd_hash_final_enqueue;
6fba00d1
HX
686 inst->alg.finup = cryptd_hash_finup_enqueue;
687 inst->alg.export = cryptd_hash_export;
688 inst->alg.import = cryptd_hash_import;
0b535adf
HX
689 inst->alg.setkey = cryptd_hash_setkey;
690 inst->alg.digest = cryptd_hash_digest_enqueue;
b8a28251 691
0b535adf 692 err = ahash_register_instance(tmpl, inst);
9cd899a3
HX
693 if (err) {
694 crypto_drop_shash(&ctx->spawn);
695out_free_inst:
696 kfree(inst);
697 }
698
b8a28251
LH
699out_put_alg:
700 crypto_mod_put(alg);
9cd899a3 701 return err;
b8a28251
LH
702}
703
92b9876b
HX
704static int cryptd_aead_setkey(struct crypto_aead *parent,
705 const u8 *key, unsigned int keylen)
706{
707 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
708 struct crypto_aead *child = ctx->child;
709
710 return crypto_aead_setkey(child, key, keylen);
711}
712
713static int cryptd_aead_setauthsize(struct crypto_aead *parent,
714 unsigned int authsize)
715{
716 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent);
717 struct crypto_aead *child = ctx->child;
718
719 return crypto_aead_setauthsize(child, authsize);
720}
721
298c926c
AH
722static void cryptd_aead_crypt(struct aead_request *req,
723 struct crypto_aead *child,
724 int err,
725 int (*crypt)(struct aead_request *req))
726{
727 struct cryptd_aead_request_ctx *rctx;
81760ea6 728 struct cryptd_aead_ctx *ctx;
ec9f2006 729 crypto_completion_t compl;
81760ea6
HX
730 struct crypto_aead *tfm;
731 int refcnt;
ec9f2006 732
298c926c 733 rctx = aead_request_ctx(req);
ec9f2006 734 compl = rctx->complete;
298c926c
AH
735
736 if (unlikely(err == -EINPROGRESS))
737 goto out;
738 aead_request_set_tfm(req, child);
739 err = crypt( req );
81760ea6 740
298c926c 741out:
81760ea6
HX
742 tfm = crypto_aead_reqtfm(req);
743 ctx = crypto_aead_ctx(tfm);
744 refcnt = atomic_read(&ctx->refcnt);
745
298c926c 746 local_bh_disable();
ec9f2006 747 compl(&req->base, err);
298c926c 748 local_bh_enable();
81760ea6
HX
749
750 if (err != -EINPROGRESS && refcnt && atomic_dec_and_test(&ctx->refcnt))
751 crypto_free_aead(tfm);
298c926c
AH
752}
753
754static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err)
755{
756 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
757 struct crypto_aead *child = ctx->child;
758 struct aead_request *req;
759
760 req = container_of(areq, struct aead_request, base);
ba3749a7 761 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt);
298c926c
AH
762}
763
764static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err)
765{
766 struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm);
767 struct crypto_aead *child = ctx->child;
768 struct aead_request *req;
769
770 req = container_of(areq, struct aead_request, base);
ba3749a7 771 cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt);
298c926c
AH
772}
773
774static int cryptd_aead_enqueue(struct aead_request *req,
3e3dc25f 775 crypto_completion_t compl)
298c926c
AH
776{
777 struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req);
778 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
779 struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm));
780
781 rctx->complete = req->base.complete;
3e3dc25f 782 req->base.complete = compl;
298c926c
AH
783 return cryptd_enqueue_request(queue, &req->base);
784}
785
786static int cryptd_aead_encrypt_enqueue(struct aead_request *req)
787{
788 return cryptd_aead_enqueue(req, cryptd_aead_encrypt );
789}
790
791static int cryptd_aead_decrypt_enqueue(struct aead_request *req)
792{
793 return cryptd_aead_enqueue(req, cryptd_aead_decrypt );
794}
795
f614e546 796static int cryptd_aead_init_tfm(struct crypto_aead *tfm)
298c926c 797{
f614e546
HX
798 struct aead_instance *inst = aead_alg_instance(tfm);
799 struct aead_instance_ctx *ictx = aead_instance_ctx(inst);
298c926c 800 struct crypto_aead_spawn *spawn = &ictx->aead_spawn;
f614e546 801 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
298c926c
AH
802 struct crypto_aead *cipher;
803
804 cipher = crypto_spawn_aead(spawn);
805 if (IS_ERR(cipher))
806 return PTR_ERR(cipher);
807
298c926c 808 ctx->child = cipher;
ec9f2006
HX
809 crypto_aead_set_reqsize(
810 tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx),
811 crypto_aead_reqsize(cipher)));
298c926c
AH
812 return 0;
813}
814
f614e546 815static void cryptd_aead_exit_tfm(struct crypto_aead *tfm)
298c926c 816{
f614e546 817 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm);
298c926c
AH
818 crypto_free_aead(ctx->child);
819}
820
821static int cryptd_create_aead(struct crypto_template *tmpl,
822 struct rtattr **tb,
823 struct cryptd_queue *queue)
824{
825 struct aead_instance_ctx *ctx;
f614e546
HX
826 struct aead_instance *inst;
827 struct aead_alg *alg;
9b8c456e
HX
828 const char *name;
829 u32 type = 0;
ec9f2006 830 u32 mask = CRYPTO_ALG_ASYNC;
298c926c
AH
831 int err;
832
466a7b9e
SM
833 cryptd_check_internal(tb, &type, &mask);
834
9b8c456e
HX
835 name = crypto_attr_alg_name(tb[1]);
836 if (IS_ERR(name))
837 return PTR_ERR(name);
298c926c 838
9b8c456e
HX
839 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
840 if (!inst)
841 return -ENOMEM;
298c926c 842
f614e546 843 ctx = aead_instance_ctx(inst);
298c926c
AH
844 ctx->queue = queue;
845
f614e546 846 crypto_set_aead_spawn(&ctx->aead_spawn, aead_crypto_instance(inst));
9b8c456e 847 err = crypto_grab_aead(&ctx->aead_spawn, name, type, mask);
298c926c
AH
848 if (err)
849 goto out_free_inst;
850
f614e546
HX
851 alg = crypto_spawn_aead_alg(&ctx->aead_spawn);
852 err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base);
9b8c456e
HX
853 if (err)
854 goto out_drop_aead;
855
f614e546 856 inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC |
5e4b8c1f 857 (alg->base.cra_flags & CRYPTO_ALG_INTERNAL);
f614e546 858 inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx);
298c926c 859
f614e546
HX
860 inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
861 inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
862
863 inst->alg.init = cryptd_aead_init_tfm;
864 inst->alg.exit = cryptd_aead_exit_tfm;
865 inst->alg.setkey = cryptd_aead_setkey;
866 inst->alg.setauthsize = cryptd_aead_setauthsize;
867 inst->alg.encrypt = cryptd_aead_encrypt_enqueue;
868 inst->alg.decrypt = cryptd_aead_decrypt_enqueue;
869
870 err = aead_register_instance(tmpl, inst);
298c926c 871 if (err) {
9b8c456e
HX
872out_drop_aead:
873 crypto_drop_aead(&ctx->aead_spawn);
298c926c
AH
874out_free_inst:
875 kfree(inst);
876 }
298c926c
AH
877 return err;
878}
879
254eff77 880static struct cryptd_queue queue;
124b53d0 881
9cd899a3 882static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb)
124b53d0
HX
883{
884 struct crypto_attr_type *algt;
885
886 algt = crypto_get_attr_type(tb);
887 if (IS_ERR(algt))
9cd899a3 888 return PTR_ERR(algt);
124b53d0
HX
889
890 switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
891 case CRYPTO_ALG_TYPE_BLKCIPHER:
9cd899a3 892 return cryptd_create_blkcipher(tmpl, tb, &queue);
b8a28251 893 case CRYPTO_ALG_TYPE_DIGEST:
9cd899a3 894 return cryptd_create_hash(tmpl, tb, &queue);
298c926c
AH
895 case CRYPTO_ALG_TYPE_AEAD:
896 return cryptd_create_aead(tmpl, tb, &queue);
124b53d0
HX
897 }
898
9cd899a3 899 return -EINVAL;
124b53d0
HX
900}
901
902static void cryptd_free(struct crypto_instance *inst)
903{
904 struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst);
0b535adf 905 struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst);
298c926c 906 struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst);
0b535adf
HX
907
908 switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) {
909 case CRYPTO_ALG_TYPE_AHASH:
910 crypto_drop_shash(&hctx->spawn);
911 kfree(ahash_instance(inst));
912 return;
298c926c 913 case CRYPTO_ALG_TYPE_AEAD:
f614e546
HX
914 crypto_drop_aead(&aead_ctx->aead_spawn);
915 kfree(aead_instance(inst));
298c926c
AH
916 return;
917 default:
918 crypto_drop_spawn(&ctx->spawn);
919 kfree(inst);
0b535adf 920 }
124b53d0
HX
921}
922
923static struct crypto_template cryptd_tmpl = {
924 .name = "cryptd",
9cd899a3 925 .create = cryptd_create,
124b53d0
HX
926 .free = cryptd_free,
927 .module = THIS_MODULE,
928};
929
1cac2cbc
HY
930struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name,
931 u32 type, u32 mask)
932{
933 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
81760ea6 934 struct cryptd_blkcipher_ctx *ctx;
505fd21d 935 struct crypto_tfm *tfm;
1cac2cbc
HY
936
937 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
938 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
939 return ERR_PTR(-EINVAL);
c012a79d 940 type = crypto_skcipher_type(type);
505fd21d
HY
941 mask &= ~CRYPTO_ALG_TYPE_MASK;
942 mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK);
943 tfm = crypto_alloc_base(cryptd_alg_name, type, mask);
1cac2cbc
HY
944 if (IS_ERR(tfm))
945 return ERR_CAST(tfm);
505fd21d
HY
946 if (tfm->__crt_alg->cra_module != THIS_MODULE) {
947 crypto_free_tfm(tfm);
1cac2cbc
HY
948 return ERR_PTR(-EINVAL);
949 }
950
81760ea6
HX
951 ctx = crypto_tfm_ctx(tfm);
952 atomic_set(&ctx->refcnt, 1);
953
505fd21d 954 return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm));
1cac2cbc
HY
955}
956EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher);
957
958struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm)
959{
960 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
961 return ctx->child;
962}
963EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child);
964
81760ea6
HX
965bool cryptd_ablkcipher_queued(struct cryptd_ablkcipher *tfm)
966{
967 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
968
969 return atomic_read(&ctx->refcnt) - 1;
970}
971EXPORT_SYMBOL_GPL(cryptd_ablkcipher_queued);
972
1cac2cbc
HY
973void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm)
974{
81760ea6
HX
975 struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base);
976
977 if (atomic_dec_and_test(&ctx->refcnt))
978 crypto_free_ablkcipher(&tfm->base);
1cac2cbc
HY
979}
980EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher);
981
ace13663
HY
982struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name,
983 u32 type, u32 mask)
984{
985 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
81760ea6 986 struct cryptd_hash_ctx *ctx;
ace13663
HY
987 struct crypto_ahash *tfm;
988
989 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
990 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
991 return ERR_PTR(-EINVAL);
992 tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask);
993 if (IS_ERR(tfm))
994 return ERR_CAST(tfm);
995 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
996 crypto_free_ahash(tfm);
997 return ERR_PTR(-EINVAL);
998 }
999
81760ea6
HX
1000 ctx = crypto_ahash_ctx(tfm);
1001 atomic_set(&ctx->refcnt, 1);
1002
ace13663
HY
1003 return __cryptd_ahash_cast(tfm);
1004}
1005EXPORT_SYMBOL_GPL(cryptd_alloc_ahash);
1006
1007struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm)
1008{
1009 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1010
1011 return ctx->child;
1012}
1013EXPORT_SYMBOL_GPL(cryptd_ahash_child);
1014
0e1227d3
HY
1015struct shash_desc *cryptd_shash_desc(struct ahash_request *req)
1016{
1017 struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req);
1018 return &rctx->desc;
1019}
1020EXPORT_SYMBOL_GPL(cryptd_shash_desc);
1021
81760ea6
HX
1022bool cryptd_ahash_queued(struct cryptd_ahash *tfm)
1023{
1024 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1025
1026 return atomic_read(&ctx->refcnt) - 1;
1027}
1028EXPORT_SYMBOL_GPL(cryptd_ahash_queued);
1029
ace13663
HY
1030void cryptd_free_ahash(struct cryptd_ahash *tfm)
1031{
81760ea6
HX
1032 struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base);
1033
1034 if (atomic_dec_and_test(&ctx->refcnt))
1035 crypto_free_ahash(&tfm->base);
ace13663
HY
1036}
1037EXPORT_SYMBOL_GPL(cryptd_free_ahash);
1038
298c926c
AH
1039struct cryptd_aead *cryptd_alloc_aead(const char *alg_name,
1040 u32 type, u32 mask)
1041{
1042 char cryptd_alg_name[CRYPTO_MAX_ALG_NAME];
81760ea6 1043 struct cryptd_aead_ctx *ctx;
298c926c
AH
1044 struct crypto_aead *tfm;
1045
1046 if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME,
1047 "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME)
1048 return ERR_PTR(-EINVAL);
1049 tfm = crypto_alloc_aead(cryptd_alg_name, type, mask);
1050 if (IS_ERR(tfm))
1051 return ERR_CAST(tfm);
1052 if (tfm->base.__crt_alg->cra_module != THIS_MODULE) {
1053 crypto_free_aead(tfm);
1054 return ERR_PTR(-EINVAL);
1055 }
81760ea6
HX
1056
1057 ctx = crypto_aead_ctx(tfm);
1058 atomic_set(&ctx->refcnt, 1);
1059
298c926c
AH
1060 return __cryptd_aead_cast(tfm);
1061}
1062EXPORT_SYMBOL_GPL(cryptd_alloc_aead);
1063
1064struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm)
1065{
1066 struct cryptd_aead_ctx *ctx;
1067 ctx = crypto_aead_ctx(&tfm->base);
1068 return ctx->child;
1069}
1070EXPORT_SYMBOL_GPL(cryptd_aead_child);
1071
81760ea6
HX
1072bool cryptd_aead_queued(struct cryptd_aead *tfm)
1073{
1074 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1075
1076 return atomic_read(&ctx->refcnt) - 1;
1077}
1078EXPORT_SYMBOL_GPL(cryptd_aead_queued);
1079
298c926c
AH
1080void cryptd_free_aead(struct cryptd_aead *tfm)
1081{
81760ea6
HX
1082 struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base);
1083
1084 if (atomic_dec_and_test(&ctx->refcnt))
1085 crypto_free_aead(&tfm->base);
298c926c
AH
1086}
1087EXPORT_SYMBOL_GPL(cryptd_free_aead);
1088
124b53d0
HX
1089static int __init cryptd_init(void)
1090{
1091 int err;
1092
254eff77 1093 err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN);
124b53d0
HX
1094 if (err)
1095 return err;
1096
1097 err = crypto_register_template(&cryptd_tmpl);
1098 if (err)
254eff77 1099 cryptd_fini_queue(&queue);
124b53d0
HX
1100
1101 return err;
1102}
1103
1104static void __exit cryptd_exit(void)
1105{
254eff77 1106 cryptd_fini_queue(&queue);
124b53d0
HX
1107 crypto_unregister_template(&cryptd_tmpl);
1108}
1109
b2bac6ac 1110subsys_initcall(cryptd_init);
124b53d0
HX
1111module_exit(cryptd_exit);
1112
1113MODULE_LICENSE("GPL");
1114MODULE_DESCRIPTION("Software async crypto daemon");
4943ba16 1115MODULE_ALIAS_CRYPTO("cryptd");