]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Software async crypto daemon. | |
3 | * | |
4 | * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> | |
5 | * | |
6 | * Added AEAD support to cryptd. | |
7 | * Authors: Tadeusz Struk (tadeusz.struk@intel.com) | |
8 | * Adrian Hoban <adrian.hoban@intel.com> | |
9 | * Gabriele Paoloni <gabriele.paoloni@intel.com> | |
10 | * Aidan O'Mahony (aidan.o.mahony@intel.com) | |
11 | * Copyright (c) 2010, Intel Corporation. | |
12 | * | |
13 | * This program is free software; you can redistribute it and/or modify it | |
14 | * under the terms of the GNU General Public License as published by the Free | |
15 | * Software Foundation; either version 2 of the License, or (at your option) | |
16 | * any later version. | |
17 | * | |
18 | */ | |
19 | ||
20 | #include <crypto/algapi.h> | |
21 | #include <crypto/internal/hash.h> | |
22 | #include <crypto/internal/aead.h> | |
23 | #include <crypto/cryptd.h> | |
24 | #include <crypto/crypto_wq.h> | |
25 | #include <linux/err.h> | |
26 | #include <linux/init.h> | |
27 | #include <linux/kernel.h> | |
28 | #include <linux/list.h> | |
29 | #include <linux/module.h> | |
30 | #include <linux/scatterlist.h> | |
31 | #include <linux/sched.h> | |
32 | #include <linux/slab.h> | |
33 | ||
34 | #define CRYPTD_MAX_CPU_QLEN 100 | |
35 | ||
36 | struct cryptd_cpu_queue { | |
37 | struct crypto_queue queue; | |
38 | struct work_struct work; | |
39 | }; | |
40 | ||
41 | struct cryptd_queue { | |
42 | struct cryptd_cpu_queue __percpu *cpu_queue; | |
43 | }; | |
44 | ||
45 | struct cryptd_instance_ctx { | |
46 | struct crypto_spawn spawn; | |
47 | struct cryptd_queue *queue; | |
48 | }; | |
49 | ||
50 | struct hashd_instance_ctx { | |
51 | struct crypto_shash_spawn spawn; | |
52 | struct cryptd_queue *queue; | |
53 | }; | |
54 | ||
55 | struct aead_instance_ctx { | |
56 | struct crypto_aead_spawn aead_spawn; | |
57 | struct cryptd_queue *queue; | |
58 | }; | |
59 | ||
60 | struct cryptd_blkcipher_ctx { | |
61 | struct crypto_blkcipher *child; | |
62 | }; | |
63 | ||
64 | struct cryptd_blkcipher_request_ctx { | |
65 | crypto_completion_t complete; | |
66 | }; | |
67 | ||
68 | struct cryptd_hash_ctx { | |
69 | struct crypto_shash *child; | |
70 | }; | |
71 | ||
72 | struct cryptd_hash_request_ctx { | |
73 | crypto_completion_t complete; | |
74 | struct shash_desc desc; | |
75 | }; | |
76 | ||
77 | struct cryptd_aead_ctx { | |
78 | struct crypto_aead *child; | |
79 | }; | |
80 | ||
81 | struct cryptd_aead_request_ctx { | |
82 | crypto_completion_t complete; | |
83 | }; | |
84 | ||
85 | static void cryptd_queue_worker(struct work_struct *work); | |
86 | ||
87 | static int cryptd_init_queue(struct cryptd_queue *queue, | |
88 | unsigned int max_cpu_qlen) | |
89 | { | |
90 | int cpu; | |
91 | struct cryptd_cpu_queue *cpu_queue; | |
92 | ||
93 | queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); | |
94 | if (!queue->cpu_queue) | |
95 | return -ENOMEM; | |
96 | for_each_possible_cpu(cpu) { | |
97 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
98 | crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); | |
99 | INIT_WORK(&cpu_queue->work, cryptd_queue_worker); | |
100 | } | |
101 | return 0; | |
102 | } | |
103 | ||
104 | static void cryptd_fini_queue(struct cryptd_queue *queue) | |
105 | { | |
106 | int cpu; | |
107 | struct cryptd_cpu_queue *cpu_queue; | |
108 | ||
109 | for_each_possible_cpu(cpu) { | |
110 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
111 | BUG_ON(cpu_queue->queue.qlen); | |
112 | } | |
113 | free_percpu(queue->cpu_queue); | |
114 | } | |
115 | ||
116 | static int cryptd_enqueue_request(struct cryptd_queue *queue, | |
117 | struct crypto_async_request *request) | |
118 | { | |
119 | int cpu, err; | |
120 | struct cryptd_cpu_queue *cpu_queue; | |
121 | ||
122 | cpu = get_cpu(); | |
123 | cpu_queue = this_cpu_ptr(queue->cpu_queue); | |
124 | err = crypto_enqueue_request(&cpu_queue->queue, request); | |
125 | queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); | |
126 | put_cpu(); | |
127 | ||
128 | return err; | |
129 | } | |
130 | ||
131 | /* Called in workqueue context, do one real cryption work (via | |
132 | * req->complete) and reschedule itself if there are more work to | |
133 | * do. */ | |
134 | static void cryptd_queue_worker(struct work_struct *work) | |
135 | { | |
136 | struct cryptd_cpu_queue *cpu_queue; | |
137 | struct crypto_async_request *req, *backlog; | |
138 | ||
139 | cpu_queue = container_of(work, struct cryptd_cpu_queue, work); | |
140 | /* | |
141 | * Only handle one request at a time to avoid hogging crypto workqueue. | |
142 | * preempt_disable/enable is used to prevent being preempted by | |
143 | * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent | |
144 | * cryptd_enqueue_request() being accessed from software interrupts. | |
145 | */ | |
146 | local_bh_disable(); | |
147 | preempt_disable(); | |
148 | backlog = crypto_get_backlog(&cpu_queue->queue); | |
149 | req = crypto_dequeue_request(&cpu_queue->queue); | |
150 | preempt_enable(); | |
151 | local_bh_enable(); | |
152 | ||
153 | if (!req) | |
154 | return; | |
155 | ||
156 | if (backlog) | |
157 | backlog->complete(backlog, -EINPROGRESS); | |
158 | req->complete(req, 0); | |
159 | ||
160 | if (cpu_queue->queue.qlen) | |
161 | queue_work(kcrypto_wq, &cpu_queue->work); | |
162 | } | |
163 | ||
164 | static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) | |
165 | { | |
166 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
167 | struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
168 | return ictx->queue; | |
169 | } | |
170 | ||
171 | static int cryptd_blkcipher_setkey(struct crypto_ablkcipher *parent, | |
172 | const u8 *key, unsigned int keylen) | |
173 | { | |
174 | struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(parent); | |
175 | struct crypto_blkcipher *child = ctx->child; | |
176 | int err; | |
177 | ||
178 | crypto_blkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); | |
179 | crypto_blkcipher_set_flags(child, crypto_ablkcipher_get_flags(parent) & | |
180 | CRYPTO_TFM_REQ_MASK); | |
181 | err = crypto_blkcipher_setkey(child, key, keylen); | |
182 | crypto_ablkcipher_set_flags(parent, crypto_blkcipher_get_flags(child) & | |
183 | CRYPTO_TFM_RES_MASK); | |
184 | return err; | |
185 | } | |
186 | ||
187 | static void cryptd_blkcipher_crypt(struct ablkcipher_request *req, | |
188 | struct crypto_blkcipher *child, | |
189 | int err, | |
190 | int (*crypt)(struct blkcipher_desc *desc, | |
191 | struct scatterlist *dst, | |
192 | struct scatterlist *src, | |
193 | unsigned int len)) | |
194 | { | |
195 | struct cryptd_blkcipher_request_ctx *rctx; | |
196 | struct blkcipher_desc desc; | |
197 | ||
198 | rctx = ablkcipher_request_ctx(req); | |
199 | ||
200 | if (unlikely(err == -EINPROGRESS)) | |
201 | goto out; | |
202 | ||
203 | desc.tfm = child; | |
204 | desc.info = req->info; | |
205 | desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; | |
206 | ||
207 | err = crypt(&desc, req->dst, req->src, req->nbytes); | |
208 | ||
209 | req->base.complete = rctx->complete; | |
210 | ||
211 | out: | |
212 | local_bh_disable(); | |
213 | rctx->complete(&req->base, err); | |
214 | local_bh_enable(); | |
215 | } | |
216 | ||
217 | static void cryptd_blkcipher_encrypt(struct crypto_async_request *req, int err) | |
218 | { | |
219 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); | |
220 | struct crypto_blkcipher *child = ctx->child; | |
221 | ||
222 | cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, | |
223 | crypto_blkcipher_crt(child)->encrypt); | |
224 | } | |
225 | ||
226 | static void cryptd_blkcipher_decrypt(struct crypto_async_request *req, int err) | |
227 | { | |
228 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(req->tfm); | |
229 | struct crypto_blkcipher *child = ctx->child; | |
230 | ||
231 | cryptd_blkcipher_crypt(ablkcipher_request_cast(req), child, err, | |
232 | crypto_blkcipher_crt(child)->decrypt); | |
233 | } | |
234 | ||
235 | static int cryptd_blkcipher_enqueue(struct ablkcipher_request *req, | |
236 | crypto_completion_t compl) | |
237 | { | |
238 | struct cryptd_blkcipher_request_ctx *rctx = ablkcipher_request_ctx(req); | |
239 | struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); | |
240 | struct cryptd_queue *queue; | |
241 | ||
242 | queue = cryptd_get_queue(crypto_ablkcipher_tfm(tfm)); | |
243 | rctx->complete = req->base.complete; | |
244 | req->base.complete = compl; | |
245 | ||
246 | return cryptd_enqueue_request(queue, &req->base); | |
247 | } | |
248 | ||
249 | static int cryptd_blkcipher_encrypt_enqueue(struct ablkcipher_request *req) | |
250 | { | |
251 | return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_encrypt); | |
252 | } | |
253 | ||
254 | static int cryptd_blkcipher_decrypt_enqueue(struct ablkcipher_request *req) | |
255 | { | |
256 | return cryptd_blkcipher_enqueue(req, cryptd_blkcipher_decrypt); | |
257 | } | |
258 | ||
259 | static int cryptd_blkcipher_init_tfm(struct crypto_tfm *tfm) | |
260 | { | |
261 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
262 | struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
263 | struct crypto_spawn *spawn = &ictx->spawn; | |
264 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); | |
265 | struct crypto_blkcipher *cipher; | |
266 | ||
267 | cipher = crypto_spawn_blkcipher(spawn); | |
268 | if (IS_ERR(cipher)) | |
269 | return PTR_ERR(cipher); | |
270 | ||
271 | ctx->child = cipher; | |
272 | tfm->crt_ablkcipher.reqsize = | |
273 | sizeof(struct cryptd_blkcipher_request_ctx); | |
274 | return 0; | |
275 | } | |
276 | ||
277 | static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm) | |
278 | { | |
279 | struct cryptd_blkcipher_ctx *ctx = crypto_tfm_ctx(tfm); | |
280 | ||
281 | crypto_free_blkcipher(ctx->child); | |
282 | } | |
283 | ||
284 | static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, | |
285 | unsigned int tail) | |
286 | { | |
287 | char *p; | |
288 | struct crypto_instance *inst; | |
289 | int err; | |
290 | ||
291 | p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL); | |
292 | if (!p) | |
293 | return ERR_PTR(-ENOMEM); | |
294 | ||
295 | inst = (void *)(p + head); | |
296 | ||
297 | err = -ENAMETOOLONG; | |
298 | if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, | |
299 | "cryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) | |
300 | goto out_free_inst; | |
301 | ||
302 | memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); | |
303 | ||
304 | inst->alg.cra_priority = alg->cra_priority + 50; | |
305 | inst->alg.cra_blocksize = alg->cra_blocksize; | |
306 | inst->alg.cra_alignmask = alg->cra_alignmask; | |
307 | ||
308 | out: | |
309 | return p; | |
310 | ||
311 | out_free_inst: | |
312 | kfree(p); | |
313 | p = ERR_PTR(err); | |
314 | goto out; | |
315 | } | |
316 | ||
317 | static int cryptd_create_blkcipher(struct crypto_template *tmpl, | |
318 | struct rtattr **tb, | |
319 | struct cryptd_queue *queue) | |
320 | { | |
321 | struct cryptd_instance_ctx *ctx; | |
322 | struct crypto_instance *inst; | |
323 | struct crypto_alg *alg; | |
324 | int err; | |
325 | ||
326 | alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_BLKCIPHER, | |
327 | CRYPTO_ALG_TYPE_MASK); | |
328 | if (IS_ERR(alg)) | |
329 | return PTR_ERR(alg); | |
330 | ||
331 | inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx)); | |
332 | err = PTR_ERR(inst); | |
333 | if (IS_ERR(inst)) | |
334 | goto out_put_alg; | |
335 | ||
336 | ctx = crypto_instance_ctx(inst); | |
337 | ctx->queue = queue; | |
338 | ||
339 | err = crypto_init_spawn(&ctx->spawn, alg, inst, | |
340 | CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); | |
341 | if (err) | |
342 | goto out_free_inst; | |
343 | ||
344 | inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; | |
345 | inst->alg.cra_type = &crypto_ablkcipher_type; | |
346 | ||
347 | inst->alg.cra_ablkcipher.ivsize = alg->cra_blkcipher.ivsize; | |
348 | inst->alg.cra_ablkcipher.min_keysize = alg->cra_blkcipher.min_keysize; | |
349 | inst->alg.cra_ablkcipher.max_keysize = alg->cra_blkcipher.max_keysize; | |
350 | ||
351 | inst->alg.cra_ablkcipher.geniv = alg->cra_blkcipher.geniv; | |
352 | ||
353 | inst->alg.cra_ctxsize = sizeof(struct cryptd_blkcipher_ctx); | |
354 | ||
355 | inst->alg.cra_init = cryptd_blkcipher_init_tfm; | |
356 | inst->alg.cra_exit = cryptd_blkcipher_exit_tfm; | |
357 | ||
358 | inst->alg.cra_ablkcipher.setkey = cryptd_blkcipher_setkey; | |
359 | inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue; | |
360 | inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue; | |
361 | ||
362 | err = crypto_register_instance(tmpl, inst); | |
363 | if (err) { | |
364 | crypto_drop_spawn(&ctx->spawn); | |
365 | out_free_inst: | |
366 | kfree(inst); | |
367 | } | |
368 | ||
369 | out_put_alg: | |
370 | crypto_mod_put(alg); | |
371 | return err; | |
372 | } | |
373 | ||
374 | static int cryptd_hash_init_tfm(struct crypto_tfm *tfm) | |
375 | { | |
376 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
377 | struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
378 | struct crypto_shash_spawn *spawn = &ictx->spawn; | |
379 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); | |
380 | struct crypto_shash *hash; | |
381 | ||
382 | hash = crypto_spawn_shash(spawn); | |
383 | if (IS_ERR(hash)) | |
384 | return PTR_ERR(hash); | |
385 | ||
386 | ctx->child = hash; | |
387 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), | |
388 | sizeof(struct cryptd_hash_request_ctx) + | |
389 | crypto_shash_descsize(hash)); | |
390 | return 0; | |
391 | } | |
392 | ||
393 | static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm) | |
394 | { | |
395 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); | |
396 | ||
397 | crypto_free_shash(ctx->child); | |
398 | } | |
399 | ||
400 | static int cryptd_hash_setkey(struct crypto_ahash *parent, | |
401 | const u8 *key, unsigned int keylen) | |
402 | { | |
403 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); | |
404 | struct crypto_shash *child = ctx->child; | |
405 | int err; | |
406 | ||
407 | crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); | |
408 | crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & | |
409 | CRYPTO_TFM_REQ_MASK); | |
410 | err = crypto_shash_setkey(child, key, keylen); | |
411 | crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) & | |
412 | CRYPTO_TFM_RES_MASK); | |
413 | return err; | |
414 | } | |
415 | ||
416 | static int cryptd_hash_enqueue(struct ahash_request *req, | |
417 | crypto_completion_t compl) | |
418 | { | |
419 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
420 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
421 | struct cryptd_queue *queue = | |
422 | cryptd_get_queue(crypto_ahash_tfm(tfm)); | |
423 | ||
424 | rctx->complete = req->base.complete; | |
425 | req->base.complete = compl; | |
426 | ||
427 | return cryptd_enqueue_request(queue, &req->base); | |
428 | } | |
429 | ||
430 | static void cryptd_hash_init(struct crypto_async_request *req_async, int err) | |
431 | { | |
432 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); | |
433 | struct crypto_shash *child = ctx->child; | |
434 | struct ahash_request *req = ahash_request_cast(req_async); | |
435 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
436 | struct shash_desc *desc = &rctx->desc; | |
437 | ||
438 | if (unlikely(err == -EINPROGRESS)) | |
439 | goto out; | |
440 | ||
441 | desc->tfm = child; | |
442 | desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; | |
443 | ||
444 | err = crypto_shash_init(desc); | |
445 | ||
446 | req->base.complete = rctx->complete; | |
447 | ||
448 | out: | |
449 | local_bh_disable(); | |
450 | rctx->complete(&req->base, err); | |
451 | local_bh_enable(); | |
452 | } | |
453 | ||
454 | static int cryptd_hash_init_enqueue(struct ahash_request *req) | |
455 | { | |
456 | return cryptd_hash_enqueue(req, cryptd_hash_init); | |
457 | } | |
458 | ||
459 | static void cryptd_hash_update(struct crypto_async_request *req_async, int err) | |
460 | { | |
461 | struct ahash_request *req = ahash_request_cast(req_async); | |
462 | struct cryptd_hash_request_ctx *rctx; | |
463 | ||
464 | rctx = ahash_request_ctx(req); | |
465 | ||
466 | if (unlikely(err == -EINPROGRESS)) | |
467 | goto out; | |
468 | ||
469 | err = shash_ahash_update(req, &rctx->desc); | |
470 | ||
471 | req->base.complete = rctx->complete; | |
472 | ||
473 | out: | |
474 | local_bh_disable(); | |
475 | rctx->complete(&req->base, err); | |
476 | local_bh_enable(); | |
477 | } | |
478 | ||
479 | static int cryptd_hash_update_enqueue(struct ahash_request *req) | |
480 | { | |
481 | return cryptd_hash_enqueue(req, cryptd_hash_update); | |
482 | } | |
483 | ||
484 | static void cryptd_hash_final(struct crypto_async_request *req_async, int err) | |
485 | { | |
486 | struct ahash_request *req = ahash_request_cast(req_async); | |
487 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
488 | ||
489 | if (unlikely(err == -EINPROGRESS)) | |
490 | goto out; | |
491 | ||
492 | err = crypto_shash_final(&rctx->desc, req->result); | |
493 | ||
494 | req->base.complete = rctx->complete; | |
495 | ||
496 | out: | |
497 | local_bh_disable(); | |
498 | rctx->complete(&req->base, err); | |
499 | local_bh_enable(); | |
500 | } | |
501 | ||
502 | static int cryptd_hash_final_enqueue(struct ahash_request *req) | |
503 | { | |
504 | return cryptd_hash_enqueue(req, cryptd_hash_final); | |
505 | } | |
506 | ||
507 | static void cryptd_hash_finup(struct crypto_async_request *req_async, int err) | |
508 | { | |
509 | struct ahash_request *req = ahash_request_cast(req_async); | |
510 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
511 | ||
512 | if (unlikely(err == -EINPROGRESS)) | |
513 | goto out; | |
514 | ||
515 | err = shash_ahash_finup(req, &rctx->desc); | |
516 | ||
517 | req->base.complete = rctx->complete; | |
518 | ||
519 | out: | |
520 | local_bh_disable(); | |
521 | rctx->complete(&req->base, err); | |
522 | local_bh_enable(); | |
523 | } | |
524 | ||
525 | static int cryptd_hash_finup_enqueue(struct ahash_request *req) | |
526 | { | |
527 | return cryptd_hash_enqueue(req, cryptd_hash_finup); | |
528 | } | |
529 | ||
530 | static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) | |
531 | { | |
532 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); | |
533 | struct crypto_shash *child = ctx->child; | |
534 | struct ahash_request *req = ahash_request_cast(req_async); | |
535 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
536 | struct shash_desc *desc = &rctx->desc; | |
537 | ||
538 | if (unlikely(err == -EINPROGRESS)) | |
539 | goto out; | |
540 | ||
541 | desc->tfm = child; | |
542 | desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; | |
543 | ||
544 | err = shash_ahash_digest(req, desc); | |
545 | ||
546 | req->base.complete = rctx->complete; | |
547 | ||
548 | out: | |
549 | local_bh_disable(); | |
550 | rctx->complete(&req->base, err); | |
551 | local_bh_enable(); | |
552 | } | |
553 | ||
554 | static int cryptd_hash_digest_enqueue(struct ahash_request *req) | |
555 | { | |
556 | return cryptd_hash_enqueue(req, cryptd_hash_digest); | |
557 | } | |
558 | ||
559 | static int cryptd_hash_export(struct ahash_request *req, void *out) | |
560 | { | |
561 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
562 | ||
563 | return crypto_shash_export(&rctx->desc, out); | |
564 | } | |
565 | ||
566 | static int cryptd_hash_import(struct ahash_request *req, const void *in) | |
567 | { | |
568 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
569 | ||
570 | return crypto_shash_import(&rctx->desc, in); | |
571 | } | |
572 | ||
573 | static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, | |
574 | struct cryptd_queue *queue) | |
575 | { | |
576 | struct hashd_instance_ctx *ctx; | |
577 | struct ahash_instance *inst; | |
578 | struct shash_alg *salg; | |
579 | struct crypto_alg *alg; | |
580 | int err; | |
581 | ||
582 | salg = shash_attr_alg(tb[1], 0, 0); | |
583 | if (IS_ERR(salg)) | |
584 | return PTR_ERR(salg); | |
585 | ||
586 | alg = &salg->base; | |
587 | inst = cryptd_alloc_instance(alg, ahash_instance_headroom(), | |
588 | sizeof(*ctx)); | |
589 | err = PTR_ERR(inst); | |
590 | if (IS_ERR(inst)) | |
591 | goto out_put_alg; | |
592 | ||
593 | ctx = ahash_instance_ctx(inst); | |
594 | ctx->queue = queue; | |
595 | ||
596 | err = crypto_init_shash_spawn(&ctx->spawn, salg, | |
597 | ahash_crypto_instance(inst)); | |
598 | if (err) | |
599 | goto out_free_inst; | |
600 | ||
601 | inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC; | |
602 | ||
603 | inst->alg.halg.digestsize = salg->digestsize; | |
604 | inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); | |
605 | ||
606 | inst->alg.halg.base.cra_init = cryptd_hash_init_tfm; | |
607 | inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm; | |
608 | ||
609 | inst->alg.init = cryptd_hash_init_enqueue; | |
610 | inst->alg.update = cryptd_hash_update_enqueue; | |
611 | inst->alg.final = cryptd_hash_final_enqueue; | |
612 | inst->alg.finup = cryptd_hash_finup_enqueue; | |
613 | inst->alg.export = cryptd_hash_export; | |
614 | inst->alg.import = cryptd_hash_import; | |
615 | inst->alg.setkey = cryptd_hash_setkey; | |
616 | inst->alg.digest = cryptd_hash_digest_enqueue; | |
617 | ||
618 | err = ahash_register_instance(tmpl, inst); | |
619 | if (err) { | |
620 | crypto_drop_shash(&ctx->spawn); | |
621 | out_free_inst: | |
622 | kfree(inst); | |
623 | } | |
624 | ||
625 | out_put_alg: | |
626 | crypto_mod_put(alg); | |
627 | return err; | |
628 | } | |
629 | ||
630 | static void cryptd_aead_crypt(struct aead_request *req, | |
631 | struct crypto_aead *child, | |
632 | int err, | |
633 | int (*crypt)(struct aead_request *req)) | |
634 | { | |
635 | struct cryptd_aead_request_ctx *rctx; | |
636 | rctx = aead_request_ctx(req); | |
637 | ||
638 | if (unlikely(err == -EINPROGRESS)) | |
639 | goto out; | |
640 | aead_request_set_tfm(req, child); | |
641 | err = crypt( req ); | |
642 | req->base.complete = rctx->complete; | |
643 | out: | |
644 | local_bh_disable(); | |
645 | rctx->complete(&req->base, err); | |
646 | local_bh_enable(); | |
647 | } | |
648 | ||
649 | static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err) | |
650 | { | |
651 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
652 | struct crypto_aead *child = ctx->child; | |
653 | struct aead_request *req; | |
654 | ||
655 | req = container_of(areq, struct aead_request, base); | |
656 | cryptd_aead_crypt(req, child, err, crypto_aead_crt(child)->encrypt); | |
657 | } | |
658 | ||
659 | static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err) | |
660 | { | |
661 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
662 | struct crypto_aead *child = ctx->child; | |
663 | struct aead_request *req; | |
664 | ||
665 | req = container_of(areq, struct aead_request, base); | |
666 | cryptd_aead_crypt(req, child, err, crypto_aead_crt(child)->decrypt); | |
667 | } | |
668 | ||
669 | static int cryptd_aead_enqueue(struct aead_request *req, | |
670 | crypto_completion_t compl) | |
671 | { | |
672 | struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req); | |
673 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); | |
674 | struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm)); | |
675 | ||
676 | rctx->complete = req->base.complete; | |
677 | req->base.complete = compl; | |
678 | return cryptd_enqueue_request(queue, &req->base); | |
679 | } | |
680 | ||
681 | static int cryptd_aead_encrypt_enqueue(struct aead_request *req) | |
682 | { | |
683 | return cryptd_aead_enqueue(req, cryptd_aead_encrypt ); | |
684 | } | |
685 | ||
686 | static int cryptd_aead_decrypt_enqueue(struct aead_request *req) | |
687 | { | |
688 | return cryptd_aead_enqueue(req, cryptd_aead_decrypt ); | |
689 | } | |
690 | ||
691 | static int cryptd_aead_init_tfm(struct crypto_tfm *tfm) | |
692 | { | |
693 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
694 | struct aead_instance_ctx *ictx = crypto_instance_ctx(inst); | |
695 | struct crypto_aead_spawn *spawn = &ictx->aead_spawn; | |
696 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(tfm); | |
697 | struct crypto_aead *cipher; | |
698 | ||
699 | cipher = crypto_spawn_aead(spawn); | |
700 | if (IS_ERR(cipher)) | |
701 | return PTR_ERR(cipher); | |
702 | ||
703 | crypto_aead_set_flags(cipher, CRYPTO_TFM_REQ_MAY_SLEEP); | |
704 | ctx->child = cipher; | |
705 | tfm->crt_aead.reqsize = sizeof(struct cryptd_aead_request_ctx); | |
706 | return 0; | |
707 | } | |
708 | ||
709 | static void cryptd_aead_exit_tfm(struct crypto_tfm *tfm) | |
710 | { | |
711 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(tfm); | |
712 | crypto_free_aead(ctx->child); | |
713 | } | |
714 | ||
715 | static int cryptd_create_aead(struct crypto_template *tmpl, | |
716 | struct rtattr **tb, | |
717 | struct cryptd_queue *queue) | |
718 | { | |
719 | struct aead_instance_ctx *ctx; | |
720 | struct crypto_instance *inst; | |
721 | struct crypto_alg *alg; | |
722 | int err; | |
723 | ||
724 | alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_AEAD, | |
725 | CRYPTO_ALG_TYPE_MASK); | |
726 | if (IS_ERR(alg)) | |
727 | return PTR_ERR(alg); | |
728 | ||
729 | inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx)); | |
730 | err = PTR_ERR(inst); | |
731 | if (IS_ERR(inst)) | |
732 | goto out_put_alg; | |
733 | ||
734 | ctx = crypto_instance_ctx(inst); | |
735 | ctx->queue = queue; | |
736 | ||
737 | err = crypto_init_spawn(&ctx->aead_spawn.base, alg, inst, | |
738 | CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); | |
739 | if (err) | |
740 | goto out_free_inst; | |
741 | ||
742 | inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; | |
743 | inst->alg.cra_type = alg->cra_type; | |
744 | inst->alg.cra_ctxsize = sizeof(struct cryptd_aead_ctx); | |
745 | inst->alg.cra_init = cryptd_aead_init_tfm; | |
746 | inst->alg.cra_exit = cryptd_aead_exit_tfm; | |
747 | inst->alg.cra_aead.setkey = alg->cra_aead.setkey; | |
748 | inst->alg.cra_aead.setauthsize = alg->cra_aead.setauthsize; | |
749 | inst->alg.cra_aead.geniv = alg->cra_aead.geniv; | |
750 | inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize; | |
751 | inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize; | |
752 | inst->alg.cra_aead.encrypt = cryptd_aead_encrypt_enqueue; | |
753 | inst->alg.cra_aead.decrypt = cryptd_aead_decrypt_enqueue; | |
754 | inst->alg.cra_aead.givencrypt = alg->cra_aead.givencrypt; | |
755 | inst->alg.cra_aead.givdecrypt = alg->cra_aead.givdecrypt; | |
756 | ||
757 | err = crypto_register_instance(tmpl, inst); | |
758 | if (err) { | |
759 | crypto_drop_spawn(&ctx->aead_spawn.base); | |
760 | out_free_inst: | |
761 | kfree(inst); | |
762 | } | |
763 | out_put_alg: | |
764 | crypto_mod_put(alg); | |
765 | return err; | |
766 | } | |
767 | ||
768 | static struct cryptd_queue queue; | |
769 | ||
770 | static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) | |
771 | { | |
772 | struct crypto_attr_type *algt; | |
773 | ||
774 | algt = crypto_get_attr_type(tb); | |
775 | if (IS_ERR(algt)) | |
776 | return PTR_ERR(algt); | |
777 | ||
778 | switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { | |
779 | case CRYPTO_ALG_TYPE_BLKCIPHER: | |
780 | return cryptd_create_blkcipher(tmpl, tb, &queue); | |
781 | case CRYPTO_ALG_TYPE_DIGEST: | |
782 | return cryptd_create_hash(tmpl, tb, &queue); | |
783 | case CRYPTO_ALG_TYPE_AEAD: | |
784 | return cryptd_create_aead(tmpl, tb, &queue); | |
785 | } | |
786 | ||
787 | return -EINVAL; | |
788 | } | |
789 | ||
790 | static void cryptd_free(struct crypto_instance *inst) | |
791 | { | |
792 | struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst); | |
793 | struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst); | |
794 | struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst); | |
795 | ||
796 | switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { | |
797 | case CRYPTO_ALG_TYPE_AHASH: | |
798 | crypto_drop_shash(&hctx->spawn); | |
799 | kfree(ahash_instance(inst)); | |
800 | return; | |
801 | case CRYPTO_ALG_TYPE_AEAD: | |
802 | crypto_drop_spawn(&aead_ctx->aead_spawn.base); | |
803 | kfree(inst); | |
804 | return; | |
805 | default: | |
806 | crypto_drop_spawn(&ctx->spawn); | |
807 | kfree(inst); | |
808 | } | |
809 | } | |
810 | ||
811 | static struct crypto_template cryptd_tmpl = { | |
812 | .name = "cryptd", | |
813 | .create = cryptd_create, | |
814 | .free = cryptd_free, | |
815 | .module = THIS_MODULE, | |
816 | }; | |
817 | ||
818 | struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, | |
819 | u32 type, u32 mask) | |
820 | { | |
821 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
822 | struct crypto_tfm *tfm; | |
823 | ||
824 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
825 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
826 | return ERR_PTR(-EINVAL); | |
827 | type &= ~(CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_GENIV); | |
828 | type |= CRYPTO_ALG_TYPE_BLKCIPHER; | |
829 | mask &= ~CRYPTO_ALG_TYPE_MASK; | |
830 | mask |= (CRYPTO_ALG_GENIV | CRYPTO_ALG_TYPE_BLKCIPHER_MASK); | |
831 | tfm = crypto_alloc_base(cryptd_alg_name, type, mask); | |
832 | if (IS_ERR(tfm)) | |
833 | return ERR_CAST(tfm); | |
834 | if (tfm->__crt_alg->cra_module != THIS_MODULE) { | |
835 | crypto_free_tfm(tfm); | |
836 | return ERR_PTR(-EINVAL); | |
837 | } | |
838 | ||
839 | return __cryptd_ablkcipher_cast(__crypto_ablkcipher_cast(tfm)); | |
840 | } | |
841 | EXPORT_SYMBOL_GPL(cryptd_alloc_ablkcipher); | |
842 | ||
843 | struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm) | |
844 | { | |
845 | struct cryptd_blkcipher_ctx *ctx = crypto_ablkcipher_ctx(&tfm->base); | |
846 | return ctx->child; | |
847 | } | |
848 | EXPORT_SYMBOL_GPL(cryptd_ablkcipher_child); | |
849 | ||
850 | void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm) | |
851 | { | |
852 | crypto_free_ablkcipher(&tfm->base); | |
853 | } | |
854 | EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher); | |
855 | ||
856 | struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, | |
857 | u32 type, u32 mask) | |
858 | { | |
859 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
860 | struct crypto_ahash *tfm; | |
861 | ||
862 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
863 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
864 | return ERR_PTR(-EINVAL); | |
865 | tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); | |
866 | if (IS_ERR(tfm)) | |
867 | return ERR_CAST(tfm); | |
868 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
869 | crypto_free_ahash(tfm); | |
870 | return ERR_PTR(-EINVAL); | |
871 | } | |
872 | ||
873 | return __cryptd_ahash_cast(tfm); | |
874 | } | |
875 | EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); | |
876 | ||
877 | struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) | |
878 | { | |
879 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
880 | ||
881 | return ctx->child; | |
882 | } | |
883 | EXPORT_SYMBOL_GPL(cryptd_ahash_child); | |
884 | ||
885 | struct shash_desc *cryptd_shash_desc(struct ahash_request *req) | |
886 | { | |
887 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
888 | return &rctx->desc; | |
889 | } | |
890 | EXPORT_SYMBOL_GPL(cryptd_shash_desc); | |
891 | ||
892 | void cryptd_free_ahash(struct cryptd_ahash *tfm) | |
893 | { | |
894 | crypto_free_ahash(&tfm->base); | |
895 | } | |
896 | EXPORT_SYMBOL_GPL(cryptd_free_ahash); | |
897 | ||
898 | struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, | |
899 | u32 type, u32 mask) | |
900 | { | |
901 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
902 | struct crypto_aead *tfm; | |
903 | ||
904 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
905 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
906 | return ERR_PTR(-EINVAL); | |
907 | tfm = crypto_alloc_aead(cryptd_alg_name, type, mask); | |
908 | if (IS_ERR(tfm)) | |
909 | return ERR_CAST(tfm); | |
910 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
911 | crypto_free_aead(tfm); | |
912 | return ERR_PTR(-EINVAL); | |
913 | } | |
914 | return __cryptd_aead_cast(tfm); | |
915 | } | |
916 | EXPORT_SYMBOL_GPL(cryptd_alloc_aead); | |
917 | ||
918 | struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) | |
919 | { | |
920 | struct cryptd_aead_ctx *ctx; | |
921 | ctx = crypto_aead_ctx(&tfm->base); | |
922 | return ctx->child; | |
923 | } | |
924 | EXPORT_SYMBOL_GPL(cryptd_aead_child); | |
925 | ||
926 | void cryptd_free_aead(struct cryptd_aead *tfm) | |
927 | { | |
928 | crypto_free_aead(&tfm->base); | |
929 | } | |
930 | EXPORT_SYMBOL_GPL(cryptd_free_aead); | |
931 | ||
932 | static int __init cryptd_init(void) | |
933 | { | |
934 | int err; | |
935 | ||
936 | err = cryptd_init_queue(&queue, CRYPTD_MAX_CPU_QLEN); | |
937 | if (err) | |
938 | return err; | |
939 | ||
940 | err = crypto_register_template(&cryptd_tmpl); | |
941 | if (err) | |
942 | cryptd_fini_queue(&queue); | |
943 | ||
944 | return err; | |
945 | } | |
946 | ||
947 | static void __exit cryptd_exit(void) | |
948 | { | |
949 | cryptd_fini_queue(&queue); | |
950 | crypto_unregister_template(&cryptd_tmpl); | |
951 | } | |
952 | ||
953 | subsys_initcall(cryptd_init); | |
954 | module_exit(cryptd_exit); | |
955 | ||
956 | MODULE_LICENSE("GPL"); | |
957 | MODULE_DESCRIPTION("Software async crypto daemon"); |