]>
Commit | Line | Data |
---|---|---|
2874c5fd | 1 | // SPDX-License-Identifier: GPL-2.0-or-later |
124b53d0 HX |
2 | /* |
3 | * Software async crypto daemon. | |
4 | * | |
5 | * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> | |
6 | * | |
298c926c AH |
7 | * Added AEAD support to cryptd. |
8 | * Authors: Tadeusz Struk (tadeusz.struk@intel.com) | |
9 | * Adrian Hoban <adrian.hoban@intel.com> | |
10 | * Gabriele Paoloni <gabriele.paoloni@intel.com> | |
11 | * Aidan O'Mahony (aidan.o.mahony@intel.com) | |
12 | * Copyright (c) 2010, Intel Corporation. | |
124b53d0 HX |
13 | */ |
14 | ||
18e33e6d | 15 | #include <crypto/internal/hash.h> |
298c926c | 16 | #include <crypto/internal/aead.h> |
4e0958d1 | 17 | #include <crypto/internal/skcipher.h> |
1cac2cbc | 18 | #include <crypto/cryptd.h> |
43b970fa | 19 | #include <linux/refcount.h> |
124b53d0 HX |
20 | #include <linux/err.h> |
21 | #include <linux/init.h> | |
22 | #include <linux/kernel.h> | |
124b53d0 HX |
23 | #include <linux/list.h> |
24 | #include <linux/module.h> | |
124b53d0 HX |
25 | #include <linux/scatterlist.h> |
26 | #include <linux/sched.h> | |
27 | #include <linux/slab.h> | |
3e56e168 | 28 | #include <linux/workqueue.h> |
124b53d0 | 29 | |
eaf356e4 | 30 | static unsigned int cryptd_max_cpu_qlen = 1000; |
c3a53605 JM |
31 | module_param(cryptd_max_cpu_qlen, uint, 0); |
32 | MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); | |
124b53d0 | 33 | |
3e56e168 EB |
34 | static struct workqueue_struct *cryptd_wq; |
35 | ||
254eff77 | 36 | struct cryptd_cpu_queue { |
124b53d0 | 37 | struct crypto_queue queue; |
254eff77 HY |
38 | struct work_struct work; |
39 | }; | |
40 | ||
41 | struct cryptd_queue { | |
a29d8b8e | 42 | struct cryptd_cpu_queue __percpu *cpu_queue; |
124b53d0 HX |
43 | }; |
44 | ||
45 | struct cryptd_instance_ctx { | |
46 | struct crypto_spawn spawn; | |
254eff77 | 47 | struct cryptd_queue *queue; |
124b53d0 HX |
48 | }; |
49 | ||
4e0958d1 HX |
50 | struct skcipherd_instance_ctx { |
51 | struct crypto_skcipher_spawn spawn; | |
52 | struct cryptd_queue *queue; | |
53 | }; | |
54 | ||
46309d89 HX |
55 | struct hashd_instance_ctx { |
56 | struct crypto_shash_spawn spawn; | |
57 | struct cryptd_queue *queue; | |
58 | }; | |
59 | ||
298c926c AH |
60 | struct aead_instance_ctx { |
61 | struct crypto_aead_spawn aead_spawn; | |
62 | struct cryptd_queue *queue; | |
63 | }; | |
64 | ||
4e0958d1 | 65 | struct cryptd_skcipher_ctx { |
43b970fa | 66 | refcount_t refcnt; |
36b3875a | 67 | struct crypto_sync_skcipher *child; |
4e0958d1 HX |
68 | }; |
69 | ||
70 | struct cryptd_skcipher_request_ctx { | |
71 | crypto_completion_t complete; | |
72 | }; | |
73 | ||
b8a28251 | 74 | struct cryptd_hash_ctx { |
43b970fa | 75 | refcount_t refcnt; |
46309d89 | 76 | struct crypto_shash *child; |
b8a28251 LH |
77 | }; |
78 | ||
79 | struct cryptd_hash_request_ctx { | |
80 | crypto_completion_t complete; | |
46309d89 | 81 | struct shash_desc desc; |
b8a28251 | 82 | }; |
124b53d0 | 83 | |
298c926c | 84 | struct cryptd_aead_ctx { |
43b970fa | 85 | refcount_t refcnt; |
298c926c AH |
86 | struct crypto_aead *child; |
87 | }; | |
88 | ||
89 | struct cryptd_aead_request_ctx { | |
90 | crypto_completion_t complete; | |
91 | }; | |
92 | ||
254eff77 HY |
93 | static void cryptd_queue_worker(struct work_struct *work); |
94 | ||
95 | static int cryptd_init_queue(struct cryptd_queue *queue, | |
96 | unsigned int max_cpu_qlen) | |
97 | { | |
98 | int cpu; | |
99 | struct cryptd_cpu_queue *cpu_queue; | |
100 | ||
101 | queue->cpu_queue = alloc_percpu(struct cryptd_cpu_queue); | |
102 | if (!queue->cpu_queue) | |
103 | return -ENOMEM; | |
104 | for_each_possible_cpu(cpu) { | |
105 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
106 | crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); | |
107 | INIT_WORK(&cpu_queue->work, cryptd_queue_worker); | |
108 | } | |
c3a53605 | 109 | pr_info("cryptd: max_cpu_qlen set to %d\n", max_cpu_qlen); |
254eff77 HY |
110 | return 0; |
111 | } | |
112 | ||
113 | static void cryptd_fini_queue(struct cryptd_queue *queue) | |
114 | { | |
115 | int cpu; | |
116 | struct cryptd_cpu_queue *cpu_queue; | |
117 | ||
118 | for_each_possible_cpu(cpu) { | |
119 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
120 | BUG_ON(cpu_queue->queue.qlen); | |
121 | } | |
122 | free_percpu(queue->cpu_queue); | |
123 | } | |
124 | ||
125 | static int cryptd_enqueue_request(struct cryptd_queue *queue, | |
126 | struct crypto_async_request *request) | |
127 | { | |
128 | int cpu, err; | |
129 | struct cryptd_cpu_queue *cpu_queue; | |
43b970fa | 130 | refcount_t *refcnt; |
254eff77 HY |
131 | |
132 | cpu = get_cpu(); | |
0b44f486 | 133 | cpu_queue = this_cpu_ptr(queue->cpu_queue); |
254eff77 | 134 | err = crypto_enqueue_request(&cpu_queue->queue, request); |
81760ea6 HX |
135 | |
136 | refcnt = crypto_tfm_ctx(request->tfm); | |
81760ea6 | 137 | |
6b80ea38 | 138 | if (err == -ENOSPC) |
81760ea6 HX |
139 | goto out_put_cpu; |
140 | ||
3e56e168 | 141 | queue_work_on(cpu, cryptd_wq, &cpu_queue->work); |
81760ea6 | 142 | |
43b970fa | 143 | if (!refcount_read(refcnt)) |
81760ea6 HX |
144 | goto out_put_cpu; |
145 | ||
43b970fa | 146 | refcount_inc(refcnt); |
81760ea6 HX |
147 | |
148 | out_put_cpu: | |
254eff77 HY |
149 | put_cpu(); |
150 | ||
151 | return err; | |
152 | } | |
153 | ||
154 | /* Called in workqueue context, do one real cryption work (via | |
155 | * req->complete) and reschedule itself if there are more work to | |
156 | * do. */ | |
157 | static void cryptd_queue_worker(struct work_struct *work) | |
158 | { | |
159 | struct cryptd_cpu_queue *cpu_queue; | |
160 | struct crypto_async_request *req, *backlog; | |
161 | ||
162 | cpu_queue = container_of(work, struct cryptd_cpu_queue, work); | |
9efade1b JK |
163 | /* |
164 | * Only handle one request at a time to avoid hogging crypto workqueue. | |
165 | * preempt_disable/enable is used to prevent being preempted by | |
166 | * cryptd_enqueue_request(). local_bh_disable/enable is used to prevent | |
167 | * cryptd_enqueue_request() being accessed from software interrupts. | |
168 | */ | |
169 | local_bh_disable(); | |
254eff77 HY |
170 | preempt_disable(); |
171 | backlog = crypto_get_backlog(&cpu_queue->queue); | |
172 | req = crypto_dequeue_request(&cpu_queue->queue); | |
173 | preempt_enable(); | |
9efade1b | 174 | local_bh_enable(); |
254eff77 HY |
175 | |
176 | if (!req) | |
177 | return; | |
178 | ||
179 | if (backlog) | |
180 | backlog->complete(backlog, -EINPROGRESS); | |
181 | req->complete(req, 0); | |
182 | ||
183 | if (cpu_queue->queue.qlen) | |
3e56e168 | 184 | queue_work(cryptd_wq, &cpu_queue->work); |
254eff77 HY |
185 | } |
186 | ||
187 | static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) | |
124b53d0 HX |
188 | { |
189 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
190 | struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
254eff77 | 191 | return ictx->queue; |
124b53d0 HX |
192 | } |
193 | ||
466a7b9e SM |
194 | static inline void cryptd_check_internal(struct rtattr **tb, u32 *type, |
195 | u32 *mask) | |
196 | { | |
197 | struct crypto_attr_type *algt; | |
198 | ||
199 | algt = crypto_get_attr_type(tb); | |
200 | if (IS_ERR(algt)) | |
201 | return; | |
f6da3205 | 202 | |
5e4b8c1f HX |
203 | *type |= algt->type & CRYPTO_ALG_INTERNAL; |
204 | *mask |= algt->mask & CRYPTO_ALG_INTERNAL; | |
466a7b9e SM |
205 | } |
206 | ||
9b8c456e HX |
207 | static int cryptd_init_instance(struct crypto_instance *inst, |
208 | struct crypto_alg *alg) | |
209 | { | |
210 | if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, | |
211 | "cryptd(%s)", | |
212 | alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) | |
213 | return -ENAMETOOLONG; | |
214 | ||
215 | memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); | |
216 | ||
217 | inst->alg.cra_priority = alg->cra_priority + 50; | |
218 | inst->alg.cra_blocksize = alg->cra_blocksize; | |
219 | inst->alg.cra_alignmask = alg->cra_alignmask; | |
220 | ||
221 | return 0; | |
222 | } | |
223 | ||
4e0958d1 HX |
224 | static int cryptd_skcipher_setkey(struct crypto_skcipher *parent, |
225 | const u8 *key, unsigned int keylen) | |
226 | { | |
227 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(parent); | |
36b3875a | 228 | struct crypto_sync_skcipher *child = ctx->child; |
4e0958d1 | 229 | |
36b3875a KC |
230 | crypto_sync_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
231 | crypto_sync_skcipher_set_flags(child, | |
232 | crypto_skcipher_get_flags(parent) & | |
4e0958d1 | 233 | CRYPTO_TFM_REQ_MASK); |
af5034e8 | 234 | return crypto_sync_skcipher_setkey(child, key, keylen); |
4e0958d1 HX |
235 | } |
236 | ||
237 | static void cryptd_skcipher_complete(struct skcipher_request *req, int err) | |
238 | { | |
239 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
240 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
241 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
43b970fa | 242 | int refcnt = refcount_read(&ctx->refcnt); |
4e0958d1 HX |
243 | |
244 | local_bh_disable(); | |
245 | rctx->complete(&req->base, err); | |
246 | local_bh_enable(); | |
247 | ||
43b970fa | 248 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
4e0958d1 HX |
249 | crypto_free_skcipher(tfm); |
250 | } | |
251 | ||
252 | static void cryptd_skcipher_encrypt(struct crypto_async_request *base, | |
253 | int err) | |
254 | { | |
255 | struct skcipher_request *req = skcipher_request_cast(base); | |
256 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
257 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
258 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
36b3875a KC |
259 | struct crypto_sync_skcipher *child = ctx->child; |
260 | SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, child); | |
4e0958d1 HX |
261 | |
262 | if (unlikely(err == -EINPROGRESS)) | |
263 | goto out; | |
264 | ||
36b3875a | 265 | skcipher_request_set_sync_tfm(subreq, child); |
4e0958d1 HX |
266 | skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, |
267 | NULL, NULL); | |
268 | skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, | |
269 | req->iv); | |
270 | ||
271 | err = crypto_skcipher_encrypt(subreq); | |
272 | skcipher_request_zero(subreq); | |
273 | ||
274 | req->base.complete = rctx->complete; | |
275 | ||
276 | out: | |
277 | cryptd_skcipher_complete(req, err); | |
278 | } | |
279 | ||
280 | static void cryptd_skcipher_decrypt(struct crypto_async_request *base, | |
281 | int err) | |
282 | { | |
283 | struct skcipher_request *req = skcipher_request_cast(base); | |
284 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
285 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
286 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
36b3875a KC |
287 | struct crypto_sync_skcipher *child = ctx->child; |
288 | SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, child); | |
4e0958d1 HX |
289 | |
290 | if (unlikely(err == -EINPROGRESS)) | |
291 | goto out; | |
292 | ||
36b3875a | 293 | skcipher_request_set_sync_tfm(subreq, child); |
4e0958d1 HX |
294 | skcipher_request_set_callback(subreq, CRYPTO_TFM_REQ_MAY_SLEEP, |
295 | NULL, NULL); | |
296 | skcipher_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, | |
297 | req->iv); | |
298 | ||
299 | err = crypto_skcipher_decrypt(subreq); | |
300 | skcipher_request_zero(subreq); | |
301 | ||
302 | req->base.complete = rctx->complete; | |
303 | ||
304 | out: | |
305 | cryptd_skcipher_complete(req, err); | |
306 | } | |
307 | ||
308 | static int cryptd_skcipher_enqueue(struct skcipher_request *req, | |
309 | crypto_completion_t compl) | |
310 | { | |
311 | struct cryptd_skcipher_request_ctx *rctx = skcipher_request_ctx(req); | |
312 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); | |
313 | struct cryptd_queue *queue; | |
314 | ||
315 | queue = cryptd_get_queue(crypto_skcipher_tfm(tfm)); | |
316 | rctx->complete = req->base.complete; | |
317 | req->base.complete = compl; | |
318 | ||
319 | return cryptd_enqueue_request(queue, &req->base); | |
320 | } | |
321 | ||
322 | static int cryptd_skcipher_encrypt_enqueue(struct skcipher_request *req) | |
323 | { | |
324 | return cryptd_skcipher_enqueue(req, cryptd_skcipher_encrypt); | |
325 | } | |
326 | ||
327 | static int cryptd_skcipher_decrypt_enqueue(struct skcipher_request *req) | |
328 | { | |
329 | return cryptd_skcipher_enqueue(req, cryptd_skcipher_decrypt); | |
330 | } | |
331 | ||
332 | static int cryptd_skcipher_init_tfm(struct crypto_skcipher *tfm) | |
333 | { | |
334 | struct skcipher_instance *inst = skcipher_alg_instance(tfm); | |
335 | struct skcipherd_instance_ctx *ictx = skcipher_instance_ctx(inst); | |
336 | struct crypto_skcipher_spawn *spawn = &ictx->spawn; | |
337 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
338 | struct crypto_skcipher *cipher; | |
339 | ||
340 | cipher = crypto_spawn_skcipher(spawn); | |
341 | if (IS_ERR(cipher)) | |
342 | return PTR_ERR(cipher); | |
343 | ||
36b3875a | 344 | ctx->child = (struct crypto_sync_skcipher *)cipher; |
4e0958d1 HX |
345 | crypto_skcipher_set_reqsize( |
346 | tfm, sizeof(struct cryptd_skcipher_request_ctx)); | |
347 | return 0; | |
348 | } | |
349 | ||
350 | static void cryptd_skcipher_exit_tfm(struct crypto_skcipher *tfm) | |
351 | { | |
352 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm); | |
353 | ||
36b3875a | 354 | crypto_free_sync_skcipher(ctx->child); |
4e0958d1 HX |
355 | } |
356 | ||
357 | static void cryptd_skcipher_free(struct skcipher_instance *inst) | |
358 | { | |
359 | struct skcipherd_instance_ctx *ctx = skcipher_instance_ctx(inst); | |
360 | ||
361 | crypto_drop_skcipher(&ctx->spawn); | |
1a0fad63 | 362 | kfree(inst); |
4e0958d1 HX |
363 | } |
364 | ||
365 | static int cryptd_create_skcipher(struct crypto_template *tmpl, | |
366 | struct rtattr **tb, | |
367 | struct cryptd_queue *queue) | |
368 | { | |
369 | struct skcipherd_instance_ctx *ctx; | |
370 | struct skcipher_instance *inst; | |
371 | struct skcipher_alg *alg; | |
4e0958d1 HX |
372 | u32 type; |
373 | u32 mask; | |
374 | int err; | |
375 | ||
376 | type = 0; | |
377 | mask = CRYPTO_ALG_ASYNC; | |
378 | ||
379 | cryptd_check_internal(tb, &type, &mask); | |
380 | ||
4e0958d1 HX |
381 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
382 | if (!inst) | |
383 | return -ENOMEM; | |
384 | ||
385 | ctx = skcipher_instance_ctx(inst); | |
386 | ctx->queue = queue; | |
387 | ||
b9f76ddd | 388 | err = crypto_grab_skcipher(&ctx->spawn, skcipher_crypto_instance(inst), |
b8c0d74a | 389 | crypto_attr_alg_name(tb[1]), type, mask); |
4e0958d1 | 390 | if (err) |
b8c0d74a | 391 | goto err_free_inst; |
4e0958d1 HX |
392 | |
393 | alg = crypto_spawn_skcipher_alg(&ctx->spawn); | |
394 | err = cryptd_init_instance(skcipher_crypto_instance(inst), &alg->base); | |
395 | if (err) | |
b8c0d74a | 396 | goto err_free_inst; |
4e0958d1 HX |
397 | |
398 | inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC | | |
399 | (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); | |
400 | ||
401 | inst->alg.ivsize = crypto_skcipher_alg_ivsize(alg); | |
402 | inst->alg.chunksize = crypto_skcipher_alg_chunksize(alg); | |
403 | inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg); | |
404 | inst->alg.max_keysize = crypto_skcipher_alg_max_keysize(alg); | |
405 | ||
406 | inst->alg.base.cra_ctxsize = sizeof(struct cryptd_skcipher_ctx); | |
407 | ||
408 | inst->alg.init = cryptd_skcipher_init_tfm; | |
409 | inst->alg.exit = cryptd_skcipher_exit_tfm; | |
410 | ||
411 | inst->alg.setkey = cryptd_skcipher_setkey; | |
412 | inst->alg.encrypt = cryptd_skcipher_encrypt_enqueue; | |
413 | inst->alg.decrypt = cryptd_skcipher_decrypt_enqueue; | |
414 | ||
415 | inst->free = cryptd_skcipher_free; | |
416 | ||
417 | err = skcipher_register_instance(tmpl, inst); | |
418 | if (err) { | |
b8c0d74a EB |
419 | err_free_inst: |
420 | cryptd_skcipher_free(inst); | |
4e0958d1 HX |
421 | } |
422 | return err; | |
423 | } | |
424 | ||
b8a28251 LH |
425 | static int cryptd_hash_init_tfm(struct crypto_tfm *tfm) |
426 | { | |
427 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
46309d89 HX |
428 | struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); |
429 | struct crypto_shash_spawn *spawn = &ictx->spawn; | |
b8a28251 | 430 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); |
46309d89 | 431 | struct crypto_shash *hash; |
b8a28251 | 432 | |
46309d89 HX |
433 | hash = crypto_spawn_shash(spawn); |
434 | if (IS_ERR(hash)) | |
435 | return PTR_ERR(hash); | |
b8a28251 | 436 | |
46309d89 | 437 | ctx->child = hash; |
0d6669e2 HX |
438 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), |
439 | sizeof(struct cryptd_hash_request_ctx) + | |
440 | crypto_shash_descsize(hash)); | |
b8a28251 LH |
441 | return 0; |
442 | } | |
443 | ||
444 | static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm) | |
445 | { | |
446 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); | |
b8a28251 | 447 | |
46309d89 | 448 | crypto_free_shash(ctx->child); |
b8a28251 LH |
449 | } |
450 | ||
451 | static int cryptd_hash_setkey(struct crypto_ahash *parent, | |
452 | const u8 *key, unsigned int keylen) | |
453 | { | |
454 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); | |
46309d89 | 455 | struct crypto_shash *child = ctx->child; |
b8a28251 | 456 | |
46309d89 HX |
457 | crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
458 | crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & | |
459 | CRYPTO_TFM_REQ_MASK); | |
af5034e8 | 460 | return crypto_shash_setkey(child, key, keylen); |
b8a28251 LH |
461 | } |
462 | ||
463 | static int cryptd_hash_enqueue(struct ahash_request *req, | |
3e3dc25f | 464 | crypto_completion_t compl) |
b8a28251 LH |
465 | { |
466 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
467 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
254eff77 HY |
468 | struct cryptd_queue *queue = |
469 | cryptd_get_queue(crypto_ahash_tfm(tfm)); | |
b8a28251 LH |
470 | |
471 | rctx->complete = req->base.complete; | |
3e3dc25f | 472 | req->base.complete = compl; |
b8a28251 | 473 | |
254eff77 | 474 | return cryptd_enqueue_request(queue, &req->base); |
b8a28251 LH |
475 | } |
476 | ||
81760ea6 HX |
477 | static void cryptd_hash_complete(struct ahash_request *req, int err) |
478 | { | |
479 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
480 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); | |
481 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
43b970fa | 482 | int refcnt = refcount_read(&ctx->refcnt); |
81760ea6 HX |
483 | |
484 | local_bh_disable(); | |
485 | rctx->complete(&req->base, err); | |
486 | local_bh_enable(); | |
487 | ||
43b970fa | 488 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 HX |
489 | crypto_free_ahash(tfm); |
490 | } | |
491 | ||
b8a28251 LH |
492 | static void cryptd_hash_init(struct crypto_async_request *req_async, int err) |
493 | { | |
46309d89 HX |
494 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
495 | struct crypto_shash *child = ctx->child; | |
496 | struct ahash_request *req = ahash_request_cast(req_async); | |
497 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
498 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
499 | |
500 | if (unlikely(err == -EINPROGRESS)) | |
501 | goto out; | |
502 | ||
46309d89 | 503 | desc->tfm = child; |
b8a28251 | 504 | |
46309d89 | 505 | err = crypto_shash_init(desc); |
b8a28251 LH |
506 | |
507 | req->base.complete = rctx->complete; | |
508 | ||
509 | out: | |
81760ea6 | 510 | cryptd_hash_complete(req, err); |
b8a28251 LH |
511 | } |
512 | ||
513 | static int cryptd_hash_init_enqueue(struct ahash_request *req) | |
514 | { | |
515 | return cryptd_hash_enqueue(req, cryptd_hash_init); | |
516 | } | |
517 | ||
518 | static void cryptd_hash_update(struct crypto_async_request *req_async, int err) | |
519 | { | |
46309d89 | 520 | struct ahash_request *req = ahash_request_cast(req_async); |
b8a28251 | 521 | struct cryptd_hash_request_ctx *rctx; |
b8a28251 LH |
522 | |
523 | rctx = ahash_request_ctx(req); | |
524 | ||
525 | if (unlikely(err == -EINPROGRESS)) | |
526 | goto out; | |
527 | ||
46309d89 | 528 | err = shash_ahash_update(req, &rctx->desc); |
b8a28251 LH |
529 | |
530 | req->base.complete = rctx->complete; | |
531 | ||
532 | out: | |
81760ea6 | 533 | cryptd_hash_complete(req, err); |
b8a28251 LH |
534 | } |
535 | ||
536 | static int cryptd_hash_update_enqueue(struct ahash_request *req) | |
537 | { | |
538 | return cryptd_hash_enqueue(req, cryptd_hash_update); | |
539 | } | |
540 | ||
541 | static void cryptd_hash_final(struct crypto_async_request *req_async, int err) | |
542 | { | |
46309d89 HX |
543 | struct ahash_request *req = ahash_request_cast(req_async); |
544 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
b8a28251 LH |
545 | |
546 | if (unlikely(err == -EINPROGRESS)) | |
547 | goto out; | |
548 | ||
46309d89 | 549 | err = crypto_shash_final(&rctx->desc, req->result); |
b8a28251 LH |
550 | |
551 | req->base.complete = rctx->complete; | |
552 | ||
553 | out: | |
81760ea6 | 554 | cryptd_hash_complete(req, err); |
b8a28251 LH |
555 | } |
556 | ||
557 | static int cryptd_hash_final_enqueue(struct ahash_request *req) | |
558 | { | |
559 | return cryptd_hash_enqueue(req, cryptd_hash_final); | |
560 | } | |
561 | ||
6fba00d1 HX |
562 | static void cryptd_hash_finup(struct crypto_async_request *req_async, int err) |
563 | { | |
564 | struct ahash_request *req = ahash_request_cast(req_async); | |
565 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
566 | ||
567 | if (unlikely(err == -EINPROGRESS)) | |
568 | goto out; | |
569 | ||
570 | err = shash_ahash_finup(req, &rctx->desc); | |
571 | ||
572 | req->base.complete = rctx->complete; | |
573 | ||
574 | out: | |
81760ea6 | 575 | cryptd_hash_complete(req, err); |
6fba00d1 HX |
576 | } |
577 | ||
578 | static int cryptd_hash_finup_enqueue(struct ahash_request *req) | |
579 | { | |
580 | return cryptd_hash_enqueue(req, cryptd_hash_finup); | |
581 | } | |
582 | ||
b8a28251 LH |
583 | static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) |
584 | { | |
46309d89 HX |
585 | struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); |
586 | struct crypto_shash *child = ctx->child; | |
587 | struct ahash_request *req = ahash_request_cast(req_async); | |
588 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
589 | struct shash_desc *desc = &rctx->desc; | |
b8a28251 LH |
590 | |
591 | if (unlikely(err == -EINPROGRESS)) | |
592 | goto out; | |
593 | ||
46309d89 | 594 | desc->tfm = child; |
b8a28251 | 595 | |
46309d89 | 596 | err = shash_ahash_digest(req, desc); |
b8a28251 LH |
597 | |
598 | req->base.complete = rctx->complete; | |
599 | ||
600 | out: | |
81760ea6 | 601 | cryptd_hash_complete(req, err); |
b8a28251 LH |
602 | } |
603 | ||
604 | static int cryptd_hash_digest_enqueue(struct ahash_request *req) | |
605 | { | |
606 | return cryptd_hash_enqueue(req, cryptd_hash_digest); | |
607 | } | |
608 | ||
6fba00d1 HX |
609 | static int cryptd_hash_export(struct ahash_request *req, void *out) |
610 | { | |
611 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
612 | ||
613 | return crypto_shash_export(&rctx->desc, out); | |
614 | } | |
615 | ||
616 | static int cryptd_hash_import(struct ahash_request *req, const void *in) | |
617 | { | |
0bd22235 AB |
618 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); |
619 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(tfm); | |
620 | struct shash_desc *desc = cryptd_shash_desc(req); | |
621 | ||
622 | desc->tfm = ctx->child; | |
6fba00d1 | 623 | |
0bd22235 | 624 | return crypto_shash_import(desc, in); |
6fba00d1 HX |
625 | } |
626 | ||
758ec5ac EB |
627 | static void cryptd_hash_free(struct ahash_instance *inst) |
628 | { | |
629 | struct hashd_instance_ctx *ctx = ahash_instance_ctx(inst); | |
630 | ||
631 | crypto_drop_shash(&ctx->spawn); | |
632 | kfree(inst); | |
633 | } | |
634 | ||
9cd899a3 HX |
635 | static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, |
636 | struct cryptd_queue *queue) | |
b8a28251 | 637 | { |
46309d89 | 638 | struct hashd_instance_ctx *ctx; |
0b535adf | 639 | struct ahash_instance *inst; |
218c5035 | 640 | struct shash_alg *alg; |
466a7b9e SM |
641 | u32 type = 0; |
642 | u32 mask = 0; | |
46309d89 | 643 | int err; |
b8a28251 | 644 | |
466a7b9e SM |
645 | cryptd_check_internal(tb, &type, &mask); |
646 | ||
218c5035 EB |
647 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
648 | if (!inst) | |
649 | return -ENOMEM; | |
b8a28251 | 650 | |
0b535adf | 651 | ctx = ahash_instance_ctx(inst); |
46309d89 HX |
652 | ctx->queue = queue; |
653 | ||
218c5035 EB |
654 | err = crypto_grab_shash(&ctx->spawn, ahash_crypto_instance(inst), |
655 | crypto_attr_alg_name(tb[1]), type, mask); | |
46309d89 | 656 | if (err) |
218c5035 EB |
657 | goto err_free_inst; |
658 | alg = crypto_spawn_shash_alg(&ctx->spawn); | |
659 | ||
660 | err = cryptd_init_instance(ahash_crypto_instance(inst), &alg->base); | |
661 | if (err) | |
662 | goto err_free_inst; | |
46309d89 | 663 | |
a208fa8f | 664 | inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC | |
218c5035 EB |
665 | (alg->base.cra_flags & (CRYPTO_ALG_INTERNAL | |
666 | CRYPTO_ALG_OPTIONAL_KEY)); | |
b8a28251 | 667 | |
218c5035 EB |
668 | inst->alg.halg.digestsize = alg->digestsize; |
669 | inst->alg.halg.statesize = alg->statesize; | |
0b535adf | 670 | inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); |
b8a28251 | 671 | |
0b535adf HX |
672 | inst->alg.halg.base.cra_init = cryptd_hash_init_tfm; |
673 | inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm; | |
b8a28251 | 674 | |
0b535adf HX |
675 | inst->alg.init = cryptd_hash_init_enqueue; |
676 | inst->alg.update = cryptd_hash_update_enqueue; | |
677 | inst->alg.final = cryptd_hash_final_enqueue; | |
6fba00d1 HX |
678 | inst->alg.finup = cryptd_hash_finup_enqueue; |
679 | inst->alg.export = cryptd_hash_export; | |
680 | inst->alg.import = cryptd_hash_import; | |
218c5035 | 681 | if (crypto_shash_alg_has_setkey(alg)) |
841a3ff3 | 682 | inst->alg.setkey = cryptd_hash_setkey; |
0b535adf | 683 | inst->alg.digest = cryptd_hash_digest_enqueue; |
b8a28251 | 684 | |
758ec5ac EB |
685 | inst->free = cryptd_hash_free; |
686 | ||
0b535adf | 687 | err = ahash_register_instance(tmpl, inst); |
9cd899a3 | 688 | if (err) { |
218c5035 | 689 | err_free_inst: |
b8c0d74a | 690 | cryptd_hash_free(inst); |
9cd899a3 | 691 | } |
9cd899a3 | 692 | return err; |
b8a28251 LH |
693 | } |
694 | ||
92b9876b HX |
695 | static int cryptd_aead_setkey(struct crypto_aead *parent, |
696 | const u8 *key, unsigned int keylen) | |
697 | { | |
698 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); | |
699 | struct crypto_aead *child = ctx->child; | |
700 | ||
701 | return crypto_aead_setkey(child, key, keylen); | |
702 | } | |
703 | ||
704 | static int cryptd_aead_setauthsize(struct crypto_aead *parent, | |
705 | unsigned int authsize) | |
706 | { | |
707 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(parent); | |
708 | struct crypto_aead *child = ctx->child; | |
709 | ||
710 | return crypto_aead_setauthsize(child, authsize); | |
711 | } | |
712 | ||
298c926c AH |
713 | static void cryptd_aead_crypt(struct aead_request *req, |
714 | struct crypto_aead *child, | |
715 | int err, | |
716 | int (*crypt)(struct aead_request *req)) | |
717 | { | |
718 | struct cryptd_aead_request_ctx *rctx; | |
81760ea6 | 719 | struct cryptd_aead_ctx *ctx; |
ec9f2006 | 720 | crypto_completion_t compl; |
81760ea6 HX |
721 | struct crypto_aead *tfm; |
722 | int refcnt; | |
ec9f2006 | 723 | |
298c926c | 724 | rctx = aead_request_ctx(req); |
ec9f2006 | 725 | compl = rctx->complete; |
298c926c | 726 | |
31bd44e7 HX |
727 | tfm = crypto_aead_reqtfm(req); |
728 | ||
298c926c AH |
729 | if (unlikely(err == -EINPROGRESS)) |
730 | goto out; | |
731 | aead_request_set_tfm(req, child); | |
732 | err = crypt( req ); | |
81760ea6 | 733 | |
298c926c | 734 | out: |
81760ea6 | 735 | ctx = crypto_aead_ctx(tfm); |
43b970fa | 736 | refcnt = refcount_read(&ctx->refcnt); |
81760ea6 | 737 | |
298c926c | 738 | local_bh_disable(); |
ec9f2006 | 739 | compl(&req->base, err); |
298c926c | 740 | local_bh_enable(); |
81760ea6 | 741 | |
43b970fa | 742 | if (err != -EINPROGRESS && refcnt && refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 743 | crypto_free_aead(tfm); |
298c926c AH |
744 | } |
745 | ||
746 | static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err) | |
747 | { | |
748 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
749 | struct crypto_aead *child = ctx->child; | |
750 | struct aead_request *req; | |
751 | ||
752 | req = container_of(areq, struct aead_request, base); | |
ba3749a7 | 753 | cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->encrypt); |
298c926c AH |
754 | } |
755 | ||
756 | static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err) | |
757 | { | |
758 | struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); | |
759 | struct crypto_aead *child = ctx->child; | |
760 | struct aead_request *req; | |
761 | ||
762 | req = container_of(areq, struct aead_request, base); | |
ba3749a7 | 763 | cryptd_aead_crypt(req, child, err, crypto_aead_alg(child)->decrypt); |
298c926c AH |
764 | } |
765 | ||
766 | static int cryptd_aead_enqueue(struct aead_request *req, | |
3e3dc25f | 767 | crypto_completion_t compl) |
298c926c AH |
768 | { |
769 | struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req); | |
770 | struct crypto_aead *tfm = crypto_aead_reqtfm(req); | |
771 | struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm)); | |
772 | ||
773 | rctx->complete = req->base.complete; | |
3e3dc25f | 774 | req->base.complete = compl; |
298c926c AH |
775 | return cryptd_enqueue_request(queue, &req->base); |
776 | } | |
777 | ||
778 | static int cryptd_aead_encrypt_enqueue(struct aead_request *req) | |
779 | { | |
780 | return cryptd_aead_enqueue(req, cryptd_aead_encrypt ); | |
781 | } | |
782 | ||
783 | static int cryptd_aead_decrypt_enqueue(struct aead_request *req) | |
784 | { | |
785 | return cryptd_aead_enqueue(req, cryptd_aead_decrypt ); | |
786 | } | |
787 | ||
f614e546 | 788 | static int cryptd_aead_init_tfm(struct crypto_aead *tfm) |
298c926c | 789 | { |
f614e546 HX |
790 | struct aead_instance *inst = aead_alg_instance(tfm); |
791 | struct aead_instance_ctx *ictx = aead_instance_ctx(inst); | |
298c926c | 792 | struct crypto_aead_spawn *spawn = &ictx->aead_spawn; |
f614e546 | 793 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); |
298c926c AH |
794 | struct crypto_aead *cipher; |
795 | ||
796 | cipher = crypto_spawn_aead(spawn); | |
797 | if (IS_ERR(cipher)) | |
798 | return PTR_ERR(cipher); | |
799 | ||
298c926c | 800 | ctx->child = cipher; |
ec9f2006 HX |
801 | crypto_aead_set_reqsize( |
802 | tfm, max((unsigned)sizeof(struct cryptd_aead_request_ctx), | |
803 | crypto_aead_reqsize(cipher))); | |
298c926c AH |
804 | return 0; |
805 | } | |
806 | ||
f614e546 | 807 | static void cryptd_aead_exit_tfm(struct crypto_aead *tfm) |
298c926c | 808 | { |
f614e546 | 809 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(tfm); |
298c926c AH |
810 | crypto_free_aead(ctx->child); |
811 | } | |
812 | ||
758ec5ac EB |
813 | static void cryptd_aead_free(struct aead_instance *inst) |
814 | { | |
815 | struct aead_instance_ctx *ctx = aead_instance_ctx(inst); | |
816 | ||
817 | crypto_drop_aead(&ctx->aead_spawn); | |
818 | kfree(inst); | |
819 | } | |
820 | ||
298c926c AH |
821 | static int cryptd_create_aead(struct crypto_template *tmpl, |
822 | struct rtattr **tb, | |
823 | struct cryptd_queue *queue) | |
824 | { | |
825 | struct aead_instance_ctx *ctx; | |
f614e546 HX |
826 | struct aead_instance *inst; |
827 | struct aead_alg *alg; | |
9b8c456e | 828 | u32 type = 0; |
ec9f2006 | 829 | u32 mask = CRYPTO_ALG_ASYNC; |
298c926c AH |
830 | int err; |
831 | ||
466a7b9e SM |
832 | cryptd_check_internal(tb, &type, &mask); |
833 | ||
9b8c456e HX |
834 | inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); |
835 | if (!inst) | |
836 | return -ENOMEM; | |
298c926c | 837 | |
f614e546 | 838 | ctx = aead_instance_ctx(inst); |
298c926c AH |
839 | ctx->queue = queue; |
840 | ||
cd900f0c | 841 | err = crypto_grab_aead(&ctx->aead_spawn, aead_crypto_instance(inst), |
b8c0d74a | 842 | crypto_attr_alg_name(tb[1]), type, mask); |
298c926c | 843 | if (err) |
b8c0d74a | 844 | goto err_free_inst; |
298c926c | 845 | |
f614e546 HX |
846 | alg = crypto_spawn_aead_alg(&ctx->aead_spawn); |
847 | err = cryptd_init_instance(aead_crypto_instance(inst), &alg->base); | |
9b8c456e | 848 | if (err) |
b8c0d74a | 849 | goto err_free_inst; |
9b8c456e | 850 | |
f614e546 | 851 | inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC | |
5e4b8c1f | 852 | (alg->base.cra_flags & CRYPTO_ALG_INTERNAL); |
f614e546 | 853 | inst->alg.base.cra_ctxsize = sizeof(struct cryptd_aead_ctx); |
298c926c | 854 | |
f614e546 HX |
855 | inst->alg.ivsize = crypto_aead_alg_ivsize(alg); |
856 | inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg); | |
857 | ||
858 | inst->alg.init = cryptd_aead_init_tfm; | |
859 | inst->alg.exit = cryptd_aead_exit_tfm; | |
860 | inst->alg.setkey = cryptd_aead_setkey; | |
861 | inst->alg.setauthsize = cryptd_aead_setauthsize; | |
862 | inst->alg.encrypt = cryptd_aead_encrypt_enqueue; | |
863 | inst->alg.decrypt = cryptd_aead_decrypt_enqueue; | |
864 | ||
758ec5ac EB |
865 | inst->free = cryptd_aead_free; |
866 | ||
f614e546 | 867 | err = aead_register_instance(tmpl, inst); |
298c926c | 868 | if (err) { |
b8c0d74a EB |
869 | err_free_inst: |
870 | cryptd_aead_free(inst); | |
298c926c | 871 | } |
298c926c AH |
872 | return err; |
873 | } | |
874 | ||
254eff77 | 875 | static struct cryptd_queue queue; |
124b53d0 | 876 | |
9cd899a3 | 877 | static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) |
124b53d0 HX |
878 | { |
879 | struct crypto_attr_type *algt; | |
880 | ||
881 | algt = crypto_get_attr_type(tb); | |
882 | if (IS_ERR(algt)) | |
9cd899a3 | 883 | return PTR_ERR(algt); |
124b53d0 HX |
884 | |
885 | switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { | |
c65058b7 | 886 | case CRYPTO_ALG_TYPE_SKCIPHER: |
4e0958d1 | 887 | return cryptd_create_skcipher(tmpl, tb, &queue); |
84ede58d | 888 | case CRYPTO_ALG_TYPE_HASH: |
9cd899a3 | 889 | return cryptd_create_hash(tmpl, tb, &queue); |
298c926c AH |
890 | case CRYPTO_ALG_TYPE_AEAD: |
891 | return cryptd_create_aead(tmpl, tb, &queue); | |
124b53d0 HX |
892 | } |
893 | ||
9cd899a3 | 894 | return -EINVAL; |
124b53d0 HX |
895 | } |
896 | ||
124b53d0 HX |
897 | static struct crypto_template cryptd_tmpl = { |
898 | .name = "cryptd", | |
9cd899a3 | 899 | .create = cryptd_create, |
124b53d0 HX |
900 | .module = THIS_MODULE, |
901 | }; | |
902 | ||
4e0958d1 HX |
903 | struct cryptd_skcipher *cryptd_alloc_skcipher(const char *alg_name, |
904 | u32 type, u32 mask) | |
905 | { | |
906 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
907 | struct cryptd_skcipher_ctx *ctx; | |
908 | struct crypto_skcipher *tfm; | |
909 | ||
910 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
911 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
912 | return ERR_PTR(-EINVAL); | |
913 | ||
914 | tfm = crypto_alloc_skcipher(cryptd_alg_name, type, mask); | |
915 | if (IS_ERR(tfm)) | |
916 | return ERR_CAST(tfm); | |
917 | ||
918 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
919 | crypto_free_skcipher(tfm); | |
920 | return ERR_PTR(-EINVAL); | |
921 | } | |
922 | ||
923 | ctx = crypto_skcipher_ctx(tfm); | |
43b970fa | 924 | refcount_set(&ctx->refcnt, 1); |
4e0958d1 HX |
925 | |
926 | return container_of(tfm, struct cryptd_skcipher, base); | |
927 | } | |
928 | EXPORT_SYMBOL_GPL(cryptd_alloc_skcipher); | |
929 | ||
930 | struct crypto_skcipher *cryptd_skcipher_child(struct cryptd_skcipher *tfm) | |
931 | { | |
932 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
933 | ||
36b3875a | 934 | return &ctx->child->base; |
4e0958d1 HX |
935 | } |
936 | EXPORT_SYMBOL_GPL(cryptd_skcipher_child); | |
937 | ||
938 | bool cryptd_skcipher_queued(struct cryptd_skcipher *tfm) | |
939 | { | |
940 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
941 | ||
43b970fa | 942 | return refcount_read(&ctx->refcnt) - 1; |
4e0958d1 HX |
943 | } |
944 | EXPORT_SYMBOL_GPL(cryptd_skcipher_queued); | |
945 | ||
946 | void cryptd_free_skcipher(struct cryptd_skcipher *tfm) | |
947 | { | |
948 | struct cryptd_skcipher_ctx *ctx = crypto_skcipher_ctx(&tfm->base); | |
949 | ||
43b970fa | 950 | if (refcount_dec_and_test(&ctx->refcnt)) |
4e0958d1 HX |
951 | crypto_free_skcipher(&tfm->base); |
952 | } | |
953 | EXPORT_SYMBOL_GPL(cryptd_free_skcipher); | |
954 | ||
ace13663 HY |
955 | struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, |
956 | u32 type, u32 mask) | |
957 | { | |
958 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 959 | struct cryptd_hash_ctx *ctx; |
ace13663 HY |
960 | struct crypto_ahash *tfm; |
961 | ||
962 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
963 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
964 | return ERR_PTR(-EINVAL); | |
965 | tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); | |
966 | if (IS_ERR(tfm)) | |
967 | return ERR_CAST(tfm); | |
968 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
969 | crypto_free_ahash(tfm); | |
970 | return ERR_PTR(-EINVAL); | |
971 | } | |
972 | ||
81760ea6 | 973 | ctx = crypto_ahash_ctx(tfm); |
43b970fa | 974 | refcount_set(&ctx->refcnt, 1); |
81760ea6 | 975 | |
ace13663 HY |
976 | return __cryptd_ahash_cast(tfm); |
977 | } | |
978 | EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); | |
979 | ||
980 | struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) | |
981 | { | |
982 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
983 | ||
984 | return ctx->child; | |
985 | } | |
986 | EXPORT_SYMBOL_GPL(cryptd_ahash_child); | |
987 | ||
0e1227d3 HY |
988 | struct shash_desc *cryptd_shash_desc(struct ahash_request *req) |
989 | { | |
990 | struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
991 | return &rctx->desc; | |
992 | } | |
993 | EXPORT_SYMBOL_GPL(cryptd_shash_desc); | |
994 | ||
81760ea6 HX |
995 | bool cryptd_ahash_queued(struct cryptd_ahash *tfm) |
996 | { | |
997 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
998 | ||
43b970fa | 999 | return refcount_read(&ctx->refcnt) - 1; |
81760ea6 HX |
1000 | } |
1001 | EXPORT_SYMBOL_GPL(cryptd_ahash_queued); | |
1002 | ||
ace13663 HY |
1003 | void cryptd_free_ahash(struct cryptd_ahash *tfm) |
1004 | { | |
81760ea6 HX |
1005 | struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); |
1006 | ||
43b970fa | 1007 | if (refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 1008 | crypto_free_ahash(&tfm->base); |
ace13663 HY |
1009 | } |
1010 | EXPORT_SYMBOL_GPL(cryptd_free_ahash); | |
1011 | ||
298c926c AH |
1012 | struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, |
1013 | u32 type, u32 mask) | |
1014 | { | |
1015 | char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
81760ea6 | 1016 | struct cryptd_aead_ctx *ctx; |
298c926c AH |
1017 | struct crypto_aead *tfm; |
1018 | ||
1019 | if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
1020 | "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
1021 | return ERR_PTR(-EINVAL); | |
1022 | tfm = crypto_alloc_aead(cryptd_alg_name, type, mask); | |
1023 | if (IS_ERR(tfm)) | |
1024 | return ERR_CAST(tfm); | |
1025 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
1026 | crypto_free_aead(tfm); | |
1027 | return ERR_PTR(-EINVAL); | |
1028 | } | |
81760ea6 HX |
1029 | |
1030 | ctx = crypto_aead_ctx(tfm); | |
43b970fa | 1031 | refcount_set(&ctx->refcnt, 1); |
81760ea6 | 1032 | |
298c926c AH |
1033 | return __cryptd_aead_cast(tfm); |
1034 | } | |
1035 | EXPORT_SYMBOL_GPL(cryptd_alloc_aead); | |
1036 | ||
1037 | struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) | |
1038 | { | |
1039 | struct cryptd_aead_ctx *ctx; | |
1040 | ctx = crypto_aead_ctx(&tfm->base); | |
1041 | return ctx->child; | |
1042 | } | |
1043 | EXPORT_SYMBOL_GPL(cryptd_aead_child); | |
1044 | ||
81760ea6 HX |
1045 | bool cryptd_aead_queued(struct cryptd_aead *tfm) |
1046 | { | |
1047 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); | |
1048 | ||
43b970fa | 1049 | return refcount_read(&ctx->refcnt) - 1; |
81760ea6 HX |
1050 | } |
1051 | EXPORT_SYMBOL_GPL(cryptd_aead_queued); | |
1052 | ||
298c926c AH |
1053 | void cryptd_free_aead(struct cryptd_aead *tfm) |
1054 | { | |
81760ea6 HX |
1055 | struct cryptd_aead_ctx *ctx = crypto_aead_ctx(&tfm->base); |
1056 | ||
43b970fa | 1057 | if (refcount_dec_and_test(&ctx->refcnt)) |
81760ea6 | 1058 | crypto_free_aead(&tfm->base); |
298c926c AH |
1059 | } |
1060 | EXPORT_SYMBOL_GPL(cryptd_free_aead); | |
1061 | ||
124b53d0 HX |
1062 | static int __init cryptd_init(void) |
1063 | { | |
1064 | int err; | |
1065 | ||
3e56e168 EB |
1066 | cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, |
1067 | 1); | |
1068 | if (!cryptd_wq) | |
1069 | return -ENOMEM; | |
1070 | ||
c3a53605 | 1071 | err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen); |
124b53d0 | 1072 | if (err) |
3e56e168 | 1073 | goto err_destroy_wq; |
124b53d0 HX |
1074 | |
1075 | err = crypto_register_template(&cryptd_tmpl); | |
1076 | if (err) | |
3e56e168 | 1077 | goto err_fini_queue; |
124b53d0 | 1078 | |
3e56e168 EB |
1079 | return 0; |
1080 | ||
1081 | err_fini_queue: | |
1082 | cryptd_fini_queue(&queue); | |
1083 | err_destroy_wq: | |
1084 | destroy_workqueue(cryptd_wq); | |
124b53d0 HX |
1085 | return err; |
1086 | } | |
1087 | ||
1088 | static void __exit cryptd_exit(void) | |
1089 | { | |
3e56e168 | 1090 | destroy_workqueue(cryptd_wq); |
254eff77 | 1091 | cryptd_fini_queue(&queue); |
124b53d0 HX |
1092 | crypto_unregister_template(&cryptd_tmpl); |
1093 | } | |
1094 | ||
b2bac6ac | 1095 | subsys_initcall(cryptd_init); |
124b53d0 HX |
1096 | module_exit(cryptd_exit); |
1097 | ||
1098 | MODULE_LICENSE("GPL"); | |
1099 | MODULE_DESCRIPTION("Software async crypto daemon"); | |
4943ba16 | 1100 | MODULE_ALIAS_CRYPTO("cryptd"); |