]>
Commit | Line | Data |
---|---|---|
1e65b81a TC |
1 | /* |
2 | * Software multibuffer async crypto daemon. | |
3 | * | |
4 | * Copyright (c) 2014 Tim Chen <tim.c.chen@linux.intel.com> | |
5 | * | |
6 | * Adapted from crypto daemon. | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify it | |
9 | * under the terms of the GNU General Public License as published by the Free | |
10 | * Software Foundation; either version 2 of the License, or (at your option) | |
11 | * any later version. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <crypto/algapi.h> | |
16 | #include <crypto/internal/hash.h> | |
17 | #include <crypto/internal/aead.h> | |
18 | #include <crypto/mcryptd.h> | |
19 | #include <crypto/crypto_wq.h> | |
20 | #include <linux/err.h> | |
21 | #include <linux/init.h> | |
22 | #include <linux/kernel.h> | |
23 | #include <linux/list.h> | |
24 | #include <linux/module.h> | |
25 | #include <linux/scatterlist.h> | |
26 | #include <linux/sched.h> | |
27 | #include <linux/slab.h> | |
28 | #include <linux/hardirq.h> | |
29 | ||
30 | #define MCRYPTD_MAX_CPU_QLEN 100 | |
31 | #define MCRYPTD_BATCH 9 | |
32 | ||
33 | static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, | |
34 | unsigned int tail); | |
35 | ||
36 | struct mcryptd_flush_list { | |
37 | struct list_head list; | |
38 | struct mutex lock; | |
39 | }; | |
40 | ||
1f6e97f6 | 41 | static struct mcryptd_flush_list __percpu *mcryptd_flist; |
1e65b81a TC |
42 | |
43 | struct hashd_instance_ctx { | |
331bf739 | 44 | struct crypto_ahash_spawn spawn; |
1e65b81a TC |
45 | struct mcryptd_queue *queue; |
46 | }; | |
47 | ||
48 | static void mcryptd_queue_worker(struct work_struct *work); | |
49 | ||
50 | void mcryptd_arm_flusher(struct mcryptd_alg_cstate *cstate, unsigned long delay) | |
51 | { | |
52 | struct mcryptd_flush_list *flist; | |
53 | ||
54 | if (!cstate->flusher_engaged) { | |
55 | /* put the flusher on the flush list */ | |
56 | flist = per_cpu_ptr(mcryptd_flist, smp_processor_id()); | |
57 | mutex_lock(&flist->lock); | |
58 | list_add_tail(&cstate->flush_list, &flist->list); | |
59 | cstate->flusher_engaged = true; | |
60 | cstate->next_flush = jiffies + delay; | |
61 | queue_delayed_work_on(smp_processor_id(), kcrypto_wq, | |
62 | &cstate->flush, delay); | |
63 | mutex_unlock(&flist->lock); | |
64 | } | |
65 | } | |
66 | EXPORT_SYMBOL(mcryptd_arm_flusher); | |
67 | ||
68 | static int mcryptd_init_queue(struct mcryptd_queue *queue, | |
69 | unsigned int max_cpu_qlen) | |
70 | { | |
71 | int cpu; | |
72 | struct mcryptd_cpu_queue *cpu_queue; | |
73 | ||
74 | queue->cpu_queue = alloc_percpu(struct mcryptd_cpu_queue); | |
75 | pr_debug("mqueue:%p mcryptd_cpu_queue %p\n", queue, queue->cpu_queue); | |
76 | if (!queue->cpu_queue) | |
77 | return -ENOMEM; | |
78 | for_each_possible_cpu(cpu) { | |
79 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
80 | pr_debug("cpu_queue #%d %p\n", cpu, queue->cpu_queue); | |
81 | crypto_init_queue(&cpu_queue->queue, max_cpu_qlen); | |
82 | INIT_WORK(&cpu_queue->work, mcryptd_queue_worker); | |
83 | } | |
84 | return 0; | |
85 | } | |
86 | ||
87 | static void mcryptd_fini_queue(struct mcryptd_queue *queue) | |
88 | { | |
89 | int cpu; | |
90 | struct mcryptd_cpu_queue *cpu_queue; | |
91 | ||
92 | for_each_possible_cpu(cpu) { | |
93 | cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); | |
94 | BUG_ON(cpu_queue->queue.qlen); | |
95 | } | |
96 | free_percpu(queue->cpu_queue); | |
97 | } | |
98 | ||
99 | static int mcryptd_enqueue_request(struct mcryptd_queue *queue, | |
100 | struct crypto_async_request *request, | |
101 | struct mcryptd_hash_request_ctx *rctx) | |
102 | { | |
103 | int cpu, err; | |
104 | struct mcryptd_cpu_queue *cpu_queue; | |
105 | ||
106 | cpu = get_cpu(); | |
107 | cpu_queue = this_cpu_ptr(queue->cpu_queue); | |
108 | rctx->tag.cpu = cpu; | |
109 | ||
110 | err = crypto_enqueue_request(&cpu_queue->queue, request); | |
111 | pr_debug("enqueue request: cpu %d cpu_queue %p request %p\n", | |
112 | cpu, cpu_queue, request); | |
113 | queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); | |
114 | put_cpu(); | |
115 | ||
116 | return err; | |
117 | } | |
118 | ||
119 | /* | |
120 | * Try to opportunisticlly flush the partially completed jobs if | |
121 | * crypto daemon is the only task running. | |
122 | */ | |
123 | static void mcryptd_opportunistic_flush(void) | |
124 | { | |
125 | struct mcryptd_flush_list *flist; | |
126 | struct mcryptd_alg_cstate *cstate; | |
127 | ||
128 | flist = per_cpu_ptr(mcryptd_flist, smp_processor_id()); | |
129 | while (single_task_running()) { | |
130 | mutex_lock(&flist->lock); | |
08346170 | 131 | cstate = list_first_entry_or_null(&flist->list, |
1e65b81a | 132 | struct mcryptd_alg_cstate, flush_list); |
08346170 | 133 | if (!cstate || !cstate->flusher_engaged) { |
1e65b81a TC |
134 | mutex_unlock(&flist->lock); |
135 | return; | |
136 | } | |
137 | list_del(&cstate->flush_list); | |
138 | cstate->flusher_engaged = false; | |
139 | mutex_unlock(&flist->lock); | |
140 | cstate->alg_state->flusher(cstate); | |
141 | } | |
142 | } | |
143 | ||
144 | /* | |
145 | * Called in workqueue context, do one real cryption work (via | |
146 | * req->complete) and reschedule itself if there are more work to | |
147 | * do. | |
148 | */ | |
149 | static void mcryptd_queue_worker(struct work_struct *work) | |
150 | { | |
151 | struct mcryptd_cpu_queue *cpu_queue; | |
152 | struct crypto_async_request *req, *backlog; | |
153 | int i; | |
154 | ||
155 | /* | |
156 | * Need to loop through more than once for multi-buffer to | |
157 | * be effective. | |
158 | */ | |
159 | ||
160 | cpu_queue = container_of(work, struct mcryptd_cpu_queue, work); | |
161 | i = 0; | |
162 | while (i < MCRYPTD_BATCH || single_task_running()) { | |
163 | /* | |
164 | * preempt_disable/enable is used to prevent | |
165 | * being preempted by mcryptd_enqueue_request() | |
166 | */ | |
167 | local_bh_disable(); | |
168 | preempt_disable(); | |
169 | backlog = crypto_get_backlog(&cpu_queue->queue); | |
170 | req = crypto_dequeue_request(&cpu_queue->queue); | |
171 | preempt_enable(); | |
172 | local_bh_enable(); | |
173 | ||
174 | if (!req) { | |
175 | mcryptd_opportunistic_flush(); | |
176 | return; | |
177 | } | |
178 | ||
179 | if (backlog) | |
180 | backlog->complete(backlog, -EINPROGRESS); | |
181 | req->complete(req, 0); | |
182 | if (!cpu_queue->queue.qlen) | |
183 | return; | |
184 | ++i; | |
185 | } | |
186 | if (cpu_queue->queue.qlen) | |
187 | queue_work(kcrypto_wq, &cpu_queue->work); | |
188 | } | |
189 | ||
190 | void mcryptd_flusher(struct work_struct *__work) | |
191 | { | |
192 | struct mcryptd_alg_cstate *alg_cpu_state; | |
193 | struct mcryptd_alg_state *alg_state; | |
194 | struct mcryptd_flush_list *flist; | |
195 | int cpu; | |
196 | ||
197 | cpu = smp_processor_id(); | |
198 | alg_cpu_state = container_of(to_delayed_work(__work), | |
199 | struct mcryptd_alg_cstate, flush); | |
200 | alg_state = alg_cpu_state->alg_state; | |
201 | if (alg_cpu_state->cpu != cpu) | |
202 | pr_debug("mcryptd error: work on cpu %d, should be cpu %d\n", | |
203 | cpu, alg_cpu_state->cpu); | |
204 | ||
205 | if (alg_cpu_state->flusher_engaged) { | |
206 | flist = per_cpu_ptr(mcryptd_flist, cpu); | |
207 | mutex_lock(&flist->lock); | |
208 | list_del(&alg_cpu_state->flush_list); | |
209 | alg_cpu_state->flusher_engaged = false; | |
210 | mutex_unlock(&flist->lock); | |
211 | alg_state->flusher(alg_cpu_state); | |
212 | } | |
213 | } | |
214 | EXPORT_SYMBOL_GPL(mcryptd_flusher); | |
215 | ||
216 | static inline struct mcryptd_queue *mcryptd_get_queue(struct crypto_tfm *tfm) | |
217 | { | |
218 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
219 | struct mcryptd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
220 | ||
221 | return ictx->queue; | |
222 | } | |
223 | ||
224 | static void *mcryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, | |
225 | unsigned int tail) | |
226 | { | |
227 | char *p; | |
228 | struct crypto_instance *inst; | |
229 | int err; | |
230 | ||
231 | p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL); | |
232 | if (!p) | |
233 | return ERR_PTR(-ENOMEM); | |
234 | ||
235 | inst = (void *)(p + head); | |
236 | ||
237 | err = -ENAMETOOLONG; | |
238 | if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, | |
239 | "mcryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) | |
240 | goto out_free_inst; | |
241 | ||
242 | memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); | |
243 | ||
244 | inst->alg.cra_priority = alg->cra_priority + 50; | |
245 | inst->alg.cra_blocksize = alg->cra_blocksize; | |
246 | inst->alg.cra_alignmask = alg->cra_alignmask; | |
247 | ||
248 | out: | |
249 | return p; | |
250 | ||
251 | out_free_inst: | |
252 | kfree(p); | |
253 | p = ERR_PTR(err); | |
254 | goto out; | |
255 | } | |
256 | ||
f52bbf55 SM |
257 | static inline void mcryptd_check_internal(struct rtattr **tb, u32 *type, |
258 | u32 *mask) | |
259 | { | |
260 | struct crypto_attr_type *algt; | |
261 | ||
262 | algt = crypto_get_attr_type(tb); | |
263 | if (IS_ERR(algt)) | |
264 | return; | |
265 | if ((algt->type & CRYPTO_ALG_INTERNAL)) | |
266 | *type |= CRYPTO_ALG_INTERNAL; | |
267 | if ((algt->mask & CRYPTO_ALG_INTERNAL)) | |
268 | *mask |= CRYPTO_ALG_INTERNAL; | |
269 | } | |
270 | ||
1e65b81a TC |
271 | static int mcryptd_hash_init_tfm(struct crypto_tfm *tfm) |
272 | { | |
273 | struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); | |
274 | struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); | |
331bf739 | 275 | struct crypto_ahash_spawn *spawn = &ictx->spawn; |
1e65b81a | 276 | struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); |
331bf739 | 277 | struct crypto_ahash *hash; |
1e65b81a | 278 | |
331bf739 | 279 | hash = crypto_spawn_ahash(spawn); |
1e65b81a TC |
280 | if (IS_ERR(hash)) |
281 | return PTR_ERR(hash); | |
282 | ||
283 | ctx->child = hash; | |
284 | crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), | |
285 | sizeof(struct mcryptd_hash_request_ctx) + | |
331bf739 | 286 | crypto_ahash_reqsize(hash)); |
1e65b81a TC |
287 | return 0; |
288 | } | |
289 | ||
290 | static void mcryptd_hash_exit_tfm(struct crypto_tfm *tfm) | |
291 | { | |
292 | struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); | |
293 | ||
331bf739 | 294 | crypto_free_ahash(ctx->child); |
1e65b81a TC |
295 | } |
296 | ||
297 | static int mcryptd_hash_setkey(struct crypto_ahash *parent, | |
298 | const u8 *key, unsigned int keylen) | |
299 | { | |
300 | struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); | |
331bf739 | 301 | struct crypto_ahash *child = ctx->child; |
1e65b81a TC |
302 | int err; |
303 | ||
331bf739 MD |
304 | crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK); |
305 | crypto_ahash_set_flags(child, crypto_ahash_get_flags(parent) & | |
1e65b81a | 306 | CRYPTO_TFM_REQ_MASK); |
331bf739 MD |
307 | err = crypto_ahash_setkey(child, key, keylen); |
308 | crypto_ahash_set_flags(parent, crypto_ahash_get_flags(child) & | |
1e65b81a TC |
309 | CRYPTO_TFM_RES_MASK); |
310 | return err; | |
311 | } | |
312 | ||
313 | static int mcryptd_hash_enqueue(struct ahash_request *req, | |
314 | crypto_completion_t complete) | |
315 | { | |
316 | int ret; | |
317 | ||
318 | struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
319 | struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); | |
320 | struct mcryptd_queue *queue = | |
321 | mcryptd_get_queue(crypto_ahash_tfm(tfm)); | |
322 | ||
323 | rctx->complete = req->base.complete; | |
324 | req->base.complete = complete; | |
325 | ||
326 | ret = mcryptd_enqueue_request(queue, &req->base, rctx); | |
327 | ||
328 | return ret; | |
329 | } | |
330 | ||
331 | static void mcryptd_hash_init(struct crypto_async_request *req_async, int err) | |
332 | { | |
333 | struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); | |
331bf739 | 334 | struct crypto_ahash *child = ctx->child; |
1e65b81a TC |
335 | struct ahash_request *req = ahash_request_cast(req_async); |
336 | struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
331bf739 | 337 | struct ahash_request *desc = &rctx->areq; |
1e65b81a TC |
338 | |
339 | if (unlikely(err == -EINPROGRESS)) | |
340 | goto out; | |
341 | ||
331bf739 MD |
342 | ahash_request_set_tfm(desc, child); |
343 | ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP, | |
344 | rctx->complete, req_async); | |
1e65b81a | 345 | |
331bf739 MD |
346 | rctx->out = req->result; |
347 | err = crypto_ahash_init(desc); | |
1e65b81a TC |
348 | |
349 | out: | |
350 | local_bh_disable(); | |
351 | rctx->complete(&req->base, err); | |
352 | local_bh_enable(); | |
353 | } | |
354 | ||
355 | static int mcryptd_hash_init_enqueue(struct ahash_request *req) | |
356 | { | |
357 | return mcryptd_hash_enqueue(req, mcryptd_hash_init); | |
358 | } | |
359 | ||
360 | static void mcryptd_hash_update(struct crypto_async_request *req_async, int err) | |
361 | { | |
362 | struct ahash_request *req = ahash_request_cast(req_async); | |
363 | struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
364 | ||
365 | if (unlikely(err == -EINPROGRESS)) | |
366 | goto out; | |
367 | ||
331bf739 MD |
368 | rctx->out = req->result; |
369 | err = ahash_mcryptd_update(&rctx->areq); | |
1e65b81a TC |
370 | if (err) { |
371 | req->base.complete = rctx->complete; | |
372 | goto out; | |
373 | } | |
374 | ||
375 | return; | |
376 | out: | |
377 | local_bh_disable(); | |
378 | rctx->complete(&req->base, err); | |
379 | local_bh_enable(); | |
380 | } | |
381 | ||
382 | static int mcryptd_hash_update_enqueue(struct ahash_request *req) | |
383 | { | |
384 | return mcryptd_hash_enqueue(req, mcryptd_hash_update); | |
385 | } | |
386 | ||
387 | static void mcryptd_hash_final(struct crypto_async_request *req_async, int err) | |
388 | { | |
389 | struct ahash_request *req = ahash_request_cast(req_async); | |
390 | struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
391 | ||
392 | if (unlikely(err == -EINPROGRESS)) | |
393 | goto out; | |
394 | ||
331bf739 MD |
395 | rctx->out = req->result; |
396 | err = ahash_mcryptd_final(&rctx->areq); | |
1e65b81a TC |
397 | if (err) { |
398 | req->base.complete = rctx->complete; | |
399 | goto out; | |
400 | } | |
401 | ||
402 | return; | |
403 | out: | |
404 | local_bh_disable(); | |
405 | rctx->complete(&req->base, err); | |
406 | local_bh_enable(); | |
407 | } | |
408 | ||
409 | static int mcryptd_hash_final_enqueue(struct ahash_request *req) | |
410 | { | |
411 | return mcryptd_hash_enqueue(req, mcryptd_hash_final); | |
412 | } | |
413 | ||
414 | static void mcryptd_hash_finup(struct crypto_async_request *req_async, int err) | |
415 | { | |
416 | struct ahash_request *req = ahash_request_cast(req_async); | |
417 | struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
418 | ||
419 | if (unlikely(err == -EINPROGRESS)) | |
420 | goto out; | |
331bf739 MD |
421 | rctx->out = req->result; |
422 | err = ahash_mcryptd_finup(&rctx->areq); | |
1e65b81a TC |
423 | |
424 | if (err) { | |
425 | req->base.complete = rctx->complete; | |
426 | goto out; | |
427 | } | |
428 | ||
429 | return; | |
430 | out: | |
431 | local_bh_disable(); | |
432 | rctx->complete(&req->base, err); | |
433 | local_bh_enable(); | |
434 | } | |
435 | ||
436 | static int mcryptd_hash_finup_enqueue(struct ahash_request *req) | |
437 | { | |
438 | return mcryptd_hash_enqueue(req, mcryptd_hash_finup); | |
439 | } | |
440 | ||
441 | static void mcryptd_hash_digest(struct crypto_async_request *req_async, int err) | |
442 | { | |
443 | struct mcryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); | |
331bf739 | 444 | struct crypto_ahash *child = ctx->child; |
1e65b81a TC |
445 | struct ahash_request *req = ahash_request_cast(req_async); |
446 | struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
331bf739 | 447 | struct ahash_request *desc = &rctx->areq; |
1e65b81a TC |
448 | |
449 | if (unlikely(err == -EINPROGRESS)) | |
450 | goto out; | |
451 | ||
331bf739 MD |
452 | ahash_request_set_tfm(desc, child); |
453 | ahash_request_set_callback(desc, CRYPTO_TFM_REQ_MAY_SLEEP, | |
454 | rctx->complete, req_async); | |
1e65b81a | 455 | |
331bf739 MD |
456 | rctx->out = req->result; |
457 | err = ahash_mcryptd_digest(desc); | |
1e65b81a | 458 | |
1e65b81a TC |
459 | out: |
460 | local_bh_disable(); | |
461 | rctx->complete(&req->base, err); | |
462 | local_bh_enable(); | |
463 | } | |
464 | ||
465 | static int mcryptd_hash_digest_enqueue(struct ahash_request *req) | |
466 | { | |
467 | return mcryptd_hash_enqueue(req, mcryptd_hash_digest); | |
468 | } | |
469 | ||
470 | static int mcryptd_hash_export(struct ahash_request *req, void *out) | |
471 | { | |
472 | struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
473 | ||
331bf739 | 474 | return crypto_ahash_export(&rctx->areq, out); |
1e65b81a TC |
475 | } |
476 | ||
477 | static int mcryptd_hash_import(struct ahash_request *req, const void *in) | |
478 | { | |
479 | struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
480 | ||
331bf739 | 481 | return crypto_ahash_import(&rctx->areq, in); |
1e65b81a TC |
482 | } |
483 | ||
484 | static int mcryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, | |
485 | struct mcryptd_queue *queue) | |
486 | { | |
487 | struct hashd_instance_ctx *ctx; | |
488 | struct ahash_instance *inst; | |
331bf739 | 489 | struct hash_alg_common *halg; |
1e65b81a | 490 | struct crypto_alg *alg; |
f52bbf55 SM |
491 | u32 type = 0; |
492 | u32 mask = 0; | |
1e65b81a TC |
493 | int err; |
494 | ||
f52bbf55 SM |
495 | mcryptd_check_internal(tb, &type, &mask); |
496 | ||
331bf739 MD |
497 | halg = ahash_attr_alg(tb[1], type, mask); |
498 | if (IS_ERR(halg)) | |
499 | return PTR_ERR(halg); | |
1e65b81a | 500 | |
331bf739 | 501 | alg = &halg->base; |
1e65b81a TC |
502 | pr_debug("crypto: mcryptd hash alg: %s\n", alg->cra_name); |
503 | inst = mcryptd_alloc_instance(alg, ahash_instance_headroom(), | |
504 | sizeof(*ctx)); | |
505 | err = PTR_ERR(inst); | |
506 | if (IS_ERR(inst)) | |
507 | goto out_put_alg; | |
508 | ||
509 | ctx = ahash_instance_ctx(inst); | |
510 | ctx->queue = queue; | |
511 | ||
331bf739 | 512 | err = crypto_init_ahash_spawn(&ctx->spawn, halg, |
1e65b81a TC |
513 | ahash_crypto_instance(inst)); |
514 | if (err) | |
515 | goto out_free_inst; | |
516 | ||
f52bbf55 SM |
517 | type = CRYPTO_ALG_ASYNC; |
518 | if (alg->cra_flags & CRYPTO_ALG_INTERNAL) | |
519 | type |= CRYPTO_ALG_INTERNAL; | |
520 | inst->alg.halg.base.cra_flags = type; | |
1e65b81a | 521 | |
331bf739 MD |
522 | inst->alg.halg.digestsize = halg->digestsize; |
523 | inst->alg.halg.statesize = halg->statesize; | |
1e65b81a TC |
524 | inst->alg.halg.base.cra_ctxsize = sizeof(struct mcryptd_hash_ctx); |
525 | ||
526 | inst->alg.halg.base.cra_init = mcryptd_hash_init_tfm; | |
527 | inst->alg.halg.base.cra_exit = mcryptd_hash_exit_tfm; | |
528 | ||
529 | inst->alg.init = mcryptd_hash_init_enqueue; | |
530 | inst->alg.update = mcryptd_hash_update_enqueue; | |
531 | inst->alg.final = mcryptd_hash_final_enqueue; | |
532 | inst->alg.finup = mcryptd_hash_finup_enqueue; | |
533 | inst->alg.export = mcryptd_hash_export; | |
534 | inst->alg.import = mcryptd_hash_import; | |
535 | inst->alg.setkey = mcryptd_hash_setkey; | |
536 | inst->alg.digest = mcryptd_hash_digest_enqueue; | |
537 | ||
538 | err = ahash_register_instance(tmpl, inst); | |
539 | if (err) { | |
331bf739 | 540 | crypto_drop_ahash(&ctx->spawn); |
1e65b81a TC |
541 | out_free_inst: |
542 | kfree(inst); | |
543 | } | |
544 | ||
545 | out_put_alg: | |
546 | crypto_mod_put(alg); | |
547 | return err; | |
548 | } | |
549 | ||
550 | static struct mcryptd_queue mqueue; | |
551 | ||
552 | static int mcryptd_create(struct crypto_template *tmpl, struct rtattr **tb) | |
553 | { | |
554 | struct crypto_attr_type *algt; | |
555 | ||
556 | algt = crypto_get_attr_type(tb); | |
557 | if (IS_ERR(algt)) | |
558 | return PTR_ERR(algt); | |
559 | ||
560 | switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { | |
561 | case CRYPTO_ALG_TYPE_DIGEST: | |
562 | return mcryptd_create_hash(tmpl, tb, &mqueue); | |
563 | break; | |
564 | } | |
565 | ||
566 | return -EINVAL; | |
567 | } | |
568 | ||
569 | static void mcryptd_free(struct crypto_instance *inst) | |
570 | { | |
571 | struct mcryptd_instance_ctx *ctx = crypto_instance_ctx(inst); | |
572 | struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst); | |
573 | ||
574 | switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { | |
575 | case CRYPTO_ALG_TYPE_AHASH: | |
331bf739 | 576 | crypto_drop_ahash(&hctx->spawn); |
1e65b81a TC |
577 | kfree(ahash_instance(inst)); |
578 | return; | |
579 | default: | |
580 | crypto_drop_spawn(&ctx->spawn); | |
581 | kfree(inst); | |
582 | } | |
583 | } | |
584 | ||
585 | static struct crypto_template mcryptd_tmpl = { | |
586 | .name = "mcryptd", | |
587 | .create = mcryptd_create, | |
588 | .free = mcryptd_free, | |
589 | .module = THIS_MODULE, | |
590 | }; | |
591 | ||
592 | struct mcryptd_ahash *mcryptd_alloc_ahash(const char *alg_name, | |
593 | u32 type, u32 mask) | |
594 | { | |
595 | char mcryptd_alg_name[CRYPTO_MAX_ALG_NAME]; | |
596 | struct crypto_ahash *tfm; | |
597 | ||
598 | if (snprintf(mcryptd_alg_name, CRYPTO_MAX_ALG_NAME, | |
599 | "mcryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) | |
600 | return ERR_PTR(-EINVAL); | |
601 | tfm = crypto_alloc_ahash(mcryptd_alg_name, type, mask); | |
602 | if (IS_ERR(tfm)) | |
603 | return ERR_CAST(tfm); | |
604 | if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { | |
605 | crypto_free_ahash(tfm); | |
606 | return ERR_PTR(-EINVAL); | |
607 | } | |
608 | ||
609 | return __mcryptd_ahash_cast(tfm); | |
610 | } | |
611 | EXPORT_SYMBOL_GPL(mcryptd_alloc_ahash); | |
612 | ||
331bf739 | 613 | int ahash_mcryptd_digest(struct ahash_request *desc) |
1e65b81a | 614 | { |
36e09e1f | 615 | return crypto_ahash_init(desc) ?: ahash_mcryptd_finup(desc); |
1e65b81a | 616 | } |
1e65b81a | 617 | |
331bf739 | 618 | int ahash_mcryptd_update(struct ahash_request *desc) |
1e65b81a | 619 | { |
1e65b81a TC |
620 | /* alignment is to be done by multi-buffer crypto algorithm if needed */ |
621 | ||
331bf739 | 622 | return crypto_ahash_update(desc); |
1e65b81a | 623 | } |
1e65b81a | 624 | |
331bf739 | 625 | int ahash_mcryptd_finup(struct ahash_request *desc) |
1e65b81a | 626 | { |
1e65b81a TC |
627 | /* alignment is to be done by multi-buffer crypto algorithm if needed */ |
628 | ||
331bf739 | 629 | return crypto_ahash_finup(desc); |
1e65b81a | 630 | } |
1e65b81a | 631 | |
331bf739 | 632 | int ahash_mcryptd_final(struct ahash_request *desc) |
1e65b81a | 633 | { |
1e65b81a TC |
634 | /* alignment is to be done by multi-buffer crypto algorithm if needed */ |
635 | ||
331bf739 | 636 | return crypto_ahash_final(desc); |
1e65b81a | 637 | } |
1e65b81a | 638 | |
331bf739 | 639 | struct crypto_ahash *mcryptd_ahash_child(struct mcryptd_ahash *tfm) |
1e65b81a TC |
640 | { |
641 | struct mcryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); | |
642 | ||
643 | return ctx->child; | |
644 | } | |
645 | EXPORT_SYMBOL_GPL(mcryptd_ahash_child); | |
646 | ||
331bf739 | 647 | struct ahash_request *mcryptd_ahash_desc(struct ahash_request *req) |
1e65b81a TC |
648 | { |
649 | struct mcryptd_hash_request_ctx *rctx = ahash_request_ctx(req); | |
331bf739 | 650 | return &rctx->areq; |
1e65b81a | 651 | } |
331bf739 | 652 | EXPORT_SYMBOL_GPL(mcryptd_ahash_desc); |
1e65b81a TC |
653 | |
654 | void mcryptd_free_ahash(struct mcryptd_ahash *tfm) | |
655 | { | |
656 | crypto_free_ahash(&tfm->base); | |
657 | } | |
658 | EXPORT_SYMBOL_GPL(mcryptd_free_ahash); | |
659 | ||
1e65b81a TC |
660 | static int __init mcryptd_init(void) |
661 | { | |
662 | int err, cpu; | |
663 | struct mcryptd_flush_list *flist; | |
664 | ||
665 | mcryptd_flist = alloc_percpu(struct mcryptd_flush_list); | |
666 | for_each_possible_cpu(cpu) { | |
667 | flist = per_cpu_ptr(mcryptd_flist, cpu); | |
668 | INIT_LIST_HEAD(&flist->list); | |
669 | mutex_init(&flist->lock); | |
670 | } | |
671 | ||
672 | err = mcryptd_init_queue(&mqueue, MCRYPTD_MAX_CPU_QLEN); | |
673 | if (err) { | |
674 | free_percpu(mcryptd_flist); | |
675 | return err; | |
676 | } | |
677 | ||
678 | err = crypto_register_template(&mcryptd_tmpl); | |
679 | if (err) { | |
680 | mcryptd_fini_queue(&mqueue); | |
681 | free_percpu(mcryptd_flist); | |
682 | } | |
683 | ||
684 | return err; | |
685 | } | |
686 | ||
687 | static void __exit mcryptd_exit(void) | |
688 | { | |
689 | mcryptd_fini_queue(&mqueue); | |
690 | crypto_unregister_template(&mcryptd_tmpl); | |
691 | free_percpu(mcryptd_flist); | |
692 | } | |
693 | ||
694 | subsys_initcall(mcryptd_init); | |
695 | module_exit(mcryptd_exit); | |
696 | ||
697 | MODULE_LICENSE("GPL"); | |
698 | MODULE_DESCRIPTION("Software async multibuffer crypto daemon"); | |
4943ba16 | 699 | MODULE_ALIAS_CRYPTO("mcryptd"); |