]>
Commit | Line | Data |
---|---|---|
8ff59090 HX |
1 | /* |
2 | * algif_skcipher: User-space interface for skcipher algorithms | |
3 | * | |
4 | * This file provides the user-space API for symmetric key ciphers. | |
5 | * | |
6 | * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au> | |
7 | * | |
8 | * This program is free software; you can redistribute it and/or modify it | |
9 | * under the terms of the GNU General Public License as published by the Free | |
10 | * Software Foundation; either version 2 of the License, or (at your option) | |
11 | * any later version. | |
12 | * | |
13 | */ | |
14 | ||
15 | #include <crypto/scatterwalk.h> | |
16 | #include <crypto/skcipher.h> | |
17 | #include <crypto/if_alg.h> | |
18 | #include <linux/init.h> | |
19 | #include <linux/list.h> | |
20 | #include <linux/kernel.h> | |
21 | #include <linux/mm.h> | |
22 | #include <linux/module.h> | |
23 | #include <linux/net.h> | |
24 | #include <net/sock.h> | |
25 | ||
26 | struct skcipher_sg_list { | |
27 | struct list_head list; | |
28 | ||
29 | int cur; | |
30 | ||
31 | struct scatterlist sg[0]; | |
32 | }; | |
33 | ||
b238717c HX |
34 | struct skcipher_tfm { |
35 | struct crypto_skcipher *skcipher; | |
36 | bool has_key; | |
37 | }; | |
38 | ||
8ff59090 HX |
39 | struct skcipher_ctx { |
40 | struct list_head tsgl; | |
41 | struct af_alg_sgl rsgl; | |
42 | ||
43 | void *iv; | |
44 | ||
45 | struct af_alg_completion completion; | |
46 | ||
a596999b | 47 | atomic_t inflight; |
8ff59090 HX |
48 | unsigned used; |
49 | ||
50 | unsigned int len; | |
51 | bool more; | |
52 | bool merge; | |
53 | bool enc; | |
54 | ||
0d96e4ba | 55 | struct skcipher_request req; |
8ff59090 HX |
56 | }; |
57 | ||
a596999b TS |
58 | struct skcipher_async_rsgl { |
59 | struct af_alg_sgl sgl; | |
60 | struct list_head list; | |
61 | }; | |
62 | ||
63 | struct skcipher_async_req { | |
64 | struct kiocb *iocb; | |
65 | struct skcipher_async_rsgl first_sgl; | |
66 | struct list_head list; | |
67 | struct scatterlist *tsg; | |
3e597bb7 HX |
68 | atomic_t *inflight; |
69 | struct skcipher_request req; | |
a596999b TS |
70 | }; |
71 | ||
e2cffb5f | 72 | #define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_sg_list)) / \ |
8ff59090 HX |
73 | sizeof(struct scatterlist) - 1) |
74 | ||
a596999b TS |
75 | static void skcipher_free_async_sgls(struct skcipher_async_req *sreq) |
76 | { | |
77 | struct skcipher_async_rsgl *rsgl, *tmp; | |
78 | struct scatterlist *sgl; | |
79 | struct scatterlist *sg; | |
80 | int i, n; | |
81 | ||
82 | list_for_each_entry_safe(rsgl, tmp, &sreq->list, list) { | |
83 | af_alg_free_sg(&rsgl->sgl); | |
84 | if (rsgl != &sreq->first_sgl) | |
85 | kfree(rsgl); | |
86 | } | |
87 | sgl = sreq->tsg; | |
88 | n = sg_nents(sgl); | |
1adade8f SM |
89 | for_each_sg(sgl, sg, n, i) { |
90 | struct page *page = sg_page(sg); | |
91 | ||
92 | /* some SGs may not have a page mapped */ | |
93 | if (page && atomic_read(&page->_count)) | |
94 | put_page(page); | |
95 | } | |
a596999b TS |
96 | |
97 | kfree(sreq->tsg); | |
98 | } | |
99 | ||
100 | static void skcipher_async_cb(struct crypto_async_request *req, int err) | |
101 | { | |
3e597bb7 | 102 | struct skcipher_async_req *sreq = req->data; |
a596999b TS |
103 | struct kiocb *iocb = sreq->iocb; |
104 | ||
3e597bb7 | 105 | atomic_dec(sreq->inflight); |
a596999b | 106 | skcipher_free_async_sgls(sreq); |
3e597bb7 | 107 | kzfree(sreq); |
237dae88 | 108 | iocb->ki_complete(iocb, err, err); |
a596999b TS |
109 | } |
110 | ||
0f6bb83c | 111 | static inline int skcipher_sndbuf(struct sock *sk) |
8ff59090 HX |
112 | { |
113 | struct alg_sock *ask = alg_sk(sk); | |
114 | struct skcipher_ctx *ctx = ask->private; | |
115 | ||
0f6bb83c HX |
116 | return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) - |
117 | ctx->used, 0); | |
118 | } | |
119 | ||
120 | static inline bool skcipher_writable(struct sock *sk) | |
121 | { | |
122 | return PAGE_SIZE <= skcipher_sndbuf(sk); | |
8ff59090 HX |
123 | } |
124 | ||
125 | static int skcipher_alloc_sgl(struct sock *sk) | |
126 | { | |
127 | struct alg_sock *ask = alg_sk(sk); | |
128 | struct skcipher_ctx *ctx = ask->private; | |
129 | struct skcipher_sg_list *sgl; | |
130 | struct scatterlist *sg = NULL; | |
131 | ||
132 | sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); | |
133 | if (!list_empty(&ctx->tsgl)) | |
134 | sg = sgl->sg; | |
135 | ||
136 | if (!sg || sgl->cur >= MAX_SGL_ENTS) { | |
137 | sgl = sock_kmalloc(sk, sizeof(*sgl) + | |
138 | sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1), | |
139 | GFP_KERNEL); | |
140 | if (!sgl) | |
141 | return -ENOMEM; | |
142 | ||
143 | sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); | |
144 | sgl->cur = 0; | |
145 | ||
24a65a22 | 146 | if (sg) { |
c56f6d12 | 147 | sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); |
24a65a22 SM |
148 | sg_unmark_end(sg + (MAX_SGL_ENTS - 1)); |
149 | } | |
8ff59090 HX |
150 | |
151 | list_add_tail(&sgl->list, &ctx->tsgl); | |
152 | } | |
153 | ||
154 | return 0; | |
155 | } | |
156 | ||
a596999b | 157 | static void skcipher_pull_sgl(struct sock *sk, int used, int put) |
8ff59090 HX |
158 | { |
159 | struct alg_sock *ask = alg_sk(sk); | |
160 | struct skcipher_ctx *ctx = ask->private; | |
161 | struct skcipher_sg_list *sgl; | |
162 | struct scatterlist *sg; | |
163 | int i; | |
164 | ||
165 | while (!list_empty(&ctx->tsgl)) { | |
166 | sgl = list_first_entry(&ctx->tsgl, struct skcipher_sg_list, | |
167 | list); | |
168 | sg = sgl->sg; | |
169 | ||
170 | for (i = 0; i < sgl->cur; i++) { | |
171 | int plen = min_t(int, used, sg[i].length); | |
172 | ||
173 | if (!sg_page(sg + i)) | |
174 | continue; | |
175 | ||
176 | sg[i].length -= plen; | |
177 | sg[i].offset += plen; | |
178 | ||
179 | used -= plen; | |
180 | ctx->used -= plen; | |
181 | ||
182 | if (sg[i].length) | |
183 | return; | |
a596999b TS |
184 | if (put) |
185 | put_page(sg_page(sg + i)); | |
8ff59090 HX |
186 | sg_assign_page(sg + i, NULL); |
187 | } | |
188 | ||
189 | list_del(&sgl->list); | |
190 | sock_kfree_s(sk, sgl, | |
191 | sizeof(*sgl) + sizeof(sgl->sg[0]) * | |
192 | (MAX_SGL_ENTS + 1)); | |
193 | } | |
194 | ||
195 | if (!ctx->used) | |
196 | ctx->merge = 0; | |
197 | } | |
198 | ||
199 | static void skcipher_free_sgl(struct sock *sk) | |
200 | { | |
201 | struct alg_sock *ask = alg_sk(sk); | |
202 | struct skcipher_ctx *ctx = ask->private; | |
203 | ||
a596999b | 204 | skcipher_pull_sgl(sk, ctx->used, 1); |
8ff59090 HX |
205 | } |
206 | ||
207 | static int skcipher_wait_for_wmem(struct sock *sk, unsigned flags) | |
208 | { | |
209 | long timeout; | |
210 | DEFINE_WAIT(wait); | |
211 | int err = -ERESTARTSYS; | |
212 | ||
213 | if (flags & MSG_DONTWAIT) | |
214 | return -EAGAIN; | |
215 | ||
9cd3e072 | 216 | sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); |
8ff59090 HX |
217 | |
218 | for (;;) { | |
219 | if (signal_pending(current)) | |
220 | break; | |
221 | prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); | |
222 | timeout = MAX_SCHEDULE_TIMEOUT; | |
223 | if (sk_wait_event(sk, &timeout, skcipher_writable(sk))) { | |
224 | err = 0; | |
225 | break; | |
226 | } | |
227 | } | |
228 | finish_wait(sk_sleep(sk), &wait); | |
229 | ||
230 | return err; | |
231 | } | |
232 | ||
233 | static void skcipher_wmem_wakeup(struct sock *sk) | |
234 | { | |
235 | struct socket_wq *wq; | |
236 | ||
237 | if (!skcipher_writable(sk)) | |
238 | return; | |
239 | ||
240 | rcu_read_lock(); | |
241 | wq = rcu_dereference(sk->sk_wq); | |
242 | if (wq_has_sleeper(wq)) | |
243 | wake_up_interruptible_sync_poll(&wq->wait, POLLIN | | |
244 | POLLRDNORM | | |
245 | POLLRDBAND); | |
246 | sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); | |
247 | rcu_read_unlock(); | |
248 | } | |
249 | ||
250 | static int skcipher_wait_for_data(struct sock *sk, unsigned flags) | |
251 | { | |
252 | struct alg_sock *ask = alg_sk(sk); | |
253 | struct skcipher_ctx *ctx = ask->private; | |
254 | long timeout; | |
255 | DEFINE_WAIT(wait); | |
256 | int err = -ERESTARTSYS; | |
257 | ||
258 | if (flags & MSG_DONTWAIT) { | |
259 | return -EAGAIN; | |
260 | } | |
261 | ||
9cd3e072 | 262 | sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); |
8ff59090 HX |
263 | |
264 | for (;;) { | |
265 | if (signal_pending(current)) | |
266 | break; | |
267 | prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE); | |
268 | timeout = MAX_SCHEDULE_TIMEOUT; | |
269 | if (sk_wait_event(sk, &timeout, ctx->used)) { | |
270 | err = 0; | |
271 | break; | |
272 | } | |
273 | } | |
274 | finish_wait(sk_sleep(sk), &wait); | |
275 | ||
9cd3e072 | 276 | sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); |
8ff59090 HX |
277 | |
278 | return err; | |
279 | } | |
280 | ||
281 | static void skcipher_data_wakeup(struct sock *sk) | |
282 | { | |
283 | struct alg_sock *ask = alg_sk(sk); | |
284 | struct skcipher_ctx *ctx = ask->private; | |
285 | struct socket_wq *wq; | |
286 | ||
287 | if (!ctx->used) | |
288 | return; | |
289 | ||
290 | rcu_read_lock(); | |
291 | wq = rcu_dereference(sk->sk_wq); | |
292 | if (wq_has_sleeper(wq)) | |
293 | wake_up_interruptible_sync_poll(&wq->wait, POLLOUT | | |
294 | POLLRDNORM | | |
295 | POLLRDBAND); | |
296 | sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); | |
297 | rcu_read_unlock(); | |
298 | } | |
299 | ||
1b784140 YX |
300 | static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg, |
301 | size_t size) | |
8ff59090 HX |
302 | { |
303 | struct sock *sk = sock->sk; | |
304 | struct alg_sock *ask = alg_sk(sk); | |
66bca97f HX |
305 | struct sock *psk = ask->parent; |
306 | struct alg_sock *pask = alg_sk(psk); | |
8ff59090 | 307 | struct skcipher_ctx *ctx = ask->private; |
66bca97f HX |
308 | struct skcipher_tfm *skc = pask->private; |
309 | struct crypto_skcipher *tfm = skc->skcipher; | |
0d96e4ba | 310 | unsigned ivsize = crypto_skcipher_ivsize(tfm); |
8ff59090 HX |
311 | struct skcipher_sg_list *sgl; |
312 | struct af_alg_control con = {}; | |
313 | long copied = 0; | |
314 | bool enc = 0; | |
f26b7b80 | 315 | bool init = 0; |
8ff59090 HX |
316 | int err; |
317 | int i; | |
318 | ||
319 | if (msg->msg_controllen) { | |
320 | err = af_alg_cmsg_send(msg, &con); | |
321 | if (err) | |
322 | return err; | |
323 | ||
f26b7b80 | 324 | init = 1; |
8ff59090 HX |
325 | switch (con.op) { |
326 | case ALG_OP_ENCRYPT: | |
327 | enc = 1; | |
328 | break; | |
329 | case ALG_OP_DECRYPT: | |
330 | enc = 0; | |
331 | break; | |
332 | default: | |
333 | return -EINVAL; | |
334 | } | |
335 | ||
336 | if (con.iv && con.iv->ivlen != ivsize) | |
337 | return -EINVAL; | |
338 | } | |
339 | ||
340 | err = -EINVAL; | |
341 | ||
342 | lock_sock(sk); | |
343 | if (!ctx->more && ctx->used) | |
344 | goto unlock; | |
345 | ||
f26b7b80 | 346 | if (init) { |
8ff59090 HX |
347 | ctx->enc = enc; |
348 | if (con.iv) | |
349 | memcpy(ctx->iv, con.iv->iv, ivsize); | |
350 | } | |
351 | ||
8ff59090 HX |
352 | while (size) { |
353 | struct scatterlist *sg; | |
354 | unsigned long len = size; | |
355 | int plen; | |
356 | ||
357 | if (ctx->merge) { | |
358 | sgl = list_entry(ctx->tsgl.prev, | |
359 | struct skcipher_sg_list, list); | |
360 | sg = sgl->sg + sgl->cur - 1; | |
361 | len = min_t(unsigned long, len, | |
362 | PAGE_SIZE - sg->offset - sg->length); | |
363 | ||
6ce8e9ce AV |
364 | err = memcpy_from_msg(page_address(sg_page(sg)) + |
365 | sg->offset + sg->length, | |
366 | msg, len); | |
8ff59090 HX |
367 | if (err) |
368 | goto unlock; | |
369 | ||
370 | sg->length += len; | |
371 | ctx->merge = (sg->offset + sg->length) & | |
372 | (PAGE_SIZE - 1); | |
373 | ||
374 | ctx->used += len; | |
375 | copied += len; | |
376 | size -= len; | |
8ff59090 HX |
377 | continue; |
378 | } | |
379 | ||
0f6bb83c | 380 | if (!skcipher_writable(sk)) { |
8ff59090 HX |
381 | err = skcipher_wait_for_wmem(sk, msg->msg_flags); |
382 | if (err) | |
383 | goto unlock; | |
8ff59090 HX |
384 | } |
385 | ||
0f6bb83c | 386 | len = min_t(unsigned long, len, skcipher_sndbuf(sk)); |
8ff59090 HX |
387 | |
388 | err = skcipher_alloc_sgl(sk); | |
389 | if (err) | |
390 | goto unlock; | |
391 | ||
392 | sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); | |
393 | sg = sgl->sg; | |
3dd3e254 HX |
394 | if (sgl->cur) |
395 | sg_unmark_end(sg + sgl->cur - 1); | |
8ff59090 HX |
396 | do { |
397 | i = sgl->cur; | |
398 | plen = min_t(int, len, PAGE_SIZE); | |
399 | ||
400 | sg_assign_page(sg + i, alloc_page(GFP_KERNEL)); | |
401 | err = -ENOMEM; | |
402 | if (!sg_page(sg + i)) | |
403 | goto unlock; | |
404 | ||
6ce8e9ce AV |
405 | err = memcpy_from_msg(page_address(sg_page(sg + i)), |
406 | msg, plen); | |
8ff59090 HX |
407 | if (err) { |
408 | __free_page(sg_page(sg + i)); | |
409 | sg_assign_page(sg + i, NULL); | |
410 | goto unlock; | |
411 | } | |
412 | ||
413 | sg[i].length = plen; | |
414 | len -= plen; | |
415 | ctx->used += plen; | |
416 | copied += plen; | |
417 | size -= plen; | |
8ff59090 HX |
418 | sgl->cur++; |
419 | } while (len && sgl->cur < MAX_SGL_ENTS); | |
420 | ||
0f477b65 TS |
421 | if (!size) |
422 | sg_mark_end(sg + sgl->cur - 1); | |
423 | ||
8ff59090 HX |
424 | ctx->merge = plen & (PAGE_SIZE - 1); |
425 | } | |
426 | ||
427 | err = 0; | |
428 | ||
429 | ctx->more = msg->msg_flags & MSG_MORE; | |
8ff59090 HX |
430 | |
431 | unlock: | |
432 | skcipher_data_wakeup(sk); | |
433 | release_sock(sk); | |
434 | ||
435 | return copied ?: err; | |
436 | } | |
437 | ||
438 | static ssize_t skcipher_sendpage(struct socket *sock, struct page *page, | |
439 | int offset, size_t size, int flags) | |
440 | { | |
441 | struct sock *sk = sock->sk; | |
442 | struct alg_sock *ask = alg_sk(sk); | |
443 | struct skcipher_ctx *ctx = ask->private; | |
444 | struct skcipher_sg_list *sgl; | |
445 | int err = -EINVAL; | |
8ff59090 | 446 | |
d3f7d56a SL |
447 | if (flags & MSG_SENDPAGE_NOTLAST) |
448 | flags |= MSG_MORE; | |
449 | ||
8ff59090 HX |
450 | lock_sock(sk); |
451 | if (!ctx->more && ctx->used) | |
452 | goto unlock; | |
453 | ||
454 | if (!size) | |
455 | goto done; | |
456 | ||
0f6bb83c | 457 | if (!skcipher_writable(sk)) { |
8ff59090 HX |
458 | err = skcipher_wait_for_wmem(sk, flags); |
459 | if (err) | |
460 | goto unlock; | |
8ff59090 HX |
461 | } |
462 | ||
463 | err = skcipher_alloc_sgl(sk); | |
464 | if (err) | |
465 | goto unlock; | |
466 | ||
467 | ctx->merge = 0; | |
468 | sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); | |
469 | ||
0f477b65 TS |
470 | if (sgl->cur) |
471 | sg_unmark_end(sgl->sg + sgl->cur - 1); | |
472 | ||
473 | sg_mark_end(sgl->sg + sgl->cur); | |
8ff59090 HX |
474 | get_page(page); |
475 | sg_set_page(sgl->sg + sgl->cur, page, size, offset); | |
476 | sgl->cur++; | |
477 | ctx->used += size; | |
478 | ||
479 | done: | |
480 | ctx->more = flags & MSG_MORE; | |
8ff59090 HX |
481 | |
482 | unlock: | |
483 | skcipher_data_wakeup(sk); | |
484 | release_sock(sk); | |
485 | ||
486 | return err ?: size; | |
487 | } | |
488 | ||
a596999b TS |
489 | static int skcipher_all_sg_nents(struct skcipher_ctx *ctx) |
490 | { | |
491 | struct skcipher_sg_list *sgl; | |
492 | struct scatterlist *sg; | |
493 | int nents = 0; | |
494 | ||
495 | list_for_each_entry(sgl, &ctx->tsgl, list) { | |
496 | sg = sgl->sg; | |
497 | ||
498 | while (!sg->length) | |
499 | sg++; | |
500 | ||
501 | nents += sg_nents(sg); | |
502 | } | |
503 | return nents; | |
504 | } | |
505 | ||
506 | static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg, | |
507 | int flags) | |
508 | { | |
509 | struct sock *sk = sock->sk; | |
510 | struct alg_sock *ask = alg_sk(sk); | |
3e597bb7 HX |
511 | struct sock *psk = ask->parent; |
512 | struct alg_sock *pask = alg_sk(psk); | |
a596999b | 513 | struct skcipher_ctx *ctx = ask->private; |
3e597bb7 HX |
514 | struct skcipher_tfm *skc = pask->private; |
515 | struct crypto_skcipher *tfm = skc->skcipher; | |
a596999b TS |
516 | struct skcipher_sg_list *sgl; |
517 | struct scatterlist *sg; | |
518 | struct skcipher_async_req *sreq; | |
0d96e4ba | 519 | struct skcipher_request *req; |
a596999b | 520 | struct skcipher_async_rsgl *last_rsgl = NULL; |
66bca97f | 521 | unsigned int txbufs = 0, len = 0, tx_nents; |
3e597bb7 HX |
522 | unsigned int reqsize = crypto_skcipher_reqsize(tfm); |
523 | unsigned int ivsize = crypto_skcipher_ivsize(tfm); | |
a596999b | 524 | int err = -ENOMEM; |
033f46b3 | 525 | bool mark = false; |
3e597bb7 | 526 | char *iv; |
a596999b | 527 | |
3e597bb7 HX |
528 | sreq = kzalloc(sizeof(*sreq) + reqsize + ivsize, GFP_KERNEL); |
529 | if (unlikely(!sreq)) | |
530 | goto out; | |
a596999b | 531 | |
3e597bb7 HX |
532 | req = &sreq->req; |
533 | iv = (char *)(req + 1) + reqsize; | |
a596999b | 534 | sreq->iocb = msg->msg_iocb; |
a596999b | 535 | INIT_LIST_HEAD(&sreq->list); |
3e597bb7 HX |
536 | sreq->inflight = &ctx->inflight; |
537 | ||
538 | lock_sock(sk); | |
66bca97f | 539 | tx_nents = skcipher_all_sg_nents(ctx); |
a596999b | 540 | sreq->tsg = kcalloc(tx_nents, sizeof(*sg), GFP_KERNEL); |
3e597bb7 | 541 | if (unlikely(!sreq->tsg)) |
a596999b | 542 | goto unlock; |
a596999b | 543 | sg_init_table(sreq->tsg, tx_nents); |
3e597bb7 HX |
544 | memcpy(iv, ctx->iv, ivsize); |
545 | skcipher_request_set_tfm(req, tfm); | |
12426bd6 | 546 | skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, |
3e597bb7 | 547 | skcipher_async_cb, sreq); |
a596999b TS |
548 | |
549 | while (iov_iter_count(&msg->msg_iter)) { | |
550 | struct skcipher_async_rsgl *rsgl; | |
ac110f49 | 551 | int used; |
a596999b TS |
552 | |
553 | if (!ctx->used) { | |
554 | err = skcipher_wait_for_data(sk, flags); | |
555 | if (err) | |
556 | goto free; | |
557 | } | |
558 | sgl = list_first_entry(&ctx->tsgl, | |
559 | struct skcipher_sg_list, list); | |
560 | sg = sgl->sg; | |
561 | ||
562 | while (!sg->length) | |
563 | sg++; | |
564 | ||
565 | used = min_t(unsigned long, ctx->used, | |
566 | iov_iter_count(&msg->msg_iter)); | |
567 | used = min_t(unsigned long, used, sg->length); | |
568 | ||
033f46b3 | 569 | if (txbufs == tx_nents) { |
a596999b TS |
570 | struct scatterlist *tmp; |
571 | int x; | |
572 | /* Ran out of tx slots in async request | |
573 | * need to expand */ | |
574 | tmp = kcalloc(tx_nents * 2, sizeof(*tmp), | |
575 | GFP_KERNEL); | |
576 | if (!tmp) | |
577 | goto free; | |
578 | ||
579 | sg_init_table(tmp, tx_nents * 2); | |
580 | for (x = 0; x < tx_nents; x++) | |
581 | sg_set_page(&tmp[x], sg_page(&sreq->tsg[x]), | |
582 | sreq->tsg[x].length, | |
583 | sreq->tsg[x].offset); | |
584 | kfree(sreq->tsg); | |
585 | sreq->tsg = tmp; | |
586 | tx_nents *= 2; | |
033f46b3 | 587 | mark = true; |
a596999b TS |
588 | } |
589 | /* Need to take over the tx sgl from ctx | |
590 | * to the asynch req - these sgls will be freed later */ | |
033f46b3 | 591 | sg_set_page(sreq->tsg + txbufs++, sg_page(sg), sg->length, |
a596999b TS |
592 | sg->offset); |
593 | ||
594 | if (list_empty(&sreq->list)) { | |
595 | rsgl = &sreq->first_sgl; | |
596 | list_add_tail(&rsgl->list, &sreq->list); | |
597 | } else { | |
82d92920 | 598 | rsgl = kmalloc(sizeof(*rsgl), GFP_KERNEL); |
a596999b TS |
599 | if (!rsgl) { |
600 | err = -ENOMEM; | |
601 | goto free; | |
602 | } | |
603 | list_add_tail(&rsgl->list, &sreq->list); | |
604 | } | |
605 | ||
606 | used = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, used); | |
607 | err = used; | |
608 | if (used < 0) | |
609 | goto free; | |
610 | if (last_rsgl) | |
611 | af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl); | |
612 | ||
613 | last_rsgl = rsgl; | |
614 | len += used; | |
615 | skcipher_pull_sgl(sk, used, 0); | |
616 | iov_iter_advance(&msg->msg_iter, used); | |
617 | } | |
618 | ||
033f46b3 | 619 | if (mark) |
620 | sg_mark_end(sreq->tsg + txbufs - 1); | |
621 | ||
0d96e4ba | 622 | skcipher_request_set_crypt(req, sreq->tsg, sreq->first_sgl.sgl.sg, |
3e597bb7 | 623 | len, iv); |
0d96e4ba HX |
624 | err = ctx->enc ? crypto_skcipher_encrypt(req) : |
625 | crypto_skcipher_decrypt(req); | |
a596999b TS |
626 | if (err == -EINPROGRESS) { |
627 | atomic_inc(&ctx->inflight); | |
628 | err = -EIOCBQUEUED; | |
3e597bb7 | 629 | sreq = NULL; |
a596999b TS |
630 | goto unlock; |
631 | } | |
632 | free: | |
633 | skcipher_free_async_sgls(sreq); | |
a596999b TS |
634 | unlock: |
635 | skcipher_wmem_wakeup(sk); | |
636 | release_sock(sk); | |
3e597bb7 HX |
637 | kzfree(sreq); |
638 | out: | |
a596999b TS |
639 | return err; |
640 | } | |
641 | ||
642 | static int skcipher_recvmsg_sync(struct socket *sock, struct msghdr *msg, | |
643 | int flags) | |
8ff59090 HX |
644 | { |
645 | struct sock *sk = sock->sk; | |
646 | struct alg_sock *ask = alg_sk(sk); | |
66bca97f HX |
647 | struct sock *psk = ask->parent; |
648 | struct alg_sock *pask = alg_sk(psk); | |
8ff59090 | 649 | struct skcipher_ctx *ctx = ask->private; |
66bca97f HX |
650 | struct skcipher_tfm *skc = pask->private; |
651 | struct crypto_skcipher *tfm = skc->skcipher; | |
652 | unsigned bs = crypto_skcipher_blocksize(tfm); | |
8ff59090 HX |
653 | struct skcipher_sg_list *sgl; |
654 | struct scatterlist *sg; | |
8ff59090 HX |
655 | int err = -EAGAIN; |
656 | int used; | |
657 | long copied = 0; | |
658 | ||
659 | lock_sock(sk); | |
01e97e65 | 660 | while (msg_data_left(msg)) { |
9399f0c5 | 661 | if (!ctx->used) { |
1d10eb2f AV |
662 | err = skcipher_wait_for_data(sk, flags); |
663 | if (err) | |
bc97e57e | 664 | goto unlock; |
1d10eb2f AV |
665 | } |
666 | ||
01e97e65 | 667 | used = min_t(unsigned long, ctx->used, msg_data_left(msg)); |
1d10eb2f AV |
668 | |
669 | used = af_alg_make_sg(&ctx->rsgl, &msg->msg_iter, used); | |
670 | err = used; | |
671 | if (err < 0) | |
672 | goto unlock; | |
bc97e57e | 673 | |
1d10eb2f AV |
674 | if (ctx->more || used < ctx->used) |
675 | used -= used % bs; | |
8ff59090 | 676 | |
1d10eb2f AV |
677 | err = -EINVAL; |
678 | if (!used) | |
679 | goto free; | |
8ff59090 | 680 | |
fec8beab HX |
681 | sgl = list_first_entry(&ctx->tsgl, |
682 | struct skcipher_sg_list, list); | |
683 | sg = sgl->sg; | |
684 | ||
685 | while (!sg->length) | |
686 | sg++; | |
687 | ||
0d96e4ba HX |
688 | skcipher_request_set_crypt(&ctx->req, sg, ctx->rsgl.sg, used, |
689 | ctx->iv); | |
8ff59090 | 690 | |
1d10eb2f | 691 | err = af_alg_wait_for_completion( |
8ff59090 | 692 | ctx->enc ? |
0d96e4ba HX |
693 | crypto_skcipher_encrypt(&ctx->req) : |
694 | crypto_skcipher_decrypt(&ctx->req), | |
8ff59090 HX |
695 | &ctx->completion); |
696 | ||
bc97e57e | 697 | free: |
1d10eb2f | 698 | af_alg_free_sg(&ctx->rsgl); |
8ff59090 | 699 | |
1d10eb2f AV |
700 | if (err) |
701 | goto unlock; | |
8ff59090 | 702 | |
1d10eb2f | 703 | copied += used; |
a596999b | 704 | skcipher_pull_sgl(sk, used, 1); |
1d10eb2f | 705 | iov_iter_advance(&msg->msg_iter, used); |
8ff59090 HX |
706 | } |
707 | ||
708 | err = 0; | |
709 | ||
710 | unlock: | |
711 | skcipher_wmem_wakeup(sk); | |
712 | release_sock(sk); | |
713 | ||
714 | return copied ?: err; | |
715 | } | |
716 | ||
a596999b TS |
717 | static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg, |
718 | size_t ignored, int flags) | |
719 | { | |
720 | return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ? | |
721 | skcipher_recvmsg_async(sock, msg, flags) : | |
722 | skcipher_recvmsg_sync(sock, msg, flags); | |
723 | } | |
8ff59090 HX |
724 | |
725 | static unsigned int skcipher_poll(struct file *file, struct socket *sock, | |
726 | poll_table *wait) | |
727 | { | |
728 | struct sock *sk = sock->sk; | |
729 | struct alg_sock *ask = alg_sk(sk); | |
730 | struct skcipher_ctx *ctx = ask->private; | |
731 | unsigned int mask; | |
732 | ||
733 | sock_poll_wait(file, sk_sleep(sk), wait); | |
734 | mask = 0; | |
735 | ||
736 | if (ctx->used) | |
737 | mask |= POLLIN | POLLRDNORM; | |
738 | ||
739 | if (skcipher_writable(sk)) | |
740 | mask |= POLLOUT | POLLWRNORM | POLLWRBAND; | |
741 | ||
742 | return mask; | |
743 | } | |
744 | ||
745 | static struct proto_ops algif_skcipher_ops = { | |
746 | .family = PF_ALG, | |
747 | ||
748 | .connect = sock_no_connect, | |
749 | .socketpair = sock_no_socketpair, | |
750 | .getname = sock_no_getname, | |
751 | .ioctl = sock_no_ioctl, | |
752 | .listen = sock_no_listen, | |
753 | .shutdown = sock_no_shutdown, | |
754 | .getsockopt = sock_no_getsockopt, | |
755 | .mmap = sock_no_mmap, | |
756 | .bind = sock_no_bind, | |
757 | .accept = sock_no_accept, | |
758 | .setsockopt = sock_no_setsockopt, | |
759 | ||
760 | .release = af_alg_release, | |
761 | .sendmsg = skcipher_sendmsg, | |
762 | .sendpage = skcipher_sendpage, | |
763 | .recvmsg = skcipher_recvmsg, | |
764 | .poll = skcipher_poll, | |
765 | }; | |
766 | ||
eeb2e2e7 HX |
767 | static int skcipher_check_key(struct socket *sock) |
768 | { | |
f84d1fcb | 769 | int err = 0; |
eeb2e2e7 HX |
770 | struct sock *psk; |
771 | struct alg_sock *pask; | |
772 | struct skcipher_tfm *tfm; | |
773 | struct sock *sk = sock->sk; | |
774 | struct alg_sock *ask = alg_sk(sk); | |
775 | ||
f84d1fcb | 776 | lock_sock(sk); |
eeb2e2e7 | 777 | if (ask->refcnt) |
f84d1fcb | 778 | goto unlock_child; |
eeb2e2e7 HX |
779 | |
780 | psk = ask->parent; | |
781 | pask = alg_sk(ask->parent); | |
782 | tfm = pask->private; | |
783 | ||
784 | err = -ENOKEY; | |
f84d1fcb | 785 | lock_sock_nested(psk, SINGLE_DEPTH_NESTING); |
eeb2e2e7 HX |
786 | if (!tfm->has_key) |
787 | goto unlock; | |
788 | ||
789 | if (!pask->refcnt++) | |
790 | sock_hold(psk); | |
791 | ||
792 | ask->refcnt = 1; | |
793 | sock_put(psk); | |
794 | ||
795 | err = 0; | |
796 | ||
797 | unlock: | |
798 | release_sock(psk); | |
f84d1fcb HX |
799 | unlock_child: |
800 | release_sock(sk); | |
eeb2e2e7 HX |
801 | |
802 | return err; | |
803 | } | |
804 | ||
805 | static int skcipher_sendmsg_nokey(struct socket *sock, struct msghdr *msg, | |
806 | size_t size) | |
807 | { | |
808 | int err; | |
809 | ||
810 | err = skcipher_check_key(sock); | |
811 | if (err) | |
812 | return err; | |
813 | ||
814 | return skcipher_sendmsg(sock, msg, size); | |
815 | } | |
816 | ||
817 | static ssize_t skcipher_sendpage_nokey(struct socket *sock, struct page *page, | |
818 | int offset, size_t size, int flags) | |
819 | { | |
820 | int err; | |
821 | ||
822 | err = skcipher_check_key(sock); | |
823 | if (err) | |
824 | return err; | |
825 | ||
826 | return skcipher_sendpage(sock, page, offset, size, flags); | |
827 | } | |
828 | ||
829 | static int skcipher_recvmsg_nokey(struct socket *sock, struct msghdr *msg, | |
830 | size_t ignored, int flags) | |
831 | { | |
832 | int err; | |
833 | ||
834 | err = skcipher_check_key(sock); | |
835 | if (err) | |
836 | return err; | |
837 | ||
838 | return skcipher_recvmsg(sock, msg, ignored, flags); | |
839 | } | |
840 | ||
841 | static struct proto_ops algif_skcipher_ops_nokey = { | |
842 | .family = PF_ALG, | |
843 | ||
844 | .connect = sock_no_connect, | |
845 | .socketpair = sock_no_socketpair, | |
846 | .getname = sock_no_getname, | |
847 | .ioctl = sock_no_ioctl, | |
848 | .listen = sock_no_listen, | |
849 | .shutdown = sock_no_shutdown, | |
850 | .getsockopt = sock_no_getsockopt, | |
851 | .mmap = sock_no_mmap, | |
852 | .bind = sock_no_bind, | |
853 | .accept = sock_no_accept, | |
854 | .setsockopt = sock_no_setsockopt, | |
855 | ||
856 | .release = af_alg_release, | |
857 | .sendmsg = skcipher_sendmsg_nokey, | |
858 | .sendpage = skcipher_sendpage_nokey, | |
859 | .recvmsg = skcipher_recvmsg_nokey, | |
860 | .poll = skcipher_poll, | |
861 | }; | |
862 | ||
8ff59090 HX |
863 | static void *skcipher_bind(const char *name, u32 type, u32 mask) |
864 | { | |
b238717c HX |
865 | struct skcipher_tfm *tfm; |
866 | struct crypto_skcipher *skcipher; | |
867 | ||
868 | tfm = kzalloc(sizeof(*tfm), GFP_KERNEL); | |
869 | if (!tfm) | |
870 | return ERR_PTR(-ENOMEM); | |
871 | ||
872 | skcipher = crypto_alloc_skcipher(name, type, mask); | |
873 | if (IS_ERR(skcipher)) { | |
874 | kfree(tfm); | |
875 | return ERR_CAST(skcipher); | |
876 | } | |
877 | ||
878 | tfm->skcipher = skcipher; | |
879 | ||
880 | return tfm; | |
8ff59090 HX |
881 | } |
882 | ||
883 | static void skcipher_release(void *private) | |
884 | { | |
b238717c HX |
885 | struct skcipher_tfm *tfm = private; |
886 | ||
887 | crypto_free_skcipher(tfm->skcipher); | |
888 | kfree(tfm); | |
8ff59090 HX |
889 | } |
890 | ||
891 | static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen) | |
892 | { | |
b238717c HX |
893 | struct skcipher_tfm *tfm = private; |
894 | int err; | |
895 | ||
896 | err = crypto_skcipher_setkey(tfm->skcipher, key, keylen); | |
897 | tfm->has_key = !err; | |
898 | ||
899 | return err; | |
8ff59090 HX |
900 | } |
901 | ||
a596999b TS |
902 | static void skcipher_wait(struct sock *sk) |
903 | { | |
904 | struct alg_sock *ask = alg_sk(sk); | |
905 | struct skcipher_ctx *ctx = ask->private; | |
906 | int ctr = 0; | |
907 | ||
908 | while (atomic_read(&ctx->inflight) && ctr++ < 100) | |
909 | msleep(100); | |
910 | } | |
911 | ||
40bfd37a | 912 | static void skcipher_sock_destruct(struct sock *sk) |
8ff59090 HX |
913 | { |
914 | struct alg_sock *ask = alg_sk(sk); | |
915 | struct skcipher_ctx *ctx = ask->private; | |
0d96e4ba | 916 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(&ctx->req); |
8ff59090 | 917 | |
a596999b TS |
918 | if (atomic_read(&ctx->inflight)) |
919 | skcipher_wait(sk); | |
920 | ||
8ff59090 | 921 | skcipher_free_sgl(sk); |
0d96e4ba | 922 | sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm)); |
8ff59090 | 923 | sock_kfree_s(sk, ctx, ctx->len); |
eeb2e2e7 HX |
924 | af_alg_release_parent(sk); |
925 | } | |
926 | ||
40bfd37a | 927 | static int skcipher_accept_parent_nokey(void *private, struct sock *sk) |
8ff59090 HX |
928 | { |
929 | struct skcipher_ctx *ctx; | |
930 | struct alg_sock *ask = alg_sk(sk); | |
b238717c HX |
931 | struct skcipher_tfm *tfm = private; |
932 | struct crypto_skcipher *skcipher = tfm->skcipher; | |
933 | unsigned int len = sizeof(*ctx) + crypto_skcipher_reqsize(skcipher); | |
934 | ||
8ff59090 HX |
935 | ctx = sock_kmalloc(sk, len, GFP_KERNEL); |
936 | if (!ctx) | |
937 | return -ENOMEM; | |
938 | ||
b238717c | 939 | ctx->iv = sock_kmalloc(sk, crypto_skcipher_ivsize(skcipher), |
8ff59090 HX |
940 | GFP_KERNEL); |
941 | if (!ctx->iv) { | |
942 | sock_kfree_s(sk, ctx, len); | |
943 | return -ENOMEM; | |
944 | } | |
945 | ||
b238717c | 946 | memset(ctx->iv, 0, crypto_skcipher_ivsize(skcipher)); |
8ff59090 HX |
947 | |
948 | INIT_LIST_HEAD(&ctx->tsgl); | |
949 | ctx->len = len; | |
950 | ctx->used = 0; | |
951 | ctx->more = 0; | |
952 | ctx->merge = 0; | |
953 | ctx->enc = 0; | |
a596999b | 954 | atomic_set(&ctx->inflight, 0); |
8ff59090 HX |
955 | af_alg_init_completion(&ctx->completion); |
956 | ||
957 | ask->private = ctx; | |
958 | ||
b238717c | 959 | skcipher_request_set_tfm(&ctx->req, skcipher); |
12426bd6 HX |
960 | skcipher_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_SLEEP | |
961 | CRYPTO_TFM_REQ_MAY_BACKLOG, | |
0d96e4ba | 962 | af_alg_complete, &ctx->completion); |
8ff59090 HX |
963 | |
964 | sk->sk_destruct = skcipher_sock_destruct; | |
965 | ||
966 | return 0; | |
967 | } | |
968 | ||
eeb2e2e7 HX |
969 | static int skcipher_accept_parent(void *private, struct sock *sk) |
970 | { | |
971 | struct skcipher_tfm *tfm = private; | |
972 | ||
a8e70a1a | 973 | if (!tfm->has_key && crypto_skcipher_has_setkey(tfm->skcipher)) |
eeb2e2e7 HX |
974 | return -ENOKEY; |
975 | ||
40bfd37a | 976 | return skcipher_accept_parent_nokey(private, sk); |
eeb2e2e7 HX |
977 | } |
978 | ||
8ff59090 HX |
979 | static const struct af_alg_type algif_type_skcipher = { |
980 | .bind = skcipher_bind, | |
981 | .release = skcipher_release, | |
982 | .setkey = skcipher_setkey, | |
983 | .accept = skcipher_accept_parent, | |
eeb2e2e7 | 984 | .accept_nokey = skcipher_accept_parent_nokey, |
8ff59090 | 985 | .ops = &algif_skcipher_ops, |
eeb2e2e7 | 986 | .ops_nokey = &algif_skcipher_ops_nokey, |
8ff59090 HX |
987 | .name = "skcipher", |
988 | .owner = THIS_MODULE | |
989 | }; | |
990 | ||
991 | static int __init algif_skcipher_init(void) | |
992 | { | |
993 | return af_alg_register_type(&algif_type_skcipher); | |
994 | } | |
995 | ||
996 | static void __exit algif_skcipher_exit(void) | |
997 | { | |
998 | int err = af_alg_unregister_type(&algif_type_skcipher); | |
999 | BUG_ON(err); | |
1000 | } | |
1001 | ||
1002 | module_init(algif_skcipher_init); | |
1003 | module_exit(algif_skcipher_exit); | |
1004 | MODULE_LICENSE("GPL"); |