]> git.proxmox.com Git - mirror_ubuntu-hirsute-kernel.git/blob - net/tls/tls_sw.c
projects / mirror_ubuntu-hirsute-kernel.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
tls: Fix recvmsg() to be able to peek across multiple records
[mirror_ubuntu-hirsute-kernel.git] / net / tls / tls_sw.c
1 /*
2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
4 * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
5 * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
6 * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
7 * Copyright (c) 2018, Covalent IO, Inc. http://covalent.io
8 *
9 * This software is available to you under a choice of one of two
10 * licenses. You may choose to be licensed under the terms of the GNU
11 * General Public License (GPL) Version 2, available from the file
12 * COPYING in the main directory of this source tree, or the
13 * OpenIB.org BSD license below:
14 *
15 * Redistribution and use in source and binary forms, with or
16 * without modification, are permitted provided that the following
17 * conditions are met:
18 *
19 * - Redistributions of source code must retain the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer.
22 *
23 * - Redistributions in binary form must reproduce the above
24 * copyright notice, this list of conditions and the following
25 * disclaimer in the documentation and/or other materials
26 * provided with the distribution.
27 *
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
29 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
30 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
31 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
32 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
33 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
34 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35 * SOFTWARE.
36 */
37
38 #include <linux/sched/signal.h>
39 #include <linux/module.h>
40 #include <crypto/aead.h>
41
42 #include <net/strparser.h>
43 #include <net/tls.h>
44
45 #define MAX_IV_SIZE TLS_CIPHER_AES_GCM_128_IV_SIZE
46
47 static int __skb_nsg(struct sk_buff *skb, int offset, int len,
48 unsigned int recursion_level)
49 {
50 int start = skb_headlen(skb);
51 int i, chunk = start - offset;
52 struct sk_buff *frag_iter;
53 int elt = 0;
54
55 if (unlikely(recursion_level >= 24))
56 return -EMSGSIZE;
57
58 if (chunk > 0) {
59 if (chunk > len)
60 chunk = len;
61 elt++;
62 len -= chunk;
63 if (len == 0)
64 return elt;
65 offset += chunk;
66 }
67
68 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
69 int end;
70
71 WARN_ON(start > offset + len);
72
73 end = start + skb_frag_size(&skb_shinfo(skb)->frags[i]);
74 chunk = end - offset;
75 if (chunk > 0) {
76 if (chunk > len)
77 chunk = len;
78 elt++;
79 len -= chunk;
80 if (len == 0)
81 return elt;
82 offset += chunk;
83 }
84 start = end;
85 }
86
87 if (unlikely(skb_has_frag_list(skb))) {
88 skb_walk_frags(skb, frag_iter) {
89 int end, ret;
90
91 WARN_ON(start > offset + len);
92
93 end = start + frag_iter->len;
94 chunk = end - offset;
95 if (chunk > 0) {
96 if (chunk > len)
97 chunk = len;
98 ret = __skb_nsg(frag_iter, offset - start, chunk,
99 recursion_level + 1);
100 if (unlikely(ret < 0))
101 return ret;
102 elt += ret;
103 len -= chunk;
104 if (len == 0)
105 return elt;
106 offset += chunk;
107 }
108 start = end;
109 }
110 }
111 BUG_ON(len);
112 return elt;
113 }
114
115 /* Return the number of scatterlist elements required to completely map the
116 * skb, or -EMSGSIZE if the recursion depth is exceeded.
117 */
118 static int skb_nsg(struct sk_buff *skb, int offset, int len)
119 {
120 return __skb_nsg(skb, offset, len, 0);
121 }
122
123 static void tls_decrypt_done(struct crypto_async_request *req, int err)
124 {
125 struct aead_request *aead_req = (struct aead_request *)req;
126 struct scatterlist *sgout = aead_req->dst;
127 struct scatterlist *sgin = aead_req->src;
128 struct tls_sw_context_rx *ctx;
129 struct tls_context *tls_ctx;
130 struct scatterlist *sg;
131 struct sk_buff *skb;
132 unsigned int pages;
133 int pending;
134
135 skb = (struct sk_buff *)req->data;
136 tls_ctx = tls_get_ctx(skb->sk);
137 ctx = tls_sw_ctx_rx(tls_ctx);
138
139 /* Propagate if there was an err */
140 if (err) {
141 ctx->async_wait.err = err;
142 tls_err_abort(skb->sk, err);
143 } else {
144 struct strp_msg *rxm = strp_msg(skb);
145
146 rxm->offset += tls_ctx->rx.prepend_size;
147 rxm->full_len -= tls_ctx->rx.overhead_size;
148 }
149
150 /* After using skb->sk to propagate sk through crypto async callback
151 * we need to NULL it again.
152 */
153 skb->sk = NULL;
154
155
156 /* Free the destination pages if skb was not decrypted inplace */
157 if (sgout != sgin) {
158 /* Skip the first S/G entry as it points to AAD */
159 for_each_sg(sg_next(sgout), sg, UINT_MAX, pages) {
160 if (!sg)
161 break;
162 put_page(sg_page(sg));
163 }
164 }
165
166 kfree(aead_req);
167
168 pending = atomic_dec_return(&ctx->decrypt_pending);
169
170 if (!pending && READ_ONCE(ctx->async_notify))
171 complete(&ctx->async_wait.completion);
172 }
173
174 static int tls_do_decryption(struct sock *sk,
175 struct sk_buff *skb,
176 struct scatterlist *sgin,
177 struct scatterlist *sgout,
178 char *iv_recv,
179 size_t data_len,
180 struct aead_request *aead_req,
181 bool async)
182 {
183 struct tls_context *tls_ctx = tls_get_ctx(sk);
184 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
185 int ret;
186
187 aead_request_set_tfm(aead_req, ctx->aead_recv);
188 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
189 aead_request_set_crypt(aead_req, sgin, sgout,
190 data_len + tls_ctx->rx.tag_size,
191 (u8 *)iv_recv);
192
193 if (async) {
194 /* Using skb->sk to push sk through to crypto async callback
195 * handler. This allows propagating errors up to the socket
196 * if needed. It _must_ be cleared in the async handler
197 * before kfree_skb is called. We _know_ skb->sk is NULL
198 * because it is a clone from strparser.
199 */
200 skb->sk = sk;
201 aead_request_set_callback(aead_req,
202 CRYPTO_TFM_REQ_MAY_BACKLOG,
203 tls_decrypt_done, skb);
204 atomic_inc(&ctx->decrypt_pending);
205 } else {
206 aead_request_set_callback(aead_req,
207 CRYPTO_TFM_REQ_MAY_BACKLOG,
208 crypto_req_done, &ctx->async_wait);
209 }
210
211 ret = crypto_aead_decrypt(aead_req);
212 if (ret == -EINPROGRESS) {
213 if (async)
214 return ret;
215
216 ret = crypto_wait_req(ret, &ctx->async_wait);
217 }
218
219 if (async)
220 atomic_dec(&ctx->decrypt_pending);
221
222 return ret;
223 }
224
225 static void tls_trim_both_msgs(struct sock *sk, int target_size)
226 {
227 struct tls_context *tls_ctx = tls_get_ctx(sk);
228 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
229 struct tls_rec *rec = ctx->open_rec;
230
231 sk_msg_trim(sk, &rec->msg_plaintext, target_size);
232 if (target_size > 0)
233 target_size += tls_ctx->tx.overhead_size;
234 sk_msg_trim(sk, &rec->msg_encrypted, target_size);
235 }
236
237 static int tls_alloc_encrypted_msg(struct sock *sk, int len)
238 {
239 struct tls_context *tls_ctx = tls_get_ctx(sk);
240 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
241 struct tls_rec *rec = ctx->open_rec;
242 struct sk_msg *msg_en = &rec->msg_encrypted;
243
244 return sk_msg_alloc(sk, msg_en, len, 0);
245 }
246
247 static int tls_clone_plaintext_msg(struct sock *sk, int required)
248 {
249 struct tls_context *tls_ctx = tls_get_ctx(sk);
250 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
251 struct tls_rec *rec = ctx->open_rec;
252 struct sk_msg *msg_pl = &rec->msg_plaintext;
253 struct sk_msg *msg_en = &rec->msg_encrypted;
254 int skip, len;
255
256 /* We add page references worth len bytes from encrypted sg
257 * at the end of plaintext sg. It is guaranteed that msg_en
258 * has enough required room (ensured by caller).
259 */
260 len = required - msg_pl->sg.size;
261
262 /* Skip initial bytes in msg_en's data to be able to use
263 * same offset of both plain and encrypted data.
264 */
265 skip = tls_ctx->tx.prepend_size + msg_pl->sg.size;
266
267 return sk_msg_clone(sk, msg_pl, msg_en, skip, len);
268 }
269
270 static struct tls_rec *tls_get_rec(struct sock *sk)
271 {
272 struct tls_context *tls_ctx = tls_get_ctx(sk);
273 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
274 struct sk_msg *msg_pl, *msg_en;
275 struct tls_rec *rec;
276 int mem_size;
277
278 mem_size = sizeof(struct tls_rec) + crypto_aead_reqsize(ctx->aead_send);
279
280 rec = kzalloc(mem_size, sk->sk_allocation);
281 if (!rec)
282 return NULL;
283
284 msg_pl = &rec->msg_plaintext;
285 msg_en = &rec->msg_encrypted;
286
287 sk_msg_init(msg_pl);
288 sk_msg_init(msg_en);
289
290 sg_init_table(rec->sg_aead_in, 2);
291 sg_set_buf(&rec->sg_aead_in[0], rec->aad_space,
292 sizeof(rec->aad_space));
293 sg_unmark_end(&rec->sg_aead_in[1]);
294
295 sg_init_table(rec->sg_aead_out, 2);
296 sg_set_buf(&rec->sg_aead_out[0], rec->aad_space,
297 sizeof(rec->aad_space));
298 sg_unmark_end(&rec->sg_aead_out[1]);
299
300 return rec;
301 }
302
303 static void tls_free_rec(struct sock *sk, struct tls_rec *rec)
304 {
305 sk_msg_free(sk, &rec->msg_encrypted);
306 sk_msg_free(sk, &rec->msg_plaintext);
307 kfree(rec);
308 }
309
310 static void tls_free_open_rec(struct sock *sk)
311 {
312 struct tls_context *tls_ctx = tls_get_ctx(sk);
313 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
314 struct tls_rec *rec = ctx->open_rec;
315
316 if (rec) {
317 tls_free_rec(sk, rec);
318 ctx->open_rec = NULL;
319 }
320 }
321
322 int tls_tx_records(struct sock *sk, int flags)
323 {
324 struct tls_context *tls_ctx = tls_get_ctx(sk);
325 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
326 struct tls_rec *rec, *tmp;
327 struct sk_msg *msg_en;
328 int tx_flags, rc = 0;
329
330 if (tls_is_partially_sent_record(tls_ctx)) {
331 rec = list_first_entry(&ctx->tx_list,
332 struct tls_rec, list);
333
334 if (flags == -1)
335 tx_flags = rec->tx_flags;
336 else
337 tx_flags = flags;
338
339 rc = tls_push_partial_record(sk, tls_ctx, tx_flags);
340 if (rc)
341 goto tx_err;
342
343 /* Full record has been transmitted.
344 * Remove the head of tx_list
345 */
346 list_del(&rec->list);
347 sk_msg_free(sk, &rec->msg_plaintext);
348 kfree(rec);
349 }
350
351 /* Tx all ready records */
352 list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) {
353 if (READ_ONCE(rec->tx_ready)) {
354 if (flags == -1)
355 tx_flags = rec->tx_flags;
356 else
357 tx_flags = flags;
358
359 msg_en = &rec->msg_encrypted;
360 rc = tls_push_sg(sk, tls_ctx,
361 &msg_en->sg.data[msg_en->sg.curr],
362 0, tx_flags);
363 if (rc)
364 goto tx_err;
365
366 list_del(&rec->list);
367 sk_msg_free(sk, &rec->msg_plaintext);
368 kfree(rec);
369 } else {
370 break;
371 }
372 }
373
374 tx_err:
375 if (rc < 0 && rc != -EAGAIN)
376 tls_err_abort(sk, EBADMSG);
377
378 return rc;
379 }
380
381 static void tls_encrypt_done(struct crypto_async_request *req, int err)
382 {
383 struct aead_request *aead_req = (struct aead_request *)req;
384 struct sock *sk = req->data;
385 struct tls_context *tls_ctx = tls_get_ctx(sk);
386 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
387 struct scatterlist *sge;
388 struct sk_msg *msg_en;
389 struct tls_rec *rec;
390 bool ready = false;
391 int pending;
392
393 rec = container_of(aead_req, struct tls_rec, aead_req);
394 msg_en = &rec->msg_encrypted;
395
396 sge = sk_msg_elem(msg_en, msg_en->sg.curr);
397 sge->offset -= tls_ctx->tx.prepend_size;
398 sge->length += tls_ctx->tx.prepend_size;
399
400 /* Check if error is previously set on socket */
401 if (err || sk->sk_err) {
402 rec = NULL;
403
404 /* If err is already set on socket, return the same code */
405 if (sk->sk_err) {
406 ctx->async_wait.err = sk->sk_err;
407 } else {
408 ctx->async_wait.err = err;
409 tls_err_abort(sk, err);
410 }
411 }
412
413 if (rec) {
414 struct tls_rec *first_rec;
415
416 /* Mark the record as ready for transmission */
417 smp_store_mb(rec->tx_ready, true);
418
419 /* If received record is at head of tx_list, schedule tx */
420 first_rec = list_first_entry(&ctx->tx_list,
421 struct tls_rec, list);
422 if (rec == first_rec)
423 ready = true;
424 }
425
426 pending = atomic_dec_return(&ctx->encrypt_pending);
427
428 if (!pending && READ_ONCE(ctx->async_notify))
429 complete(&ctx->async_wait.completion);
430
431 if (!ready)
432 return;
433
434 /* Schedule the transmission */
435 if (!test_and_set_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask))
436 schedule_delayed_work(&ctx->tx_work.work, 1);
437 }
438
439 static int tls_do_encryption(struct sock *sk,
440 struct tls_context *tls_ctx,
441 struct tls_sw_context_tx *ctx,
442 struct aead_request *aead_req,
443 size_t data_len, u32 start)
444 {
445 struct tls_rec *rec = ctx->open_rec;
446 struct sk_msg *msg_en = &rec->msg_encrypted;
447 struct scatterlist *sge = sk_msg_elem(msg_en, start);
448 int rc;
449
450 sge->offset += tls_ctx->tx.prepend_size;
451 sge->length -= tls_ctx->tx.prepend_size;
452
453 msg_en->sg.curr = start;
454
455 aead_request_set_tfm(aead_req, ctx->aead_send);
456 aead_request_set_ad(aead_req, TLS_AAD_SPACE_SIZE);
457 aead_request_set_crypt(aead_req, rec->sg_aead_in,
458 rec->sg_aead_out,
459 data_len, tls_ctx->tx.iv);
460
461 aead_request_set_callback(aead_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
462 tls_encrypt_done, sk);
463
464 /* Add the record in tx_list */
465 list_add_tail((struct list_head *)&rec->list, &ctx->tx_list);
466 atomic_inc(&ctx->encrypt_pending);
467
468 rc = crypto_aead_encrypt(aead_req);
469 if (!rc || rc != -EINPROGRESS) {
470 atomic_dec(&ctx->encrypt_pending);
471 sge->offset -= tls_ctx->tx.prepend_size;
472 sge->length += tls_ctx->tx.prepend_size;
473 }
474
475 if (!rc) {
476 WRITE_ONCE(rec->tx_ready, true);
477 } else if (rc != -EINPROGRESS) {
478 list_del(&rec->list);
479 return rc;
480 }
481
482 /* Unhook the record from context if encryption is not failure */
483 ctx->open_rec = NULL;
484 tls_advance_record_sn(sk, &tls_ctx->tx);
485 return rc;
486 }
487
488 static int tls_split_open_record(struct sock *sk, struct tls_rec *from,
489 struct tls_rec **to, struct sk_msg *msg_opl,
490 struct sk_msg *msg_oen, u32 split_point,
491 u32 tx_overhead_size, u32 *orig_end)
492 {
493 u32 i, j, bytes = 0, apply = msg_opl->apply_bytes;
494 struct scatterlist *sge, *osge, *nsge;
495 u32 orig_size = msg_opl->sg.size;
496 struct scatterlist tmp = { };
497 struct sk_msg *msg_npl;
498 struct tls_rec *new;
499 int ret;
500
501 new = tls_get_rec(sk);
502 if (!new)
503 return -ENOMEM;
504 ret = sk_msg_alloc(sk, &new->msg_encrypted, msg_opl->sg.size +
505 tx_overhead_size, 0);
506 if (ret < 0) {
507 tls_free_rec(sk, new);
508 return ret;
509 }
510
511 *orig_end = msg_opl->sg.end;
512 i = msg_opl->sg.start;
513 sge = sk_msg_elem(msg_opl, i);
514 while (apply && sge->length) {
515 if (sge->length > apply) {
516 u32 len = sge->length - apply;
517
518 get_page(sg_page(sge));
519 sg_set_page(&tmp, sg_page(sge), len,
520 sge->offset + apply);
521 sge->length = apply;
522 bytes += apply;
523 apply = 0;
524 } else {
525 apply -= sge->length;
526 bytes += sge->length;
527 }
528
529 sk_msg_iter_var_next(i);
530 if (i == msg_opl->sg.end)
531 break;
532 sge = sk_msg_elem(msg_opl, i);
533 }
534
535 msg_opl->sg.end = i;
536 msg_opl->sg.curr = i;
537 msg_opl->sg.copybreak = 0;
538 msg_opl->apply_bytes = 0;
539 msg_opl->sg.size = bytes;
540
541 msg_npl = &new->msg_plaintext;
542 msg_npl->apply_bytes = apply;
543 msg_npl->sg.size = orig_size - bytes;
544
545 j = msg_npl->sg.start;
546 nsge = sk_msg_elem(msg_npl, j);
547 if (tmp.length) {
548 memcpy(nsge, &tmp, sizeof(*nsge));
549 sk_msg_iter_var_next(j);
550 nsge = sk_msg_elem(msg_npl, j);
551 }
552
553 osge = sk_msg_elem(msg_opl, i);
554 while (osge->length) {
555 memcpy(nsge, osge, sizeof(*nsge));
556 sg_unmark_end(nsge);
557 sk_msg_iter_var_next(i);
558 sk_msg_iter_var_next(j);
559 if (i == *orig_end)
560 break;
561 osge = sk_msg_elem(msg_opl, i);
562 nsge = sk_msg_elem(msg_npl, j);
563 }
564
565 msg_npl->sg.end = j;
566 msg_npl->sg.curr = j;
567 msg_npl->sg.copybreak = 0;
568
569 *to = new;
570 return 0;
571 }
572
573 static void tls_merge_open_record(struct sock *sk, struct tls_rec *to,
574 struct tls_rec *from, u32 orig_end)
575 {
576 struct sk_msg *msg_npl = &from->msg_plaintext;
577 struct sk_msg *msg_opl = &to->msg_plaintext;
578 struct scatterlist *osge, *nsge;
579 u32 i, j;
580
581 i = msg_opl->sg.end;
582 sk_msg_iter_var_prev(i);
583 j = msg_npl->sg.start;
584
585 osge = sk_msg_elem(msg_opl, i);
586 nsge = sk_msg_elem(msg_npl, j);
587
588 if (sg_page(osge) == sg_page(nsge) &&
589 osge->offset + osge->length == nsge->offset) {
590 osge->length += nsge->length;
591 put_page(sg_page(nsge));
592 }
593
594 msg_opl->sg.end = orig_end;
595 msg_opl->sg.curr = orig_end;
596 msg_opl->sg.copybreak = 0;
597 msg_opl->apply_bytes = msg_opl->sg.size + msg_npl->sg.size;
598 msg_opl->sg.size += msg_npl->sg.size;
599
600 sk_msg_free(sk, &to->msg_encrypted);
601 sk_msg_xfer_full(&to->msg_encrypted, &from->msg_encrypted);
602
603 kfree(from);
604 }
605
606 static int tls_push_record(struct sock *sk, int flags,
607 unsigned char record_type)
608 {
609 struct tls_context *tls_ctx = tls_get_ctx(sk);
610 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
611 struct tls_rec *rec = ctx->open_rec, *tmp = NULL;
612 u32 i, split_point, uninitialized_var(orig_end);
613 struct sk_msg *msg_pl, *msg_en;
614 struct aead_request *req;
615 bool split;
616 int rc;
617
618 if (!rec)
619 return 0;
620
621 msg_pl = &rec->msg_plaintext;
622 msg_en = &rec->msg_encrypted;
623
624 split_point = msg_pl->apply_bytes;
625 split = split_point && split_point < msg_pl->sg.size;
626 if (split) {
627 rc = tls_split_open_record(sk, rec, &tmp, msg_pl, msg_en,
628 split_point, tls_ctx->tx.overhead_size,
629 &orig_end);
630 if (rc < 0)
631 return rc;
632 sk_msg_trim(sk, msg_en, msg_pl->sg.size +
633 tls_ctx->tx.overhead_size);
634 }
635
636 rec->tx_flags = flags;
637 req = &rec->aead_req;
638
639 i = msg_pl->sg.end;
640 sk_msg_iter_var_prev(i);
641 sg_mark_end(sk_msg_elem(msg_pl, i));
642
643 i = msg_pl->sg.start;
644 sg_chain(rec->sg_aead_in, 2, rec->inplace_crypto ?
645 &msg_en->sg.data[i] : &msg_pl->sg.data[i]);
646
647 i = msg_en->sg.end;
648 sk_msg_iter_var_prev(i);
649 sg_mark_end(sk_msg_elem(msg_en, i));
650
651 i = msg_en->sg.start;
652 sg_chain(rec->sg_aead_out, 2, &msg_en->sg.data[i]);
653
654 tls_make_aad(rec->aad_space, msg_pl->sg.size,
655 tls_ctx->tx.rec_seq, tls_ctx->tx.rec_seq_size,
656 record_type);
657
658 tls_fill_prepend(tls_ctx,
659 page_address(sg_page(&msg_en->sg.data[i])) +
660 msg_en->sg.data[i].offset, msg_pl->sg.size,
661 record_type);
662
663 tls_ctx->pending_open_record_frags = false;
664
665 rc = tls_do_encryption(sk, tls_ctx, ctx, req, msg_pl->sg.size, i);
666 if (rc < 0) {
667 if (rc != -EINPROGRESS) {
668 tls_err_abort(sk, EBADMSG);
669 if (split) {
670 tls_ctx->pending_open_record_frags = true;
671 tls_merge_open_record(sk, rec, tmp, orig_end);
672 }
673 }
674 return rc;
675 } else if (split) {
676 msg_pl = &tmp->msg_plaintext;
677 msg_en = &tmp->msg_encrypted;
678 sk_msg_trim(sk, msg_en, msg_pl->sg.size +
679 tls_ctx->tx.overhead_size);
680 tls_ctx->pending_open_record_frags = true;
681 ctx->open_rec = tmp;
682 }
683
684 return tls_tx_records(sk, flags);
685 }
686
687 static int bpf_exec_tx_verdict(struct sk_msg *msg, struct sock *sk,
688 bool full_record, u8 record_type,
689 size_t *copied, int flags)
690 {
691 struct tls_context *tls_ctx = tls_get_ctx(sk);
692 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
693 struct sk_msg msg_redir = { };
694 struct sk_psock *psock;
695 struct sock *sk_redir;
696 struct tls_rec *rec;
697 bool enospc, policy;
698 int err = 0, send;
699 u32 delta = 0;
700
701 policy = !(flags & MSG_SENDPAGE_NOPOLICY);
702 psock = sk_psock_get(sk);
703 if (!psock || !policy)
704 return tls_push_record(sk, flags, record_type);
705 more_data:
706 enospc = sk_msg_full(msg);
707 if (psock->eval == __SK_NONE) {
708 delta = msg->sg.size;
709 psock->eval = sk_psock_msg_verdict(sk, psock, msg);
710 if (delta < msg->sg.size)
711 delta -= msg->sg.size;
712 else
713 delta = 0;
714 }
715 if (msg->cork_bytes && msg->cork_bytes > msg->sg.size &&
716 !enospc && !full_record) {
717 err = -ENOSPC;
718 goto out_err;
719 }
720 msg->cork_bytes = 0;
721 send = msg->sg.size;
722 if (msg->apply_bytes && msg->apply_bytes < send)
723 send = msg->apply_bytes;
724
725 switch (psock->eval) {
726 case __SK_PASS:
727 err = tls_push_record(sk, flags, record_type);
728 if (err < 0) {
729 *copied -= sk_msg_free(sk, msg);
730 tls_free_open_rec(sk);
731 goto out_err;
732 }
733 break;
734 case __SK_REDIRECT:
735 sk_redir = psock->sk_redir;
736 memcpy(&msg_redir, msg, sizeof(*msg));
737 if (msg->apply_bytes < send)
738 msg->apply_bytes = 0;
739 else
740 msg->apply_bytes -= send;
741 sk_msg_return_zero(sk, msg, send);
742 msg->sg.size -= send;
743 release_sock(sk);
744 err = tcp_bpf_sendmsg_redir(sk_redir, &msg_redir, send, flags);
745 lock_sock(sk);
746 if (err < 0) {
747 *copied -= sk_msg_free_nocharge(sk, &msg_redir);
748 msg->sg.size = 0;
749 }
750 if (msg->sg.size == 0)
751 tls_free_open_rec(sk);
752 break;
753 case __SK_DROP:
754 default:
755 sk_msg_free_partial(sk, msg, send);
756 if (msg->apply_bytes < send)
757 msg->apply_bytes = 0;
758 else
759 msg->apply_bytes -= send;
760 if (msg->sg.size == 0)
761 tls_free_open_rec(sk);
762 *copied -= (send + delta);
763 err = -EACCES;
764 }
765
766 if (likely(!err)) {
767 bool reset_eval = !ctx->open_rec;
768
769 rec = ctx->open_rec;
770 if (rec) {
771 msg = &rec->msg_plaintext;
772 if (!msg->apply_bytes)
773 reset_eval = true;
774 }
775 if (reset_eval) {
776 psock->eval = __SK_NONE;
777 if (psock->sk_redir) {
778 sock_put(psock->sk_redir);
779 psock->sk_redir = NULL;
780 }
781 }
782 if (rec)
783 goto more_data;
784 }
785 out_err:
786 sk_psock_put(sk, psock);
787 return err;
788 }
789
790 static int tls_sw_push_pending_record(struct sock *sk, int flags)
791 {
792 struct tls_context *tls_ctx = tls_get_ctx(sk);
793 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
794 struct tls_rec *rec = ctx->open_rec;
795 struct sk_msg *msg_pl;
796 size_t copied;
797
798 if (!rec)
799 return 0;
800
801 msg_pl = &rec->msg_plaintext;
802 copied = msg_pl->sg.size;
803 if (!copied)
804 return 0;
805
806 return bpf_exec_tx_verdict(msg_pl, sk, true, TLS_RECORD_TYPE_DATA,
807 &copied, flags);
808 }
809
810 int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
811 {
812 long timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
813 struct tls_context *tls_ctx = tls_get_ctx(sk);
814 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
815 struct crypto_tfm *tfm = crypto_aead_tfm(ctx->aead_send);
816 bool async_capable = tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC;
817 unsigned char record_type = TLS_RECORD_TYPE_DATA;
818 bool is_kvec = iov_iter_is_kvec(&msg->msg_iter);
819 bool eor = !(msg->msg_flags & MSG_MORE);
820 size_t try_to_copy, copied = 0;
821 struct sk_msg *msg_pl, *msg_en;
822 struct tls_rec *rec;
823 int required_size;
824 int num_async = 0;
825 bool full_record;
826 int record_room;
827 int num_zc = 0;
828 int orig_size;
829 int ret = 0;
830
831 if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
832 return -ENOTSUPP;
833
834 lock_sock(sk);
835
836 /* Wait till there is any pending write on socket */
837 if (unlikely(sk->sk_write_pending)) {
838 ret = wait_on_pending_writer(sk, &timeo);
839 if (unlikely(ret))
840 goto send_end;
841 }
842
843 if (unlikely(msg->msg_controllen)) {
844 ret = tls_proccess_cmsg(sk, msg, &record_type);
845 if (ret) {
846 if (ret == -EINPROGRESS)
847 num_async++;
848 else if (ret != -EAGAIN)
849 goto send_end;
850 }
851 }
852
853 while (msg_data_left(msg)) {
854 if (sk->sk_err) {
855 ret = -sk->sk_err;
856 goto send_end;
857 }
858
859 if (ctx->open_rec)
860 rec = ctx->open_rec;
861 else
862 rec = ctx->open_rec = tls_get_rec(sk);
863 if (!rec) {
864 ret = -ENOMEM;
865 goto send_end;
866 }
867
868 msg_pl = &rec->msg_plaintext;
869 msg_en = &rec->msg_encrypted;
870
871 orig_size = msg_pl->sg.size;
872 full_record = false;
873 try_to_copy = msg_data_left(msg);
874 record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size;
875 if (try_to_copy >= record_room) {
876 try_to_copy = record_room;
877 full_record = true;
878 }
879
880 required_size = msg_pl->sg.size + try_to_copy +
881 tls_ctx->tx.overhead_size;
882
883 if (!sk_stream_memory_free(sk))
884 goto wait_for_sndbuf;
885
886 alloc_encrypted:
887 ret = tls_alloc_encrypted_msg(sk, required_size);
888 if (ret) {
889 if (ret != -ENOSPC)
890 goto wait_for_memory;
891
892 /* Adjust try_to_copy according to the amount that was
893 * actually allocated. The difference is due
894 * to max sg elements limit
895 */
896 try_to_copy -= required_size - msg_en->sg.size;
897 full_record = true;
898 }
899
900 if (!is_kvec && (full_record || eor) && !async_capable) {
901 u32 first = msg_pl->sg.end;
902
903 ret = sk_msg_zerocopy_from_iter(sk, &msg->msg_iter,
904 msg_pl, try_to_copy);
905 if (ret)
906 goto fallback_to_reg_send;
907
908 rec->inplace_crypto = 0;
909
910 num_zc++;
911 copied += try_to_copy;
912
913 sk_msg_sg_copy_set(msg_pl, first);
914 ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
915 record_type, &copied,
916 msg->msg_flags);
917 if (ret) {
918 if (ret == -EINPROGRESS)
919 num_async++;
920 else if (ret == -ENOMEM)
921 goto wait_for_memory;
922 else if (ret == -ENOSPC)
923 goto rollback_iter;
924 else if (ret != -EAGAIN)
925 goto send_end;
926 }
927 continue;
928 rollback_iter:
929 copied -= try_to_copy;
930 sk_msg_sg_copy_clear(msg_pl, first);
931 iov_iter_revert(&msg->msg_iter,
932 msg_pl->sg.size - orig_size);
933 fallback_to_reg_send:
934 sk_msg_trim(sk, msg_pl, orig_size);
935 }
936
937 required_size = msg_pl->sg.size + try_to_copy;
938
939 ret = tls_clone_plaintext_msg(sk, required_size);
940 if (ret) {
941 if (ret != -ENOSPC)
942 goto send_end;
943
944 /* Adjust try_to_copy according to the amount that was
945 * actually allocated. The difference is due
946 * to max sg elements limit
947 */
948 try_to_copy -= required_size - msg_pl->sg.size;
949 full_record = true;
950 sk_msg_trim(sk, msg_en, msg_pl->sg.size +
951 tls_ctx->tx.overhead_size);
952 }
953
954 if (try_to_copy) {
955 ret = sk_msg_memcopy_from_iter(sk, &msg->msg_iter,
956 msg_pl, try_to_copy);
957 if (ret < 0)
958 goto trim_sgl;
959 }
960
961 /* Open records defined only if successfully copied, otherwise
962 * we would trim the sg but not reset the open record frags.
963 */
964 tls_ctx->pending_open_record_frags = true;
965 copied += try_to_copy;
966 if (full_record || eor) {
967 ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
968 record_type, &copied,
969 msg->msg_flags);
970 if (ret) {
971 if (ret == -EINPROGRESS)
972 num_async++;
973 else if (ret == -ENOMEM)
974 goto wait_for_memory;
975 else if (ret != -EAGAIN) {
976 if (ret == -ENOSPC)
977 ret = 0;
978 goto send_end;
979 }
980 }
981 }
982
983 continue;
984
985 wait_for_sndbuf:
986 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
987 wait_for_memory:
988 ret = sk_stream_wait_memory(sk, &timeo);
989 if (ret) {
990 trim_sgl:
991 tls_trim_both_msgs(sk, orig_size);
992 goto send_end;
993 }
994
995 if (msg_en->sg.size < required_size)
996 goto alloc_encrypted;
997 }
998
999 if (!num_async) {
1000 goto send_end;
1001 } else if (num_zc) {
1002 /* Wait for pending encryptions to get completed */
1003 smp_store_mb(ctx->async_notify, true);
1004
1005 if (atomic_read(&ctx->encrypt_pending))
1006 crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
1007 else
1008 reinit_completion(&ctx->async_wait.completion);
1009
1010 WRITE_ONCE(ctx->async_notify, false);
1011
1012 if (ctx->async_wait.err) {
1013 ret = ctx->async_wait.err;
1014 copied = 0;
1015 }
1016 }
1017
1018 /* Transmit if any encryptions have completed */
1019 if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
1020 cancel_delayed_work(&ctx->tx_work.work);
1021 tls_tx_records(sk, msg->msg_flags);
1022 }
1023
1024 send_end:
1025 ret = sk_stream_error(sk, msg->msg_flags, ret);
1026
1027 release_sock(sk);
1028 return copied ? copied : ret;
1029 }
1030
1031 static int tls_sw_do_sendpage(struct sock *sk, struct page *page,
1032 int offset, size_t size, int flags)
1033 {
1034 long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1035 struct tls_context *tls_ctx = tls_get_ctx(sk);
1036 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
1037 unsigned char record_type = TLS_RECORD_TYPE_DATA;
1038 struct sk_msg *msg_pl;
1039 struct tls_rec *rec;
1040 int num_async = 0;
1041 size_t copied = 0;
1042 bool full_record;
1043 int record_room;
1044 int ret = 0;
1045 bool eor;
1046
1047 eor = !(flags & (MSG_MORE | MSG_SENDPAGE_NOTLAST));
1048 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1049
1050 /* Wait till there is any pending write on socket */
1051 if (unlikely(sk->sk_write_pending)) {
1052 ret = wait_on_pending_writer(sk, &timeo);
1053 if (unlikely(ret))
1054 goto sendpage_end;
1055 }
1056
1057 /* Call the sk_stream functions to manage the sndbuf mem. */
1058 while (size > 0) {
1059 size_t copy, required_size;
1060
1061 if (sk->sk_err) {
1062 ret = -sk->sk_err;
1063 goto sendpage_end;
1064 }
1065
1066 if (ctx->open_rec)
1067 rec = ctx->open_rec;
1068 else
1069 rec = ctx->open_rec = tls_get_rec(sk);
1070 if (!rec) {
1071 ret = -ENOMEM;
1072 goto sendpage_end;
1073 }
1074
1075 msg_pl = &rec->msg_plaintext;
1076
1077 full_record = false;
1078 record_room = TLS_MAX_PAYLOAD_SIZE - msg_pl->sg.size;
1079 copied = 0;
1080 copy = size;
1081 if (copy >= record_room) {
1082 copy = record_room;
1083 full_record = true;
1084 }
1085
1086 required_size = msg_pl->sg.size + copy +
1087 tls_ctx->tx.overhead_size;
1088
1089 if (!sk_stream_memory_free(sk))
1090 goto wait_for_sndbuf;
1091 alloc_payload:
1092 ret = tls_alloc_encrypted_msg(sk, required_size);
1093 if (ret) {
1094 if (ret != -ENOSPC)
1095 goto wait_for_memory;
1096
1097 /* Adjust copy according to the amount that was
1098 * actually allocated. The difference is due
1099 * to max sg elements limit
1100 */
1101 copy -= required_size - msg_pl->sg.size;
1102 full_record = true;
1103 }
1104
1105 sk_msg_page_add(msg_pl, page, copy, offset);
1106 sk_mem_charge(sk, copy);
1107
1108 offset += copy;
1109 size -= copy;
1110 copied += copy;
1111
1112 tls_ctx->pending_open_record_frags = true;
1113 if (full_record || eor || sk_msg_full(msg_pl)) {
1114 rec->inplace_crypto = 0;
1115 ret = bpf_exec_tx_verdict(msg_pl, sk, full_record,
1116 record_type, &copied, flags);
1117 if (ret) {
1118 if (ret == -EINPROGRESS)
1119 num_async++;
1120 else if (ret == -ENOMEM)
1121 goto wait_for_memory;
1122 else if (ret != -EAGAIN) {
1123 if (ret == -ENOSPC)
1124 ret = 0;
1125 goto sendpage_end;
1126 }
1127 }
1128 }
1129 continue;
1130 wait_for_sndbuf:
1131 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1132 wait_for_memory:
1133 ret = sk_stream_wait_memory(sk, &timeo);
1134 if (ret) {
1135 tls_trim_both_msgs(sk, msg_pl->sg.size);
1136 goto sendpage_end;
1137 }
1138
1139 goto alloc_payload;
1140 }
1141
1142 if (num_async) {
1143 /* Transmit if any encryptions have completed */
1144 if (test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask)) {
1145 cancel_delayed_work(&ctx->tx_work.work);
1146 tls_tx_records(sk, flags);
1147 }
1148 }
1149 sendpage_end:
1150 ret = sk_stream_error(sk, flags, ret);
1151 return copied ? copied : ret;
1152 }
1153
1154 int tls_sw_sendpage(struct sock *sk, struct page *page,
1155 int offset, size_t size, int flags)
1156 {
1157 int ret;
1158
1159 if (flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL |
1160 MSG_SENDPAGE_NOTLAST | MSG_SENDPAGE_NOPOLICY))
1161 return -ENOTSUPP;
1162
1163 lock_sock(sk);
1164 ret = tls_sw_do_sendpage(sk, page, offset, size, flags);
1165 release_sock(sk);
1166 return ret;
1167 }
1168
1169 static struct sk_buff *tls_wait_data(struct sock *sk, struct sk_psock *psock,
1170 int flags, long timeo, int *err)
1171 {
1172 struct tls_context *tls_ctx = tls_get_ctx(sk);
1173 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1174 struct sk_buff *skb;
1175 DEFINE_WAIT_FUNC(wait, woken_wake_function);
1176
1177 while (!(skb = ctx->recv_pkt) && sk_psock_queue_empty(psock)) {
1178 if (sk->sk_err) {
1179 *err = sock_error(sk);
1180 return NULL;
1181 }
1182
1183 if (sk->sk_shutdown & RCV_SHUTDOWN)
1184 return NULL;
1185
1186 if (sock_flag(sk, SOCK_DONE))
1187 return NULL;
1188
1189 if ((flags & MSG_DONTWAIT) || !timeo) {
1190 *err = -EAGAIN;
1191 return NULL;
1192 }
1193
1194 add_wait_queue(sk_sleep(sk), &wait);
1195 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
1196 sk_wait_event(sk, &timeo,
1197 ctx->recv_pkt != skb ||
1198 !sk_psock_queue_empty(psock),
1199 &wait);
1200 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
1201 remove_wait_queue(sk_sleep(sk), &wait);
1202
1203 /* Handle signals */
1204 if (signal_pending(current)) {
1205 *err = sock_intr_errno(timeo);
1206 return NULL;
1207 }
1208 }
1209
1210 return skb;
1211 }
1212
1213 static int tls_setup_from_iter(struct sock *sk, struct iov_iter *from,
1214 int length, int *pages_used,
1215 unsigned int *size_used,
1216 struct scatterlist *to,
1217 int to_max_pages)
1218 {
1219 int rc = 0, i = 0, num_elem = *pages_used, maxpages;
1220 struct page *pages[MAX_SKB_FRAGS];
1221 unsigned int size = *size_used;
1222 ssize_t copied, use;
1223 size_t offset;
1224
1225 while (length > 0) {
1226 i = 0;
1227 maxpages = to_max_pages - num_elem;
1228 if (maxpages == 0) {
1229 rc = -EFAULT;
1230 goto out;
1231 }
1232 copied = iov_iter_get_pages(from, pages,
1233 length,
1234 maxpages, &offset);
1235 if (copied <= 0) {
1236 rc = -EFAULT;
1237 goto out;
1238 }
1239
1240 iov_iter_advance(from, copied);
1241
1242 length -= copied;
1243 size += copied;
1244 while (copied) {
1245 use = min_t(int, copied, PAGE_SIZE - offset);
1246
1247 sg_set_page(&to[num_elem],
1248 pages[i], use, offset);
1249 sg_unmark_end(&to[num_elem]);
1250 /* We do not uncharge memory from this API */
1251
1252 offset = 0;
1253 copied -= use;
1254
1255 i++;
1256 num_elem++;
1257 }
1258 }
1259 /* Mark the end in the last sg entry if newly added */
1260 if (num_elem > *pages_used)
1261 sg_mark_end(&to[num_elem - 1]);
1262 out:
1263 if (rc)
1264 iov_iter_revert(from, size - *size_used);
1265 *size_used = size;
1266 *pages_used = num_elem;
1267
1268 return rc;
1269 }
1270
1271 /* This function decrypts the input skb into either out_iov or in out_sg
1272 * or in skb buffers itself. The input parameter 'zc' indicates if
1273 * zero-copy mode needs to be tried or not. With zero-copy mode, either
1274 * out_iov or out_sg must be non-NULL. In case both out_iov and out_sg are
1275 * NULL, then the decryption happens inside skb buffers itself, i.e.
1276 * zero-copy gets disabled and 'zc' is updated.
1277 */
1278
1279 static int decrypt_internal(struct sock *sk, struct sk_buff *skb,
1280 struct iov_iter *out_iov,
1281 struct scatterlist *out_sg,
1282 int *chunk, bool *zc, bool async)
1283 {
1284 struct tls_context *tls_ctx = tls_get_ctx(sk);
1285 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1286 struct strp_msg *rxm = strp_msg(skb);
1287 int n_sgin, n_sgout, nsg, mem_size, aead_size, err, pages = 0;
1288 struct aead_request *aead_req;
1289 struct sk_buff *unused;
1290 u8 *aad, *iv, *mem = NULL;
1291 struct scatterlist *sgin = NULL;
1292 struct scatterlist *sgout = NULL;
1293 const int data_len = rxm->full_len - tls_ctx->rx.overhead_size;
1294
1295 if (*zc && (out_iov || out_sg)) {
1296 if (out_iov)
1297 n_sgout = iov_iter_npages(out_iov, INT_MAX) + 1;
1298 else
1299 n_sgout = sg_nents(out_sg);
1300 n_sgin = skb_nsg(skb, rxm->offset + tls_ctx->rx.prepend_size,
1301 rxm->full_len - tls_ctx->rx.prepend_size);
1302 } else {
1303 n_sgout = 0;
1304 *zc = false;
1305 n_sgin = skb_cow_data(skb, 0, &unused);
1306 }
1307
1308 if (n_sgin < 1)
1309 return -EBADMSG;
1310
1311 /* Increment to accommodate AAD */
1312 n_sgin = n_sgin + 1;
1313
1314 nsg = n_sgin + n_sgout;
1315
1316 aead_size = sizeof(*aead_req) + crypto_aead_reqsize(ctx->aead_recv);
1317 mem_size = aead_size + (nsg * sizeof(struct scatterlist));
1318 mem_size = mem_size + TLS_AAD_SPACE_SIZE;
1319 mem_size = mem_size + crypto_aead_ivsize(ctx->aead_recv);
1320
1321 /* Allocate a single block of memory which contains
1322 * aead_req || sgin[] || sgout[] || aad || iv.
1323 * This order achieves correct alignment for aead_req, sgin, sgout.
1324 */
1325 mem = kmalloc(mem_size, sk->sk_allocation);
1326 if (!mem)
1327 return -ENOMEM;
1328
1329 /* Segment the allocated memory */
1330 aead_req = (struct aead_request *)mem;
1331 sgin = (struct scatterlist *)(mem + aead_size);
1332 sgout = sgin + n_sgin;
1333 aad = (u8 *)(sgout + n_sgout);
1334 iv = aad + TLS_AAD_SPACE_SIZE;
1335
1336 /* Prepare IV */
1337 err = skb_copy_bits(skb, rxm->offset + TLS_HEADER_SIZE,
1338 iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
1339 tls_ctx->rx.iv_size);
1340 if (err < 0) {
1341 kfree(mem);
1342 return err;
1343 }
1344 memcpy(iv, tls_ctx->rx.iv, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
1345
1346 /* Prepare AAD */
1347 tls_make_aad(aad, rxm->full_len - tls_ctx->rx.overhead_size,
1348 tls_ctx->rx.rec_seq, tls_ctx->rx.rec_seq_size,
1349 ctx->control);
1350
1351 /* Prepare sgin */
1352 sg_init_table(sgin, n_sgin);
1353 sg_set_buf(&sgin[0], aad, TLS_AAD_SPACE_SIZE);
1354 err = skb_to_sgvec(skb, &sgin[1],
1355 rxm->offset + tls_ctx->rx.prepend_size,
1356 rxm->full_len - tls_ctx->rx.prepend_size);
1357 if (err < 0) {
1358 kfree(mem);
1359 return err;
1360 }
1361
1362 if (n_sgout) {
1363 if (out_iov) {
1364 sg_init_table(sgout, n_sgout);
1365 sg_set_buf(&sgout[0], aad, TLS_AAD_SPACE_SIZE);
1366
1367 *chunk = 0;
1368 err = tls_setup_from_iter(sk, out_iov, data_len,
1369 &pages, chunk, &sgout[1],
1370 (n_sgout - 1));
1371 if (err < 0)
1372 goto fallback_to_reg_recv;
1373 } else if (out_sg) {
1374 memcpy(sgout, out_sg, n_sgout * sizeof(*sgout));
1375 } else {
1376 goto fallback_to_reg_recv;
1377 }
1378 } else {
1379 fallback_to_reg_recv:
1380 sgout = sgin;
1381 pages = 0;
1382 *chunk = data_len;
1383 *zc = false;
1384 }
1385
1386 /* Prepare and submit AEAD request */
1387 err = tls_do_decryption(sk, skb, sgin, sgout, iv,
1388 data_len, aead_req, async);
1389 if (err == -EINPROGRESS)
1390 return err;
1391
1392 /* Release the pages in case iov was mapped to pages */
1393 for (; pages > 0; pages--)
1394 put_page(sg_page(&sgout[pages]));
1395
1396 kfree(mem);
1397 return err;
1398 }
1399
1400 static int decrypt_skb_update(struct sock *sk, struct sk_buff *skb,
1401 struct iov_iter *dest, int *chunk, bool *zc,
1402 bool async)
1403 {
1404 struct tls_context *tls_ctx = tls_get_ctx(sk);
1405 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1406 struct strp_msg *rxm = strp_msg(skb);
1407 int err = 0;
1408
1409 #ifdef CONFIG_TLS_DEVICE
1410 err = tls_device_decrypted(sk, skb);
1411 if (err < 0)
1412 return err;
1413 #endif
1414 if (!ctx->decrypted) {
1415 err = decrypt_internal(sk, skb, dest, NULL, chunk, zc, async);
1416 if (err < 0) {
1417 if (err == -EINPROGRESS)
1418 tls_advance_record_sn(sk, &tls_ctx->rx);
1419
1420 return err;
1421 }
1422 } else {
1423 *zc = false;
1424 }
1425
1426 rxm->offset += tls_ctx->rx.prepend_size;
1427 rxm->full_len -= tls_ctx->rx.overhead_size;
1428 tls_advance_record_sn(sk, &tls_ctx->rx);
1429 ctx->decrypted = true;
1430 ctx->saved_data_ready(sk);
1431
1432 return err;
1433 }
1434
1435 int decrypt_skb(struct sock *sk, struct sk_buff *skb,
1436 struct scatterlist *sgout)
1437 {
1438 bool zc = true;
1439 int chunk;
1440
1441 return decrypt_internal(sk, skb, NULL, sgout, &chunk, &zc, false);
1442 }
1443
1444 static bool tls_sw_advance_skb(struct sock *sk, struct sk_buff *skb,
1445 unsigned int len)
1446 {
1447 struct tls_context *tls_ctx = tls_get_ctx(sk);
1448 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1449
1450 if (skb) {
1451 struct strp_msg *rxm = strp_msg(skb);
1452
1453 if (len < rxm->full_len) {
1454 rxm->offset += len;
1455 rxm->full_len -= len;
1456 return false;
1457 }
1458 kfree_skb(skb);
1459 }
1460
1461 /* Finished with message */
1462 ctx->recv_pkt = NULL;
1463 __strp_unpause(&ctx->strp);
1464
1465 return true;
1466 }
1467
1468 /* This function traverses the rx_list in tls receive context to copies the
1469 * decrypted data records into the buffer provided by caller zero copy is not
1470 * true. Further, the records are removed from the rx_list if it is not a peek
1471 * case and the record has been consumed completely.
1472 */
1473 static int process_rx_list(struct tls_sw_context_rx *ctx,
1474 struct msghdr *msg,
1475 size_t skip,
1476 size_t len,
1477 bool zc,
1478 bool is_peek)
1479 {
1480 struct sk_buff *skb = skb_peek(&ctx->rx_list);
1481 ssize_t copied = 0;
1482
1483 while (skip && skb) {
1484 struct strp_msg *rxm = strp_msg(skb);
1485
1486 if (skip < rxm->full_len)
1487 break;
1488
1489 skip = skip - rxm->full_len;
1490 skb = skb_peek_next(skb, &ctx->rx_list);
1491 }
1492
1493 while (len && skb) {
1494 struct sk_buff *next_skb;
1495 struct strp_msg *rxm = strp_msg(skb);
1496 int chunk = min_t(unsigned int, rxm->full_len - skip, len);
1497
1498 if (!zc || (rxm->full_len - skip) > len) {
1499 int err = skb_copy_datagram_msg(skb, rxm->offset + skip,
1500 msg, chunk);
1501 if (err < 0)
1502 return err;
1503 }
1504
1505 len = len - chunk;
1506 copied = copied + chunk;
1507
1508 /* Consume the data from record if it is non-peek case*/
1509 if (!is_peek) {
1510 rxm->offset = rxm->offset + chunk;
1511 rxm->full_len = rxm->full_len - chunk;
1512
1513 /* Return if there is unconsumed data in the record */
1514 if (rxm->full_len - skip)
1515 break;
1516 }
1517
1518 /* The remaining skip-bytes must lie in 1st record in rx_list.
1519 * So from the 2nd record, 'skip' should be 0.
1520 */
1521 skip = 0;
1522
1523 if (msg)
1524 msg->msg_flags |= MSG_EOR;
1525
1526 next_skb = skb_peek_next(skb, &ctx->rx_list);
1527
1528 if (!is_peek) {
1529 skb_unlink(skb, &ctx->rx_list);
1530 kfree_skb(skb);
1531 }
1532
1533 skb = next_skb;
1534 }
1535
1536 return copied;
1537 }
1538
1539 int tls_sw_recvmsg(struct sock *sk,
1540 struct msghdr *msg,
1541 size_t len,
1542 int nonblock,
1543 int flags,
1544 int *addr_len)
1545 {
1546 struct tls_context *tls_ctx = tls_get_ctx(sk);
1547 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1548 struct sk_psock *psock;
1549 unsigned char control = 0;
1550 ssize_t decrypted = 0;
1551 struct strp_msg *rxm;
1552 struct sk_buff *skb;
1553 ssize_t copied = 0;
1554 bool cmsg = false;
1555 int target, err = 0;
1556 long timeo;
1557 bool is_kvec = iov_iter_is_kvec(&msg->msg_iter);
1558 bool is_peek = flags & MSG_PEEK;
1559 int num_async = 0;
1560
1561 flags |= nonblock;
1562
1563 if (unlikely(flags & MSG_ERRQUEUE))
1564 return sock_recv_errqueue(sk, msg, len, SOL_IP, IP_RECVERR);
1565
1566 psock = sk_psock_get(sk);
1567 lock_sock(sk);
1568
1569 /* Process pending decrypted records. It must be non-zero-copy */
1570 err = process_rx_list(ctx, msg, 0, len, false, is_peek);
1571 if (err < 0) {
1572 tls_err_abort(sk, err);
1573 goto end;
1574 } else {
1575 copied = err;
1576 }
1577
1578 len = len - copied;
1579 if (len) {
1580 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
1581 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1582 } else {
1583 goto recv_end;
1584 }
1585
1586 do {
1587 bool retain_skb = false;
1588 bool async = false;
1589 bool zc = false;
1590 int to_decrypt;
1591 int chunk = 0;
1592
1593 skb = tls_wait_data(sk, psock, flags, timeo, &err);
1594 if (!skb) {
1595 if (psock) {
1596 int ret = __tcp_bpf_recvmsg(sk, psock,
1597 msg, len, flags);
1598
1599 if (ret > 0) {
1600 decrypted += ret;
1601 len -= ret;
1602 continue;
1603 }
1604 }
1605 goto recv_end;
1606 }
1607
1608 rxm = strp_msg(skb);
1609
1610 if (!cmsg) {
1611 int cerr;
1612
1613 cerr = put_cmsg(msg, SOL_TLS, TLS_GET_RECORD_TYPE,
1614 sizeof(ctx->control), &ctx->control);
1615 cmsg = true;
1616 control = ctx->control;
1617 if (ctx->control != TLS_RECORD_TYPE_DATA) {
1618 if (cerr || msg->msg_flags & MSG_CTRUNC) {
1619 err = -EIO;
1620 goto recv_end;
1621 }
1622 }
1623 } else if (control != ctx->control) {
1624 goto recv_end;
1625 }
1626
1627 to_decrypt = rxm->full_len - tls_ctx->rx.overhead_size;
1628
1629 if (to_decrypt <= len && !is_kvec && !is_peek)
1630 zc = true;
1631
1632 err = decrypt_skb_update(sk, skb, &msg->msg_iter,
1633 &chunk, &zc, ctx->async_capable);
1634 if (err < 0 && err != -EINPROGRESS) {
1635 tls_err_abort(sk, EBADMSG);
1636 goto recv_end;
1637 }
1638
1639 if (err == -EINPROGRESS) {
1640 async = true;
1641 num_async++;
1642 goto pick_next_record;
1643 } else {
1644 if (!zc) {
1645 if (rxm->full_len > len) {
1646 retain_skb = true;
1647 chunk = len;
1648 } else {
1649 chunk = rxm->full_len;
1650 }
1651
1652 err = skb_copy_datagram_msg(skb, rxm->offset,
1653 msg, chunk);
1654 if (err < 0)
1655 goto recv_end;
1656
1657 if (!is_peek) {
1658 rxm->offset = rxm->offset + chunk;
1659 rxm->full_len = rxm->full_len - chunk;
1660 }
1661 }
1662 }
1663
1664 pick_next_record:
1665 if (chunk > len)
1666 chunk = len;
1667
1668 decrypted += chunk;
1669 len -= chunk;
1670
1671 /* For async or peek case, queue the current skb */
1672 if (async || is_peek || retain_skb) {
1673 skb_queue_tail(&ctx->rx_list, skb);
1674 skb = NULL;
1675 }
1676
1677 if (tls_sw_advance_skb(sk, skb, chunk)) {
1678 /* Return full control message to
1679 * userspace before trying to parse
1680 * another message type
1681 */
1682 msg->msg_flags |= MSG_EOR;
1683 if (ctx->control != TLS_RECORD_TYPE_DATA)
1684 goto recv_end;
1685 } else {
1686 break;
1687 }
1688
1689 /* If we have a new message from strparser, continue now. */
1690 if (decrypted >= target && !ctx->recv_pkt)
1691 break;
1692 } while (len);
1693
1694 recv_end:
1695 if (num_async) {
1696 /* Wait for all previously submitted records to be decrypted */
1697 smp_store_mb(ctx->async_notify, true);
1698 if (atomic_read(&ctx->decrypt_pending)) {
1699 err = crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
1700 if (err) {
1701 /* one of async decrypt failed */
1702 tls_err_abort(sk, err);
1703 copied = 0;
1704 decrypted = 0;
1705 goto end;
1706 }
1707 } else {
1708 reinit_completion(&ctx->async_wait.completion);
1709 }
1710 WRITE_ONCE(ctx->async_notify, false);
1711
1712 /* Drain records from the rx_list & copy if required */
1713 if (is_peek || is_kvec)
1714 err = process_rx_list(ctx, msg, copied,
1715 decrypted, false, is_peek);
1716 else
1717 err = process_rx_list(ctx, msg, 0,
1718 decrypted, true, is_peek);
1719 if (err < 0) {
1720 tls_err_abort(sk, err);
1721 copied = 0;
1722 goto end;
1723 }
1724
1725 WARN_ON(decrypted != err);
1726 }
1727
1728 copied += decrypted;
1729
1730 end:
1731 release_sock(sk);
1732 if (psock)
1733 sk_psock_put(sk, psock);
1734 return copied ? : err;
1735 }
1736
1737 ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
1738 struct pipe_inode_info *pipe,
1739 size_t len, unsigned int flags)
1740 {
1741 struct tls_context *tls_ctx = tls_get_ctx(sock->sk);
1742 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1743 struct strp_msg *rxm = NULL;
1744 struct sock *sk = sock->sk;
1745 struct sk_buff *skb;
1746 ssize_t copied = 0;
1747 int err = 0;
1748 long timeo;
1749 int chunk;
1750 bool zc = false;
1751
1752 lock_sock(sk);
1753
1754 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
1755
1756 skb = tls_wait_data(sk, NULL, flags, timeo, &err);
1757 if (!skb)
1758 goto splice_read_end;
1759
1760 /* splice does not support reading control messages */
1761 if (ctx->control != TLS_RECORD_TYPE_DATA) {
1762 err = -ENOTSUPP;
1763 goto splice_read_end;
1764 }
1765
1766 if (!ctx->decrypted) {
1767 err = decrypt_skb_update(sk, skb, NULL, &chunk, &zc, false);
1768
1769 if (err < 0) {
1770 tls_err_abort(sk, EBADMSG);
1771 goto splice_read_end;
1772 }
1773 ctx->decrypted = true;
1774 }
1775 rxm = strp_msg(skb);
1776
1777 chunk = min_t(unsigned int, rxm->full_len, len);
1778 copied = skb_splice_bits(skb, sk, rxm->offset, pipe, chunk, flags);
1779 if (copied < 0)
1780 goto splice_read_end;
1781
1782 if (likely(!(flags & MSG_PEEK)))
1783 tls_sw_advance_skb(sk, skb, copied);
1784
1785 splice_read_end:
1786 release_sock(sk);
1787 return copied ? : err;
1788 }
1789
1790 bool tls_sw_stream_read(const struct sock *sk)
1791 {
1792 struct tls_context *tls_ctx = tls_get_ctx(sk);
1793 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1794 bool ingress_empty = true;
1795 struct sk_psock *psock;
1796
1797 rcu_read_lock();
1798 psock = sk_psock(sk);
1799 if (psock)
1800 ingress_empty = list_empty(&psock->ingress_msg);
1801 rcu_read_unlock();
1802
1803 return !ingress_empty || ctx->recv_pkt;
1804 }
1805
1806 static int tls_read_size(struct strparser *strp, struct sk_buff *skb)
1807 {
1808 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
1809 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1810 char header[TLS_HEADER_SIZE + MAX_IV_SIZE];
1811 struct strp_msg *rxm = strp_msg(skb);
1812 size_t cipher_overhead;
1813 size_t data_len = 0;
1814 int ret;
1815
1816 /* Verify that we have a full TLS header, or wait for more data */
1817 if (rxm->offset + tls_ctx->rx.prepend_size > skb->len)
1818 return 0;
1819
1820 /* Sanity-check size of on-stack buffer. */
1821 if (WARN_ON(tls_ctx->rx.prepend_size > sizeof(header))) {
1822 ret = -EINVAL;
1823 goto read_failure;
1824 }
1825
1826 /* Linearize header to local buffer */
1827 ret = skb_copy_bits(skb, rxm->offset, header, tls_ctx->rx.prepend_size);
1828
1829 if (ret < 0)
1830 goto read_failure;
1831
1832 ctx->control = header[0];
1833
1834 data_len = ((header[4] & 0xFF) | (header[3] << 8));
1835
1836 cipher_overhead = tls_ctx->rx.tag_size + tls_ctx->rx.iv_size;
1837
1838 if (data_len > TLS_MAX_PAYLOAD_SIZE + cipher_overhead) {
1839 ret = -EMSGSIZE;
1840 goto read_failure;
1841 }
1842 if (data_len < cipher_overhead) {
1843 ret = -EBADMSG;
1844 goto read_failure;
1845 }
1846
1847 if (header[1] != TLS_VERSION_MINOR(tls_ctx->crypto_recv.info.version) ||
1848 header[2] != TLS_VERSION_MAJOR(tls_ctx->crypto_recv.info.version)) {
1849 ret = -EINVAL;
1850 goto read_failure;
1851 }
1852
1853 #ifdef CONFIG_TLS_DEVICE
1854 handle_device_resync(strp->sk, TCP_SKB_CB(skb)->seq + rxm->offset,
1855 *(u64*)tls_ctx->rx.rec_seq);
1856 #endif
1857 return data_len + TLS_HEADER_SIZE;
1858
1859 read_failure:
1860 tls_err_abort(strp->sk, ret);
1861
1862 return ret;
1863 }
1864
1865 static void tls_queue(struct strparser *strp, struct sk_buff *skb)
1866 {
1867 struct tls_context *tls_ctx = tls_get_ctx(strp->sk);
1868 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1869
1870 ctx->decrypted = false;
1871
1872 ctx->recv_pkt = skb;
1873 strp_pause(strp);
1874
1875 ctx->saved_data_ready(strp->sk);
1876 }
1877
1878 static void tls_data_ready(struct sock *sk)
1879 {
1880 struct tls_context *tls_ctx = tls_get_ctx(sk);
1881 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1882 struct sk_psock *psock;
1883
1884 strp_data_ready(&ctx->strp);
1885
1886 psock = sk_psock_get(sk);
1887 if (psock && !list_empty(&psock->ingress_msg)) {
1888 ctx->saved_data_ready(sk);
1889 sk_psock_put(sk, psock);
1890 }
1891 }
1892
1893 void tls_sw_free_resources_tx(struct sock *sk)
1894 {
1895 struct tls_context *tls_ctx = tls_get_ctx(sk);
1896 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
1897 struct tls_rec *rec, *tmp;
1898
1899 /* Wait for any pending async encryptions to complete */
1900 smp_store_mb(ctx->async_notify, true);
1901 if (atomic_read(&ctx->encrypt_pending))
1902 crypto_wait_req(-EINPROGRESS, &ctx->async_wait);
1903
1904 cancel_delayed_work_sync(&ctx->tx_work.work);
1905
1906 /* Tx whatever records we can transmit and abandon the rest */
1907 tls_tx_records(sk, -1);
1908
1909 /* Free up un-sent records in tx_list. First, free
1910 * the partially sent record if any at head of tx_list.
1911 */
1912 if (tls_ctx->partially_sent_record) {
1913 struct scatterlist *sg = tls_ctx->partially_sent_record;
1914
1915 while (1) {
1916 put_page(sg_page(sg));
1917 sk_mem_uncharge(sk, sg->length);
1918
1919 if (sg_is_last(sg))
1920 break;
1921 sg++;
1922 }
1923
1924 tls_ctx->partially_sent_record = NULL;
1925
1926 rec = list_first_entry(&ctx->tx_list,
1927 struct tls_rec, list);
1928 list_del(&rec->list);
1929 sk_msg_free(sk, &rec->msg_plaintext);
1930 kfree(rec);
1931 }
1932
1933 list_for_each_entry_safe(rec, tmp, &ctx->tx_list, list) {
1934 list_del(&rec->list);
1935 sk_msg_free(sk, &rec->msg_encrypted);
1936 sk_msg_free(sk, &rec->msg_plaintext);
1937 kfree(rec);
1938 }
1939
1940 crypto_free_aead(ctx->aead_send);
1941 tls_free_open_rec(sk);
1942
1943 kfree(ctx);
1944 }
1945
1946 void tls_sw_release_resources_rx(struct sock *sk)
1947 {
1948 struct tls_context *tls_ctx = tls_get_ctx(sk);
1949 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1950
1951 if (ctx->aead_recv) {
1952 kfree_skb(ctx->recv_pkt);
1953 ctx->recv_pkt = NULL;
1954 skb_queue_purge(&ctx->rx_list);
1955 crypto_free_aead(ctx->aead_recv);
1956 strp_stop(&ctx->strp);
1957 write_lock_bh(&sk->sk_callback_lock);
1958 sk->sk_data_ready = ctx->saved_data_ready;
1959 write_unlock_bh(&sk->sk_callback_lock);
1960 release_sock(sk);
1961 strp_done(&ctx->strp);
1962 lock_sock(sk);
1963 }
1964 }
1965
1966 void tls_sw_free_resources_rx(struct sock *sk)
1967 {
1968 struct tls_context *tls_ctx = tls_get_ctx(sk);
1969 struct tls_sw_context_rx *ctx = tls_sw_ctx_rx(tls_ctx);
1970
1971 tls_sw_release_resources_rx(sk);
1972
1973 kfree(ctx);
1974 }
1975
1976 /* The work handler to transmitt the encrypted records in tx_list */
1977 static void tx_work_handler(struct work_struct *work)
1978 {
1979 struct delayed_work *delayed_work = to_delayed_work(work);
1980 struct tx_work *tx_work = container_of(delayed_work,
1981 struct tx_work, work);
1982 struct sock *sk = tx_work->sk;
1983 struct tls_context *tls_ctx = tls_get_ctx(sk);
1984 struct tls_sw_context_tx *ctx = tls_sw_ctx_tx(tls_ctx);
1985
1986 if (!test_and_clear_bit(BIT_TX_SCHEDULED, &ctx->tx_bitmask))
1987 return;
1988
1989 lock_sock(sk);
1990 tls_tx_records(sk, -1);
1991 release_sock(sk);
1992 }
1993
1994 int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx)
1995 {
1996 struct tls_crypto_info *crypto_info;
1997 struct tls12_crypto_info_aes_gcm_128 *gcm_128_info;
1998 struct tls_sw_context_tx *sw_ctx_tx = NULL;
1999 struct tls_sw_context_rx *sw_ctx_rx = NULL;
2000 struct cipher_context *cctx;
2001 struct crypto_aead **aead;
2002 struct strp_callbacks cb;
2003 u16 nonce_size, tag_size, iv_size, rec_seq_size;
2004 struct crypto_tfm *tfm;
2005 char *iv, *rec_seq;
2006 int rc = 0;
2007
2008 if (!ctx) {
2009 rc = -EINVAL;
2010 goto out;
2011 }
2012
2013 if (tx) {
2014 if (!ctx->priv_ctx_tx) {
2015 sw_ctx_tx = kzalloc(sizeof(*sw_ctx_tx), GFP_KERNEL);
2016 if (!sw_ctx_tx) {
2017 rc = -ENOMEM;
2018 goto out;
2019 }
2020 ctx->priv_ctx_tx = sw_ctx_tx;
2021 } else {
2022 sw_ctx_tx =
2023 (struct tls_sw_context_tx *)ctx->priv_ctx_tx;
2024 }
2025 } else {
2026 if (!ctx->priv_ctx_rx) {
2027 sw_ctx_rx = kzalloc(sizeof(*sw_ctx_rx), GFP_KERNEL);
2028 if (!sw_ctx_rx) {
2029 rc = -ENOMEM;
2030 goto out;
2031 }
2032 ctx->priv_ctx_rx = sw_ctx_rx;
2033 } else {
2034 sw_ctx_rx =
2035 (struct tls_sw_context_rx *)ctx->priv_ctx_rx;
2036 }
2037 }
2038
2039 if (tx) {
2040 crypto_init_wait(&sw_ctx_tx->async_wait);
2041 crypto_info = &ctx->crypto_send.info;
2042 cctx = &ctx->tx;
2043 aead = &sw_ctx_tx->aead_send;
2044 INIT_LIST_HEAD(&sw_ctx_tx->tx_list);
2045 INIT_DELAYED_WORK(&sw_ctx_tx->tx_work.work, tx_work_handler);
2046 sw_ctx_tx->tx_work.sk = sk;
2047 } else {
2048 crypto_init_wait(&sw_ctx_rx->async_wait);
2049 crypto_info = &ctx->crypto_recv.info;
2050 cctx = &ctx->rx;
2051 skb_queue_head_init(&sw_ctx_rx->rx_list);
2052 aead = &sw_ctx_rx->aead_recv;
2053 }
2054
2055 switch (crypto_info->cipher_type) {
2056 case TLS_CIPHER_AES_GCM_128: {
2057 nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
2058 tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
2059 iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
2060 iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv;
2061 rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE;
2062 rec_seq =
2063 ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq;
2064 gcm_128_info =
2065 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
2066 break;
2067 }
2068 default:
2069 rc = -EINVAL;
2070 goto free_priv;
2071 }
2072
2073 /* Sanity-check the IV size for stack allocations. */
2074 if (iv_size > MAX_IV_SIZE || nonce_size > MAX_IV_SIZE) {
2075 rc = -EINVAL;
2076 goto free_priv;
2077 }
2078
2079 cctx->prepend_size = TLS_HEADER_SIZE + nonce_size;
2080 cctx->tag_size = tag_size;
2081 cctx->overhead_size = cctx->prepend_size + cctx->tag_size;
2082 cctx->iv_size = iv_size;
2083 cctx->iv = kmalloc(iv_size + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
2084 GFP_KERNEL);
2085 if (!cctx->iv) {
2086 rc = -ENOMEM;
2087 goto free_priv;
2088 }
2089 memcpy(cctx->iv, gcm_128_info->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
2090 memcpy(cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv, iv_size);
2091 cctx->rec_seq_size = rec_seq_size;
2092 cctx->rec_seq = kmemdup(rec_seq, rec_seq_size, GFP_KERNEL);
2093 if (!cctx->rec_seq) {
2094 rc = -ENOMEM;
2095 goto free_iv;
2096 }
2097
2098 if (!*aead) {
2099 *aead = crypto_alloc_aead("gcm(aes)", 0, 0);
2100 if (IS_ERR(*aead)) {
2101 rc = PTR_ERR(*aead);
2102 *aead = NULL;
2103 goto free_rec_seq;
2104 }
2105 }
2106
2107 ctx->push_pending_record = tls_sw_push_pending_record;
2108
2109 rc = crypto_aead_setkey(*aead, gcm_128_info->key,
2110 TLS_CIPHER_AES_GCM_128_KEY_SIZE);
2111 if (rc)
2112 goto free_aead;
2113
2114 rc = crypto_aead_setauthsize(*aead, cctx->tag_size);
2115 if (rc)
2116 goto free_aead;
2117
2118 if (sw_ctx_rx) {
2119 tfm = crypto_aead_tfm(sw_ctx_rx->aead_recv);
2120 sw_ctx_rx->async_capable =
2121 tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC;
2122
2123 /* Set up strparser */
2124 memset(&cb, 0, sizeof(cb));
2125 cb.rcv_msg = tls_queue;
2126 cb.parse_msg = tls_read_size;
2127
2128 strp_init(&sw_ctx_rx->strp, sk, &cb);
2129
2130 write_lock_bh(&sk->sk_callback_lock);
2131 sw_ctx_rx->saved_data_ready = sk->sk_data_ready;
2132 sk->sk_data_ready = tls_data_ready;
2133 write_unlock_bh(&sk->sk_callback_lock);
2134
2135 strp_check_rcv(&sw_ctx_rx->strp);
2136 }
2137
2138 goto out;
2139
2140 free_aead:
2141 crypto_free_aead(*aead);
2142 *aead = NULL;
2143 free_rec_seq:
2144 kfree(cctx->rec_seq);
2145 cctx->rec_seq = NULL;
2146 free_iv:
2147 kfree(cctx->iv);
2148 cctx->iv = NULL;
2149 free_priv:
2150 if (tx) {
2151 kfree(ctx->priv_ctx_tx);
2152 ctx->priv_ctx_tx = NULL;
2153 } else {
2154 kfree(ctx->priv_ctx_rx);
2155 ctx->priv_ctx_rx = NULL;
2156 }
2157 out:
2158 return rc;
2159 }
Ubuntu Hirsute Linux kernel mirror