2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
4 * Copyright (c) 2016-2017, Lance Chao <lancerchao@fb.com>. All rights reserved.
5 * Copyright (c) 2016, Fridolin Pokorny <fridolin.pokorny@gmail.com>. All rights reserved.
6 * Copyright (c) 2016, Nikos Mavrogiannopoulos <nmav@gnutls.org>. All rights reserved.
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37 #include <linux/module.h>
38 #include <crypto/aead.h>
42 static void trim_sg(struct sock
*sk
, struct scatterlist
*sg
,
43 int *sg_num_elem
, unsigned int *sg_size
, int target_size
)
45 int i
= *sg_num_elem
- 1;
46 int trim
= *sg_size
- target_size
;
53 *sg_size
= target_size
;
54 while (trim
>= sg
[i
].length
) {
56 sk_mem_uncharge(sk
, sg
[i
].length
);
57 put_page(sg_page(&sg
[i
]));
65 sk_mem_uncharge(sk
, trim
);
71 static void trim_both_sgl(struct sock
*sk
, int target_size
)
73 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
74 struct tls_sw_context
*ctx
= tls_sw_ctx(tls_ctx
);
76 trim_sg(sk
, ctx
->sg_plaintext_data
,
77 &ctx
->sg_plaintext_num_elem
,
78 &ctx
->sg_plaintext_size
,
82 target_size
+= tls_ctx
->overhead_size
;
84 trim_sg(sk
, ctx
->sg_encrypted_data
,
85 &ctx
->sg_encrypted_num_elem
,
86 &ctx
->sg_encrypted_size
,
90 static int alloc_sg(struct sock
*sk
, int len
, struct scatterlist
*sg
,
91 int *sg_num_elem
, unsigned int *sg_size
,
94 struct page_frag
*pfrag
;
95 unsigned int size
= *sg_size
;
96 int num_elem
= *sg_num_elem
, use
= 0, rc
= 0;
97 struct scatterlist
*sge
;
98 unsigned int orig_offset
;
101 pfrag
= sk_page_frag(sk
);
104 if (!sk_page_frag_refill(sk
, pfrag
)) {
109 use
= min_t(int, len
, pfrag
->size
- pfrag
->offset
);
111 if (!sk_wmem_schedule(sk
, use
)) {
116 sk_mem_charge(sk
, use
);
118 orig_offset
= pfrag
->offset
;
119 pfrag
->offset
+= use
;
121 sge
= sg
+ num_elem
- 1;
123 if (num_elem
> first_coalesce
&& sg_page(sge
) == pfrag
->page
&&
124 sge
->offset
+ sge
->length
== orig_offset
) {
129 sg_set_page(sge
, pfrag
->page
, use
, orig_offset
);
130 get_page(pfrag
->page
);
132 if (num_elem
== MAX_SKB_FRAGS
) {
144 *sg_num_elem
= num_elem
;
148 static int alloc_encrypted_sg(struct sock
*sk
, int len
)
150 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
151 struct tls_sw_context
*ctx
= tls_sw_ctx(tls_ctx
);
154 rc
= alloc_sg(sk
, len
, ctx
->sg_encrypted_data
,
155 &ctx
->sg_encrypted_num_elem
, &ctx
->sg_encrypted_size
, 0);
160 static int alloc_plaintext_sg(struct sock
*sk
, int len
)
162 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
163 struct tls_sw_context
*ctx
= tls_sw_ctx(tls_ctx
);
166 rc
= alloc_sg(sk
, len
, ctx
->sg_plaintext_data
,
167 &ctx
->sg_plaintext_num_elem
, &ctx
->sg_plaintext_size
,
168 tls_ctx
->pending_open_record_frags
);
173 static void free_sg(struct sock
*sk
, struct scatterlist
*sg
,
174 int *sg_num_elem
, unsigned int *sg_size
)
176 int i
, n
= *sg_num_elem
;
178 for (i
= 0; i
< n
; ++i
) {
179 sk_mem_uncharge(sk
, sg
[i
].length
);
180 put_page(sg_page(&sg
[i
]));
186 static void tls_free_both_sg(struct sock
*sk
)
188 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
189 struct tls_sw_context
*ctx
= tls_sw_ctx(tls_ctx
);
191 free_sg(sk
, ctx
->sg_encrypted_data
, &ctx
->sg_encrypted_num_elem
,
192 &ctx
->sg_encrypted_size
);
194 free_sg(sk
, ctx
->sg_plaintext_data
, &ctx
->sg_plaintext_num_elem
,
195 &ctx
->sg_plaintext_size
);
198 static int tls_do_encryption(struct tls_context
*tls_ctx
,
199 struct tls_sw_context
*ctx
,
200 struct aead_request
*aead_req
,
205 ctx
->sg_encrypted_data
[0].offset
+= tls_ctx
->prepend_size
;
206 ctx
->sg_encrypted_data
[0].length
-= tls_ctx
->prepend_size
;
208 aead_request_set_tfm(aead_req
, ctx
->aead_send
);
209 aead_request_set_ad(aead_req
, TLS_AAD_SPACE_SIZE
);
210 aead_request_set_crypt(aead_req
, ctx
->sg_aead_in
, ctx
->sg_aead_out
,
211 data_len
, tls_ctx
->iv
);
212 rc
= crypto_aead_encrypt(aead_req
);
214 ctx
->sg_encrypted_data
[0].offset
-= tls_ctx
->prepend_size
;
215 ctx
->sg_encrypted_data
[0].length
+= tls_ctx
->prepend_size
;
220 static int tls_push_record(struct sock
*sk
, int flags
,
221 unsigned char record_type
)
223 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
224 struct tls_sw_context
*ctx
= tls_sw_ctx(tls_ctx
);
225 struct aead_request
*req
;
228 req
= kzalloc(sizeof(struct aead_request
) +
229 crypto_aead_reqsize(ctx
->aead_send
), sk
->sk_allocation
);
233 sg_mark_end(ctx
->sg_plaintext_data
+ ctx
->sg_plaintext_num_elem
- 1);
234 sg_mark_end(ctx
->sg_encrypted_data
+ ctx
->sg_encrypted_num_elem
- 1);
236 tls_make_aad(ctx
->aad_space
, ctx
->sg_plaintext_size
,
237 tls_ctx
->rec_seq
, tls_ctx
->rec_seq_size
,
240 tls_fill_prepend(tls_ctx
,
241 page_address(sg_page(&ctx
->sg_encrypted_data
[0])) +
242 ctx
->sg_encrypted_data
[0].offset
,
243 ctx
->sg_plaintext_size
, record_type
);
245 tls_ctx
->pending_open_record_frags
= 0;
246 set_bit(TLS_PENDING_CLOSED_RECORD
, &tls_ctx
->flags
);
248 rc
= tls_do_encryption(tls_ctx
, ctx
, req
, ctx
->sg_plaintext_size
);
250 /* If we are called from write_space and
251 * we fail, we need to set this SOCK_NOSPACE
252 * to trigger another write_space in the future.
254 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
258 free_sg(sk
, ctx
->sg_plaintext_data
, &ctx
->sg_plaintext_num_elem
,
259 &ctx
->sg_plaintext_size
);
261 ctx
->sg_encrypted_num_elem
= 0;
262 ctx
->sg_encrypted_size
= 0;
264 /* Only pass through MSG_DONTWAIT and MSG_NOSIGNAL flags */
265 rc
= tls_push_sg(sk
, tls_ctx
, ctx
->sg_encrypted_data
, 0, flags
);
266 if (rc
< 0 && rc
!= -EAGAIN
)
269 tls_advance_record_sn(sk
, tls_ctx
);
275 static int tls_sw_push_pending_record(struct sock
*sk
, int flags
)
277 return tls_push_record(sk
, flags
, TLS_RECORD_TYPE_DATA
);
280 static int zerocopy_from_iter(struct sock
*sk
, struct iov_iter
*from
,
283 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
284 struct tls_sw_context
*ctx
= tls_sw_ctx(tls_ctx
);
285 struct page
*pages
[MAX_SKB_FRAGS
];
290 unsigned int size
= ctx
->sg_plaintext_size
;
291 int num_elem
= ctx
->sg_plaintext_num_elem
;
297 maxpages
= ARRAY_SIZE(ctx
->sg_plaintext_data
) - num_elem
;
302 copied
= iov_iter_get_pages(from
, pages
,
310 iov_iter_advance(from
, copied
);
315 use
= min_t(int, copied
, PAGE_SIZE
- offset
);
317 sg_set_page(&ctx
->sg_plaintext_data
[num_elem
],
318 pages
[i
], use
, offset
);
319 sg_unmark_end(&ctx
->sg_plaintext_data
[num_elem
]);
320 sk_mem_charge(sk
, use
);
331 ctx
->sg_plaintext_size
= size
;
332 ctx
->sg_plaintext_num_elem
= num_elem
;
336 static int memcopy_from_iter(struct sock
*sk
, struct iov_iter
*from
,
339 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
340 struct tls_sw_context
*ctx
= tls_sw_ctx(tls_ctx
);
341 struct scatterlist
*sg
= ctx
->sg_plaintext_data
;
344 for (i
= tls_ctx
->pending_open_record_frags
;
345 i
< ctx
->sg_plaintext_num_elem
; ++i
) {
348 page_address(sg_page(&sg
[i
])) + sg
[i
].offset
,
349 copy
, from
) != copy
) {
355 ++tls_ctx
->pending_open_record_frags
;
365 int tls_sw_sendmsg(struct sock
*sk
, struct msghdr
*msg
, size_t size
)
367 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
368 struct tls_sw_context
*ctx
= tls_sw_ctx(tls_ctx
);
371 long timeo
= sock_sndtimeo(sk
, msg
->msg_flags
& MSG_DONTWAIT
);
372 bool eor
= !(msg
->msg_flags
& MSG_MORE
);
373 size_t try_to_copy
, copied
= 0;
374 unsigned char record_type
= TLS_RECORD_TYPE_DATA
;
379 if (msg
->msg_flags
& ~(MSG_MORE
| MSG_DONTWAIT
| MSG_NOSIGNAL
))
384 if (tls_complete_pending_work(sk
, tls_ctx
, msg
->msg_flags
, &timeo
))
387 if (unlikely(msg
->msg_controllen
)) {
388 ret
= tls_proccess_cmsg(sk
, msg
, &record_type
);
393 while (msg_data_left(msg
)) {
399 orig_size
= ctx
->sg_plaintext_size
;
401 try_to_copy
= msg_data_left(msg
);
402 record_room
= TLS_MAX_PAYLOAD_SIZE
- ctx
->sg_plaintext_size
;
403 if (try_to_copy
>= record_room
) {
404 try_to_copy
= record_room
;
408 required_size
= ctx
->sg_plaintext_size
+ try_to_copy
+
409 tls_ctx
->overhead_size
;
411 if (!sk_stream_memory_free(sk
))
412 goto wait_for_sndbuf
;
414 ret
= alloc_encrypted_sg(sk
, required_size
);
417 goto wait_for_memory
;
419 /* Adjust try_to_copy according to the amount that was
420 * actually allocated. The difference is due
421 * to max sg elements limit
423 try_to_copy
-= required_size
- ctx
->sg_encrypted_size
;
427 if (full_record
|| eor
) {
428 ret
= zerocopy_from_iter(sk
, &msg
->msg_iter
,
431 goto fallback_to_reg_send
;
433 copied
+= try_to_copy
;
434 ret
= tls_push_record(sk
, msg
->msg_flags
, record_type
);
440 copied
-= try_to_copy
;
441 fallback_to_reg_send
:
442 iov_iter_revert(&msg
->msg_iter
,
443 ctx
->sg_plaintext_size
- orig_size
);
444 trim_sg(sk
, ctx
->sg_plaintext_data
,
445 &ctx
->sg_plaintext_num_elem
,
446 &ctx
->sg_plaintext_size
,
450 required_size
= ctx
->sg_plaintext_size
+ try_to_copy
;
452 ret
= alloc_plaintext_sg(sk
, required_size
);
455 goto wait_for_memory
;
457 /* Adjust try_to_copy according to the amount that was
458 * actually allocated. The difference is due
459 * to max sg elements limit
461 try_to_copy
-= required_size
- ctx
->sg_plaintext_size
;
464 trim_sg(sk
, ctx
->sg_encrypted_data
,
465 &ctx
->sg_encrypted_num_elem
,
466 &ctx
->sg_encrypted_size
,
467 ctx
->sg_plaintext_size
+
468 tls_ctx
->overhead_size
);
471 ret
= memcopy_from_iter(sk
, &msg
->msg_iter
, try_to_copy
);
475 copied
+= try_to_copy
;
476 if (full_record
|| eor
) {
478 ret
= tls_push_record(sk
, msg
->msg_flags
, record_type
);
481 goto wait_for_memory
;
490 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
492 ret
= sk_stream_wait_memory(sk
, &timeo
);
495 trim_both_sgl(sk
, orig_size
);
499 if (tls_is_pending_closed_record(tls_ctx
))
502 if (ctx
->sg_encrypted_size
< required_size
)
503 goto alloc_encrypted
;
505 goto alloc_plaintext
;
509 ret
= sk_stream_error(sk
, msg
->msg_flags
, ret
);
512 return copied
? copied
: ret
;
515 int tls_sw_sendpage(struct sock
*sk
, struct page
*page
,
516 int offset
, size_t size
, int flags
)
518 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
519 struct tls_sw_context
*ctx
= tls_sw_ctx(tls_ctx
);
521 long timeo
= sock_sndtimeo(sk
, flags
& MSG_DONTWAIT
);
523 size_t orig_size
= size
;
524 unsigned char record_type
= TLS_RECORD_TYPE_DATA
;
525 struct scatterlist
*sg
;
529 if (flags
& ~(MSG_MORE
| MSG_DONTWAIT
| MSG_NOSIGNAL
|
530 MSG_SENDPAGE_NOTLAST
))
533 /* No MSG_EOR from splice, only look at MSG_MORE */
534 eor
= !(flags
& (MSG_MORE
| MSG_SENDPAGE_NOTLAST
));
538 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE
, sk
);
540 if (tls_complete_pending_work(sk
, tls_ctx
, flags
, &timeo
))
543 /* Call the sk_stream functions to manage the sndbuf mem. */
545 size_t copy
, required_size
;
553 record_room
= TLS_MAX_PAYLOAD_SIZE
- ctx
->sg_plaintext_size
;
555 if (copy
>= record_room
) {
559 required_size
= ctx
->sg_plaintext_size
+ copy
+
560 tls_ctx
->overhead_size
;
562 if (!sk_stream_memory_free(sk
))
563 goto wait_for_sndbuf
;
565 ret
= alloc_encrypted_sg(sk
, required_size
);
568 goto wait_for_memory
;
570 /* Adjust copy according to the amount that was
571 * actually allocated. The difference is due
572 * to max sg elements limit
574 copy
-= required_size
- ctx
->sg_plaintext_size
;
579 sg
= ctx
->sg_plaintext_data
+ ctx
->sg_plaintext_num_elem
;
580 sg_set_page(sg
, page
, copy
, offset
);
583 ctx
->sg_plaintext_num_elem
++;
585 sk_mem_charge(sk
, copy
);
588 ctx
->sg_plaintext_size
+= copy
;
589 tls_ctx
->pending_open_record_frags
= ctx
->sg_plaintext_num_elem
;
591 if (full_record
|| eor
||
592 ctx
->sg_plaintext_num_elem
==
593 ARRAY_SIZE(ctx
->sg_plaintext_data
)) {
595 ret
= tls_push_record(sk
, flags
, record_type
);
598 goto wait_for_memory
;
605 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
607 ret
= sk_stream_wait_memory(sk
, &timeo
);
609 trim_both_sgl(sk
, ctx
->sg_plaintext_size
);
613 if (tls_is_pending_closed_record(tls_ctx
))
620 if (orig_size
> size
)
621 ret
= orig_size
- size
;
623 ret
= sk_stream_error(sk
, flags
, ret
);
629 void tls_sw_free_tx_resources(struct sock
*sk
)
631 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
632 struct tls_sw_context
*ctx
= tls_sw_ctx(tls_ctx
);
635 crypto_free_aead(ctx
->aead_send
);
637 tls_free_both_sg(sk
);
643 int tls_set_sw_offload(struct sock
*sk
, struct tls_context
*ctx
)
645 char keyval
[TLS_CIPHER_AES_GCM_128_KEY_SIZE
];
646 struct tls_crypto_info
*crypto_info
;
647 struct tls12_crypto_info_aes_gcm_128
*gcm_128_info
;
648 struct tls_sw_context
*sw_ctx
;
649 u16 nonce_size
, tag_size
, iv_size
, rec_seq_size
;
663 sw_ctx
= kzalloc(sizeof(*sw_ctx
), GFP_KERNEL
);
669 ctx
->priv_ctx
= (struct tls_offload_context
*)sw_ctx
;
671 crypto_info
= &ctx
->crypto_send
;
672 switch (crypto_info
->cipher_type
) {
673 case TLS_CIPHER_AES_GCM_128
: {
674 nonce_size
= TLS_CIPHER_AES_GCM_128_IV_SIZE
;
675 tag_size
= TLS_CIPHER_AES_GCM_128_TAG_SIZE
;
676 iv_size
= TLS_CIPHER_AES_GCM_128_IV_SIZE
;
677 iv
= ((struct tls12_crypto_info_aes_gcm_128
*)crypto_info
)->iv
;
678 rec_seq_size
= TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE
;
680 ((struct tls12_crypto_info_aes_gcm_128
*)crypto_info
)->rec_seq
;
682 (struct tls12_crypto_info_aes_gcm_128
*)crypto_info
;
690 ctx
->prepend_size
= TLS_HEADER_SIZE
+ nonce_size
;
691 ctx
->tag_size
= tag_size
;
692 ctx
->overhead_size
= ctx
->prepend_size
+ ctx
->tag_size
;
693 ctx
->iv_size
= iv_size
;
694 ctx
->iv
= kmalloc(iv_size
+ TLS_CIPHER_AES_GCM_128_SALT_SIZE
, GFP_KERNEL
);
699 memcpy(ctx
->iv
, gcm_128_info
->salt
, TLS_CIPHER_AES_GCM_128_SALT_SIZE
);
700 memcpy(ctx
->iv
+ TLS_CIPHER_AES_GCM_128_SALT_SIZE
, iv
, iv_size
);
701 ctx
->rec_seq_size
= rec_seq_size
;
702 ctx
->rec_seq
= kmalloc(rec_seq_size
, GFP_KERNEL
);
707 memcpy(ctx
->rec_seq
, rec_seq
, rec_seq_size
);
709 sg_init_table(sw_ctx
->sg_encrypted_data
,
710 ARRAY_SIZE(sw_ctx
->sg_encrypted_data
));
711 sg_init_table(sw_ctx
->sg_plaintext_data
,
712 ARRAY_SIZE(sw_ctx
->sg_plaintext_data
));
714 sg_init_table(sw_ctx
->sg_aead_in
, 2);
715 sg_set_buf(&sw_ctx
->sg_aead_in
[0], sw_ctx
->aad_space
,
716 sizeof(sw_ctx
->aad_space
));
717 sg_unmark_end(&sw_ctx
->sg_aead_in
[1]);
718 sg_chain(sw_ctx
->sg_aead_in
, 2, sw_ctx
->sg_plaintext_data
);
719 sg_init_table(sw_ctx
->sg_aead_out
, 2);
720 sg_set_buf(&sw_ctx
->sg_aead_out
[0], sw_ctx
->aad_space
,
721 sizeof(sw_ctx
->aad_space
));
722 sg_unmark_end(&sw_ctx
->sg_aead_out
[1]);
723 sg_chain(sw_ctx
->sg_aead_out
, 2, sw_ctx
->sg_encrypted_data
);
725 if (!sw_ctx
->aead_send
) {
726 sw_ctx
->aead_send
= crypto_alloc_aead("gcm(aes)", 0, 0);
727 if (IS_ERR(sw_ctx
->aead_send
)) {
728 rc
= PTR_ERR(sw_ctx
->aead_send
);
729 sw_ctx
->aead_send
= NULL
;
734 ctx
->push_pending_record
= tls_sw_push_pending_record
;
736 memcpy(keyval
, gcm_128_info
->key
, TLS_CIPHER_AES_GCM_128_KEY_SIZE
);
738 rc
= crypto_aead_setkey(sw_ctx
->aead_send
, keyval
,
739 TLS_CIPHER_AES_GCM_128_KEY_SIZE
);
743 rc
= crypto_aead_setauthsize(sw_ctx
->aead_send
, ctx
->tag_size
);
748 crypto_free_aead(sw_ctx
->aead_send
);
749 sw_ctx
->aead_send
= NULL
;
757 kfree(ctx
->priv_ctx
);
758 ctx
->priv_ctx
= NULL
;