1 /* Copyright (c) 2018, Mellanox Technologies All rights reserved.
3 * This software is available to you under a choice of one of two
4 * licenses. You may choose to be licensed under the terms of the GNU
5 * General Public License (GPL) Version 2, available from the file
6 * COPYING in the main directory of this source tree, or the
7 * OpenIB.org BSD license below:
9 * Redistribution and use in source and binary forms, with or
10 * without modification, are permitted provided that the following
13 * - Redistributions of source code must retain the above
14 * copyright notice, this list of conditions and the following
17 * - Redistributions in binary form must reproduce the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer in the documentation and/or other materials
20 * provided with the distribution.
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <crypto/aead.h>
33 #include <linux/highmem.h>
34 #include <linux/module.h>
35 #include <linux/netdevice.h>
37 #include <net/inet_connection_sock.h>
43 /* device_offload_lock is used to synchronize tls_dev_add
44 * against NETDEV_DOWN notifications.
46 static DECLARE_RWSEM(device_offload_lock
);
48 static void tls_device_gc_task(struct work_struct
*work
);
50 static DECLARE_WORK(tls_device_gc_work
, tls_device_gc_task
);
51 static LIST_HEAD(tls_device_gc_list
);
52 static LIST_HEAD(tls_device_list
);
53 static DEFINE_SPINLOCK(tls_device_lock
);
55 static void tls_device_free_ctx(struct tls_context
*ctx
)
57 if (ctx
->tx_conf
== TLS_HW
) {
58 kfree(tls_offload_ctx_tx(ctx
));
59 kfree(ctx
->tx
.rec_seq
);
63 if (ctx
->rx_conf
== TLS_HW
)
64 kfree(tls_offload_ctx_rx(ctx
));
66 tls_ctx_free(NULL
, ctx
);
69 static void tls_device_gc_task(struct work_struct
*work
)
71 struct tls_context
*ctx
, *tmp
;
75 spin_lock_irqsave(&tls_device_lock
, flags
);
76 list_splice_init(&tls_device_gc_list
, &gc_list
);
77 spin_unlock_irqrestore(&tls_device_lock
, flags
);
79 list_for_each_entry_safe(ctx
, tmp
, &gc_list
, list
) {
80 struct net_device
*netdev
= ctx
->netdev
;
82 if (netdev
&& ctx
->tx_conf
== TLS_HW
) {
83 netdev
->tlsdev_ops
->tls_dev_del(netdev
, ctx
,
84 TLS_OFFLOAD_CTX_DIR_TX
);
90 tls_device_free_ctx(ctx
);
94 static void tls_device_queue_ctx_destruction(struct tls_context
*ctx
)
98 spin_lock_irqsave(&tls_device_lock
, flags
);
99 list_move_tail(&ctx
->list
, &tls_device_gc_list
);
101 /* schedule_work inside the spinlock
102 * to make sure tls_device_down waits for that work.
104 schedule_work(&tls_device_gc_work
);
106 spin_unlock_irqrestore(&tls_device_lock
, flags
);
109 /* We assume that the socket is already connected */
110 static struct net_device
*get_netdev_for_sock(struct sock
*sk
)
112 struct dst_entry
*dst
= sk_dst_get(sk
);
113 struct net_device
*netdev
= NULL
;
125 static void destroy_record(struct tls_record_info
*record
)
129 for (i
= 0; i
< record
->num_frags
; i
++)
130 __skb_frag_unref(&record
->frags
[i
]);
134 static void delete_all_records(struct tls_offload_context_tx
*offload_ctx
)
136 struct tls_record_info
*info
, *temp
;
138 list_for_each_entry_safe(info
, temp
, &offload_ctx
->records_list
, list
) {
139 list_del(&info
->list
);
140 destroy_record(info
);
143 offload_ctx
->retransmit_hint
= NULL
;
146 static void tls_icsk_clean_acked(struct sock
*sk
, u32 acked_seq
)
148 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
149 struct tls_record_info
*info
, *temp
;
150 struct tls_offload_context_tx
*ctx
;
151 u64 deleted_records
= 0;
157 ctx
= tls_offload_ctx_tx(tls_ctx
);
159 spin_lock_irqsave(&ctx
->lock
, flags
);
160 info
= ctx
->retransmit_hint
;
161 if (info
&& !before(acked_seq
, info
->end_seq
))
162 ctx
->retransmit_hint
= NULL
;
164 list_for_each_entry_safe(info
, temp
, &ctx
->records_list
, list
) {
165 if (before(acked_seq
, info
->end_seq
))
167 list_del(&info
->list
);
169 destroy_record(info
);
173 ctx
->unacked_record_sn
+= deleted_records
;
174 spin_unlock_irqrestore(&ctx
->lock
, flags
);
177 /* At this point, there should be no references on this
178 * socket and no in-flight SKBs associated with this
179 * socket, so it is safe to free all the resources.
181 void tls_device_sk_destruct(struct sock
*sk
)
183 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
184 struct tls_offload_context_tx
*ctx
= tls_offload_ctx_tx(tls_ctx
);
186 tls_ctx
->sk_destruct(sk
);
188 if (tls_ctx
->tx_conf
== TLS_HW
) {
189 if (ctx
->open_record
)
190 destroy_record(ctx
->open_record
);
191 delete_all_records(ctx
);
192 crypto_free_aead(ctx
->aead_send
);
193 clean_acked_data_disable(inet_csk(sk
));
196 if (refcount_dec_and_test(&tls_ctx
->refcount
))
197 tls_device_queue_ctx_destruction(tls_ctx
);
199 EXPORT_SYMBOL_GPL(tls_device_sk_destruct
);
201 void tls_device_free_resources_tx(struct sock
*sk
)
203 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
205 tls_free_partial_record(sk
, tls_ctx
);
208 void tls_offload_tx_resync_request(struct sock
*sk
, u32 got_seq
, u32 exp_seq
)
210 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
212 trace_tls_device_tx_resync_req(sk
, got_seq
, exp_seq
);
213 WARN_ON(test_and_set_bit(TLS_TX_SYNC_SCHED
, &tls_ctx
->flags
));
215 EXPORT_SYMBOL_GPL(tls_offload_tx_resync_request
);
217 static void tls_device_resync_tx(struct sock
*sk
, struct tls_context
*tls_ctx
,
220 struct net_device
*netdev
;
225 skb
= tcp_write_queue_tail(sk
);
227 TCP_SKB_CB(skb
)->eor
= 1;
229 rcd_sn
= tls_ctx
->tx
.rec_seq
;
231 trace_tls_device_tx_resync_send(sk
, seq
, rcd_sn
);
232 down_read(&device_offload_lock
);
233 netdev
= tls_ctx
->netdev
;
235 err
= netdev
->tlsdev_ops
->tls_dev_resync(netdev
, sk
, seq
,
237 TLS_OFFLOAD_CTX_DIR_TX
);
238 up_read(&device_offload_lock
);
242 clear_bit_unlock(TLS_TX_SYNC_SCHED
, &tls_ctx
->flags
);
245 static void tls_append_frag(struct tls_record_info
*record
,
246 struct page_frag
*pfrag
,
251 frag
= &record
->frags
[record
->num_frags
- 1];
252 if (skb_frag_page(frag
) == pfrag
->page
&&
253 skb_frag_off(frag
) + skb_frag_size(frag
) == pfrag
->offset
) {
254 skb_frag_size_add(frag
, size
);
257 __skb_frag_set_page(frag
, pfrag
->page
);
258 skb_frag_off_set(frag
, pfrag
->offset
);
259 skb_frag_size_set(frag
, size
);
261 get_page(pfrag
->page
);
264 pfrag
->offset
+= size
;
268 static int tls_push_record(struct sock
*sk
,
269 struct tls_context
*ctx
,
270 struct tls_offload_context_tx
*offload_ctx
,
271 struct tls_record_info
*record
,
274 struct tls_prot_info
*prot
= &ctx
->prot_info
;
275 struct tcp_sock
*tp
= tcp_sk(sk
);
279 record
->end_seq
= tp
->write_seq
+ record
->len
;
280 list_add_tail_rcu(&record
->list
, &offload_ctx
->records_list
);
281 offload_ctx
->open_record
= NULL
;
283 if (test_bit(TLS_TX_SYNC_SCHED
, &ctx
->flags
))
284 tls_device_resync_tx(sk
, ctx
, tp
->write_seq
);
286 tls_advance_record_sn(sk
, prot
, &ctx
->tx
);
288 for (i
= 0; i
< record
->num_frags
; i
++) {
289 frag
= &record
->frags
[i
];
290 sg_unmark_end(&offload_ctx
->sg_tx_data
[i
]);
291 sg_set_page(&offload_ctx
->sg_tx_data
[i
], skb_frag_page(frag
),
292 skb_frag_size(frag
), skb_frag_off(frag
));
293 sk_mem_charge(sk
, skb_frag_size(frag
));
294 get_page(skb_frag_page(frag
));
296 sg_mark_end(&offload_ctx
->sg_tx_data
[record
->num_frags
- 1]);
298 /* all ready, send */
299 return tls_push_sg(sk
, ctx
, offload_ctx
->sg_tx_data
, 0, flags
);
302 static int tls_device_record_close(struct sock
*sk
,
303 struct tls_context
*ctx
,
304 struct tls_record_info
*record
,
305 struct page_frag
*pfrag
,
306 unsigned char record_type
)
308 struct tls_prot_info
*prot
= &ctx
->prot_info
;
312 * device will fill in the tag, we just need to append a placeholder
313 * use socket memory to improve coalescing (re-using a single buffer
314 * increases frag count)
315 * if we can't allocate memory now, steal some back from data
317 if (likely(skb_page_frag_refill(prot
->tag_size
, pfrag
,
318 sk
->sk_allocation
))) {
320 tls_append_frag(record
, pfrag
, prot
->tag_size
);
322 ret
= prot
->tag_size
;
323 if (record
->len
<= prot
->overhead_size
)
328 tls_fill_prepend(ctx
, skb_frag_address(&record
->frags
[0]),
329 record
->len
- prot
->overhead_size
,
330 record_type
, prot
->version
);
334 static int tls_create_new_record(struct tls_offload_context_tx
*offload_ctx
,
335 struct page_frag
*pfrag
,
338 struct tls_record_info
*record
;
341 record
= kmalloc(sizeof(*record
), GFP_KERNEL
);
345 frag
= &record
->frags
[0];
346 __skb_frag_set_page(frag
, pfrag
->page
);
347 skb_frag_off_set(frag
, pfrag
->offset
);
348 skb_frag_size_set(frag
, prepend_size
);
350 get_page(pfrag
->page
);
351 pfrag
->offset
+= prepend_size
;
353 record
->num_frags
= 1;
354 record
->len
= prepend_size
;
355 offload_ctx
->open_record
= record
;
359 static int tls_do_allocation(struct sock
*sk
,
360 struct tls_offload_context_tx
*offload_ctx
,
361 struct page_frag
*pfrag
,
366 if (!offload_ctx
->open_record
) {
367 if (unlikely(!skb_page_frag_refill(prepend_size
, pfrag
,
368 sk
->sk_allocation
))) {
369 READ_ONCE(sk
->sk_prot
)->enter_memory_pressure(sk
);
370 sk_stream_moderate_sndbuf(sk
);
374 ret
= tls_create_new_record(offload_ctx
, pfrag
, prepend_size
);
378 if (pfrag
->size
> pfrag
->offset
)
382 if (!sk_page_frag_refill(sk
, pfrag
))
388 static int tls_device_copy_data(void *addr
, size_t bytes
, struct iov_iter
*i
)
390 size_t pre_copy
, nocache
;
392 pre_copy
= ~((unsigned long)addr
- 1) & (SMP_CACHE_BYTES
- 1);
394 pre_copy
= min(pre_copy
, bytes
);
395 if (copy_from_iter(addr
, pre_copy
, i
) != pre_copy
)
401 nocache
= round_down(bytes
, SMP_CACHE_BYTES
);
402 if (copy_from_iter_nocache(addr
, nocache
, i
) != nocache
)
407 if (bytes
&& copy_from_iter(addr
, bytes
, i
) != bytes
)
413 static int tls_push_data(struct sock
*sk
,
414 struct iov_iter
*msg_iter
,
415 size_t size
, int flags
,
416 unsigned char record_type
)
418 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
419 struct tls_prot_info
*prot
= &tls_ctx
->prot_info
;
420 struct tls_offload_context_tx
*ctx
= tls_offload_ctx_tx(tls_ctx
);
421 int more
= flags
& (MSG_SENDPAGE_NOTLAST
| MSG_MORE
);
422 struct tls_record_info
*record
= ctx
->open_record
;
423 int tls_push_record_flags
;
424 struct page_frag
*pfrag
;
425 size_t orig_size
= size
;
426 u32 max_open_record_len
;
432 ~(MSG_MORE
| MSG_DONTWAIT
| MSG_NOSIGNAL
| MSG_SENDPAGE_NOTLAST
))
435 if (unlikely(sk
->sk_err
))
438 flags
|= MSG_SENDPAGE_DECRYPTED
;
439 tls_push_record_flags
= flags
| MSG_SENDPAGE_NOTLAST
;
441 timeo
= sock_sndtimeo(sk
, flags
& MSG_DONTWAIT
);
442 if (tls_is_partially_sent_record(tls_ctx
)) {
443 rc
= tls_push_partial_record(sk
, tls_ctx
, flags
);
448 pfrag
= sk_page_frag(sk
);
450 /* TLS_HEADER_SIZE is not counted as part of the TLS record, and
451 * we need to leave room for an authentication tag.
453 max_open_record_len
= TLS_MAX_PAYLOAD_SIZE
+
456 rc
= tls_do_allocation(sk
, ctx
, pfrag
, prot
->prepend_size
);
458 rc
= sk_stream_wait_memory(sk
, &timeo
);
462 record
= ctx
->open_record
;
466 if (record_type
!= TLS_RECORD_TYPE_DATA
) {
467 /* avoid sending partial
468 * record with type !=
472 destroy_record(record
);
473 ctx
->open_record
= NULL
;
474 } else if (record
->len
> prot
->prepend_size
) {
481 record
= ctx
->open_record
;
482 copy
= min_t(size_t, size
, (pfrag
->size
- pfrag
->offset
));
483 copy
= min_t(size_t, copy
, (max_open_record_len
- record
->len
));
485 rc
= tls_device_copy_data(page_address(pfrag
->page
) +
486 pfrag
->offset
, copy
, msg_iter
);
489 tls_append_frag(record
, pfrag
, copy
);
494 tls_push_record_flags
= flags
;
496 tls_ctx
->pending_open_record_frags
=
504 if (done
|| record
->len
>= max_open_record_len
||
505 (record
->num_frags
>= MAX_SKB_FRAGS
- 1)) {
506 rc
= tls_device_record_close(sk
, tls_ctx
, record
,
513 destroy_record(record
);
514 ctx
->open_record
= NULL
;
519 rc
= tls_push_record(sk
,
523 tls_push_record_flags
);
529 if (orig_size
- size
> 0)
530 rc
= orig_size
- size
;
535 int tls_device_sendmsg(struct sock
*sk
, struct msghdr
*msg
, size_t size
)
537 unsigned char record_type
= TLS_RECORD_TYPE_DATA
;
538 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
541 mutex_lock(&tls_ctx
->tx_lock
);
544 if (unlikely(msg
->msg_controllen
)) {
545 rc
= tls_proccess_cmsg(sk
, msg
, &record_type
);
550 rc
= tls_push_data(sk
, &msg
->msg_iter
, size
,
551 msg
->msg_flags
, record_type
);
555 mutex_unlock(&tls_ctx
->tx_lock
);
559 int tls_device_sendpage(struct sock
*sk
, struct page
*page
,
560 int offset
, size_t size
, int flags
)
562 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
563 struct iov_iter msg_iter
;
564 char *kaddr
= kmap(page
);
568 if (flags
& MSG_SENDPAGE_NOTLAST
)
571 mutex_lock(&tls_ctx
->tx_lock
);
574 if (flags
& MSG_OOB
) {
579 iov
.iov_base
= kaddr
+ offset
;
581 iov_iter_kvec(&msg_iter
, WRITE
, &iov
, 1, size
);
582 rc
= tls_push_data(sk
, &msg_iter
, size
,
583 flags
, TLS_RECORD_TYPE_DATA
);
588 mutex_unlock(&tls_ctx
->tx_lock
);
592 struct tls_record_info
*tls_get_record(struct tls_offload_context_tx
*context
,
593 u32 seq
, u64
*p_record_sn
)
595 u64 record_sn
= context
->hint_record_sn
;
596 struct tls_record_info
*info
, *last
;
598 info
= context
->retransmit_hint
;
600 before(seq
, info
->end_seq
- info
->len
)) {
601 /* if retransmit_hint is irrelevant start
602 * from the beggining of the list
604 info
= list_first_entry_or_null(&context
->records_list
,
605 struct tls_record_info
, list
);
608 /* send the start_marker record if seq number is before the
609 * tls offload start marker sequence number. This record is
610 * required to handle TCP packets which are before TLS offload
612 * And if it's not start marker, look if this seq number
613 * belongs to the list.
615 if (likely(!tls_record_is_start_marker(info
))) {
616 /* we have the first record, get the last record to see
617 * if this seq number belongs to the list.
619 last
= list_last_entry(&context
->records_list
,
620 struct tls_record_info
, list
);
622 if (!between(seq
, tls_record_start_seq(info
),
626 record_sn
= context
->unacked_record_sn
;
629 /* We just need the _rcu for the READ_ONCE() */
631 list_for_each_entry_from_rcu(info
, &context
->records_list
, list
) {
632 if (before(seq
, info
->end_seq
)) {
633 if (!context
->retransmit_hint
||
635 context
->retransmit_hint
->end_seq
)) {
636 context
->hint_record_sn
= record_sn
;
637 context
->retransmit_hint
= info
;
639 *p_record_sn
= record_sn
;
640 goto exit_rcu_unlock
;
650 EXPORT_SYMBOL(tls_get_record
);
652 static int tls_device_push_pending_record(struct sock
*sk
, int flags
)
654 struct iov_iter msg_iter
;
656 iov_iter_kvec(&msg_iter
, WRITE
, NULL
, 0, 0);
657 return tls_push_data(sk
, &msg_iter
, 0, flags
, TLS_RECORD_TYPE_DATA
);
660 void tls_device_write_space(struct sock
*sk
, struct tls_context
*ctx
)
662 if (tls_is_partially_sent_record(ctx
)) {
663 gfp_t sk_allocation
= sk
->sk_allocation
;
665 WARN_ON_ONCE(sk
->sk_write_pending
);
667 sk
->sk_allocation
= GFP_ATOMIC
;
668 tls_push_partial_record(sk
, ctx
,
669 MSG_DONTWAIT
| MSG_NOSIGNAL
|
670 MSG_SENDPAGE_DECRYPTED
);
671 sk
->sk_allocation
= sk_allocation
;
675 static void tls_device_resync_rx(struct tls_context
*tls_ctx
,
676 struct sock
*sk
, u32 seq
, u8
*rcd_sn
)
678 struct tls_offload_context_rx
*rx_ctx
= tls_offload_ctx_rx(tls_ctx
);
679 struct net_device
*netdev
;
681 if (WARN_ON(test_and_set_bit(TLS_RX_SYNC_RUNNING
, &tls_ctx
->flags
)))
684 trace_tls_device_rx_resync_send(sk
, seq
, rcd_sn
, rx_ctx
->resync_type
);
685 netdev
= READ_ONCE(tls_ctx
->netdev
);
687 netdev
->tlsdev_ops
->tls_dev_resync(netdev
, sk
, seq
, rcd_sn
,
688 TLS_OFFLOAD_CTX_DIR_RX
);
689 clear_bit_unlock(TLS_RX_SYNC_RUNNING
, &tls_ctx
->flags
);
690 TLS_INC_STATS(sock_net(sk
), LINUX_MIB_TLSRXDEVICERESYNC
);
693 void tls_device_rx_resync_new_rec(struct sock
*sk
, u32 rcd_len
, u32 seq
)
695 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
696 struct tls_offload_context_rx
*rx_ctx
;
697 u8 rcd_sn
[TLS_MAX_REC_SEQ_SIZE
];
698 u32 sock_data
, is_req_pending
;
699 struct tls_prot_info
*prot
;
703 if (tls_ctx
->rx_conf
!= TLS_HW
)
706 prot
= &tls_ctx
->prot_info
;
707 rx_ctx
= tls_offload_ctx_rx(tls_ctx
);
708 memcpy(rcd_sn
, tls_ctx
->rx
.rec_seq
, prot
->rec_seq_size
);
710 switch (rx_ctx
->resync_type
) {
711 case TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ
:
712 resync_req
= atomic64_read(&rx_ctx
->resync_req
);
713 req_seq
= resync_req
>> 32;
714 seq
+= TLS_HEADER_SIZE
- 1;
715 is_req_pending
= resync_req
;
717 if (likely(!is_req_pending
) || req_seq
!= seq
||
718 !atomic64_try_cmpxchg(&rx_ctx
->resync_req
, &resync_req
, 0))
721 case TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT
:
722 if (likely(!rx_ctx
->resync_nh_do_now
))
725 /* head of next rec is already in, note that the sock_inq will
726 * include the currently parsed message when called from parser
728 sock_data
= tcp_inq(sk
);
729 if (sock_data
> rcd_len
) {
730 trace_tls_device_rx_resync_nh_delay(sk
, sock_data
,
735 rx_ctx
->resync_nh_do_now
= 0;
737 tls_bigint_increment(rcd_sn
, prot
->rec_seq_size
);
741 tls_device_resync_rx(tls_ctx
, sk
, seq
, rcd_sn
);
744 static void tls_device_core_ctrl_rx_resync(struct tls_context
*tls_ctx
,
745 struct tls_offload_context_rx
*ctx
,
746 struct sock
*sk
, struct sk_buff
*skb
)
748 struct strp_msg
*rxm
;
750 /* device will request resyncs by itself based on stream scan */
751 if (ctx
->resync_type
!= TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT
)
753 /* already scheduled */
754 if (ctx
->resync_nh_do_now
)
756 /* seen decrypted fragments since last fully-failed record */
757 if (ctx
->resync_nh_reset
) {
758 ctx
->resync_nh_reset
= 0;
759 ctx
->resync_nh
.decrypted_failed
= 1;
760 ctx
->resync_nh
.decrypted_tgt
= TLS_DEVICE_RESYNC_NH_START_IVAL
;
764 if (++ctx
->resync_nh
.decrypted_failed
<= ctx
->resync_nh
.decrypted_tgt
)
767 /* doing resync, bump the next target in case it fails */
768 if (ctx
->resync_nh
.decrypted_tgt
< TLS_DEVICE_RESYNC_NH_MAX_IVAL
)
769 ctx
->resync_nh
.decrypted_tgt
*= 2;
771 ctx
->resync_nh
.decrypted_tgt
+= TLS_DEVICE_RESYNC_NH_MAX_IVAL
;
775 /* head of next rec is already in, parser will sync for us */
776 if (tcp_inq(sk
) > rxm
->full_len
) {
777 trace_tls_device_rx_resync_nh_schedule(sk
);
778 ctx
->resync_nh_do_now
= 1;
780 struct tls_prot_info
*prot
= &tls_ctx
->prot_info
;
781 u8 rcd_sn
[TLS_MAX_REC_SEQ_SIZE
];
783 memcpy(rcd_sn
, tls_ctx
->rx
.rec_seq
, prot
->rec_seq_size
);
784 tls_bigint_increment(rcd_sn
, prot
->rec_seq_size
);
786 tls_device_resync_rx(tls_ctx
, sk
, tcp_sk(sk
)->copied_seq
,
791 static int tls_device_reencrypt(struct sock
*sk
, struct sk_buff
*skb
)
793 struct strp_msg
*rxm
= strp_msg(skb
);
794 int err
= 0, offset
= rxm
->offset
, copy
, nsg
, data_len
, pos
;
795 struct sk_buff
*skb_iter
, *unused
;
796 struct scatterlist sg
[1];
797 char *orig_buf
, *buf
;
799 orig_buf
= kmalloc(rxm
->full_len
+ TLS_HEADER_SIZE
+
800 TLS_CIPHER_AES_GCM_128_IV_SIZE
, sk
->sk_allocation
);
805 nsg
= skb_cow_data(skb
, 0, &unused
);
806 if (unlikely(nsg
< 0)) {
811 sg_init_table(sg
, 1);
812 sg_set_buf(&sg
[0], buf
,
813 rxm
->full_len
+ TLS_HEADER_SIZE
+
814 TLS_CIPHER_AES_GCM_128_IV_SIZE
);
815 err
= skb_copy_bits(skb
, offset
, buf
,
816 TLS_HEADER_SIZE
+ TLS_CIPHER_AES_GCM_128_IV_SIZE
);
820 /* We are interested only in the decrypted data not the auth */
821 err
= decrypt_skb(sk
, skb
, sg
);
827 data_len
= rxm
->full_len
- TLS_CIPHER_AES_GCM_128_TAG_SIZE
;
829 if (skb_pagelen(skb
) > offset
) {
830 copy
= min_t(int, skb_pagelen(skb
) - offset
, data_len
);
832 if (skb
->decrypted
) {
833 err
= skb_store_bits(skb
, offset
, buf
, copy
);
842 pos
= skb_pagelen(skb
);
843 skb_walk_frags(skb
, skb_iter
) {
846 /* Practically all frags must belong to msg if reencrypt
847 * is needed with current strparser and coalescing logic,
848 * but strparser may "get optimized", so let's be safe.
850 if (pos
+ skb_iter
->len
<= offset
)
852 if (pos
>= data_len
+ rxm
->offset
)
855 frag_pos
= offset
- pos
;
856 copy
= min_t(int, skb_iter
->len
- frag_pos
,
857 data_len
+ rxm
->offset
- offset
);
859 if (skb_iter
->decrypted
) {
860 err
= skb_store_bits(skb_iter
, frag_pos
, buf
, copy
);
868 pos
+= skb_iter
->len
;
876 int tls_device_decrypted(struct sock
*sk
, struct tls_context
*tls_ctx
,
877 struct sk_buff
*skb
, struct strp_msg
*rxm
)
879 struct tls_offload_context_rx
*ctx
= tls_offload_ctx_rx(tls_ctx
);
880 int is_decrypted
= skb
->decrypted
;
881 int is_encrypted
= !is_decrypted
;
882 struct sk_buff
*skb_iter
;
884 /* Check if all the data is decrypted already */
885 skb_walk_frags(skb
, skb_iter
) {
886 is_decrypted
&= skb_iter
->decrypted
;
887 is_encrypted
&= !skb_iter
->decrypted
;
890 trace_tls_device_decrypted(sk
, tcp_sk(sk
)->copied_seq
- rxm
->full_len
,
891 tls_ctx
->rx
.rec_seq
, rxm
->full_len
,
892 is_encrypted
, is_decrypted
);
894 ctx
->sw
.decrypted
|= is_decrypted
;
896 /* Return immediately if the record is either entirely plaintext or
897 * entirely ciphertext. Otherwise handle reencrypt partially decrypted
901 ctx
->resync_nh_reset
= 1;
905 tls_device_core_ctrl_rx_resync(tls_ctx
, ctx
, sk
, skb
);
909 ctx
->resync_nh_reset
= 1;
910 return tls_device_reencrypt(sk
, skb
);
913 static void tls_device_attach(struct tls_context
*ctx
, struct sock
*sk
,
914 struct net_device
*netdev
)
916 if (sk
->sk_destruct
!= tls_device_sk_destruct
) {
917 refcount_set(&ctx
->refcount
, 1);
919 ctx
->netdev
= netdev
;
920 spin_lock_irq(&tls_device_lock
);
921 list_add_tail(&ctx
->list
, &tls_device_list
);
922 spin_unlock_irq(&tls_device_lock
);
924 ctx
->sk_destruct
= sk
->sk_destruct
;
925 smp_store_release(&sk
->sk_destruct
, tls_device_sk_destruct
);
929 int tls_set_device_offload(struct sock
*sk
, struct tls_context
*ctx
)
931 u16 nonce_size
, tag_size
, iv_size
, rec_seq_size
;
932 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
933 struct tls_prot_info
*prot
= &tls_ctx
->prot_info
;
934 struct tls_record_info
*start_marker_record
;
935 struct tls_offload_context_tx
*offload_ctx
;
936 struct tls_crypto_info
*crypto_info
;
937 struct net_device
*netdev
;
946 if (ctx
->priv_ctx_tx
)
949 start_marker_record
= kmalloc(sizeof(*start_marker_record
), GFP_KERNEL
);
950 if (!start_marker_record
)
953 offload_ctx
= kzalloc(TLS_OFFLOAD_CONTEXT_SIZE_TX
, GFP_KERNEL
);
956 goto free_marker_record
;
959 crypto_info
= &ctx
->crypto_send
.info
;
960 if (crypto_info
->version
!= TLS_1_2_VERSION
) {
962 goto free_offload_ctx
;
965 switch (crypto_info
->cipher_type
) {
966 case TLS_CIPHER_AES_GCM_128
:
967 nonce_size
= TLS_CIPHER_AES_GCM_128_IV_SIZE
;
968 tag_size
= TLS_CIPHER_AES_GCM_128_TAG_SIZE
;
969 iv_size
= TLS_CIPHER_AES_GCM_128_IV_SIZE
;
970 iv
= ((struct tls12_crypto_info_aes_gcm_128
*)crypto_info
)->iv
;
971 rec_seq_size
= TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE
;
973 ((struct tls12_crypto_info_aes_gcm_128
*)crypto_info
)->rec_seq
;
977 goto free_offload_ctx
;
980 /* Sanity-check the rec_seq_size for stack allocations */
981 if (rec_seq_size
> TLS_MAX_REC_SEQ_SIZE
) {
983 goto free_offload_ctx
;
986 prot
->version
= crypto_info
->version
;
987 prot
->cipher_type
= crypto_info
->cipher_type
;
988 prot
->prepend_size
= TLS_HEADER_SIZE
+ nonce_size
;
989 prot
->tag_size
= tag_size
;
990 prot
->overhead_size
= prot
->prepend_size
+ prot
->tag_size
;
991 prot
->iv_size
= iv_size
;
992 ctx
->tx
.iv
= kmalloc(iv_size
+ TLS_CIPHER_AES_GCM_128_SALT_SIZE
,
996 goto free_offload_ctx
;
999 memcpy(ctx
->tx
.iv
+ TLS_CIPHER_AES_GCM_128_SALT_SIZE
, iv
, iv_size
);
1001 prot
->rec_seq_size
= rec_seq_size
;
1002 ctx
->tx
.rec_seq
= kmemdup(rec_seq
, rec_seq_size
, GFP_KERNEL
);
1003 if (!ctx
->tx
.rec_seq
) {
1008 rc
= tls_sw_fallback_init(sk
, offload_ctx
, crypto_info
);
1012 /* start at rec_seq - 1 to account for the start marker record */
1013 memcpy(&rcd_sn
, ctx
->tx
.rec_seq
, sizeof(rcd_sn
));
1014 offload_ctx
->unacked_record_sn
= be64_to_cpu(rcd_sn
) - 1;
1016 start_marker_record
->end_seq
= tcp_sk(sk
)->write_seq
;
1017 start_marker_record
->len
= 0;
1018 start_marker_record
->num_frags
= 0;
1020 INIT_LIST_HEAD(&offload_ctx
->records_list
);
1021 list_add_tail(&start_marker_record
->list
, &offload_ctx
->records_list
);
1022 spin_lock_init(&offload_ctx
->lock
);
1023 sg_init_table(offload_ctx
->sg_tx_data
,
1024 ARRAY_SIZE(offload_ctx
->sg_tx_data
));
1026 clean_acked_data_enable(inet_csk(sk
), &tls_icsk_clean_acked
);
1027 ctx
->push_pending_record
= tls_device_push_pending_record
;
1029 /* TLS offload is greatly simplified if we don't send
1030 * SKBs where only part of the payload needs to be encrypted.
1031 * So mark the last skb in the write queue as end of record.
1033 skb
= tcp_write_queue_tail(sk
);
1035 TCP_SKB_CB(skb
)->eor
= 1;
1037 netdev
= get_netdev_for_sock(sk
);
1039 pr_err_ratelimited("%s: netdev not found\n", __func__
);
1044 if (!(netdev
->features
& NETIF_F_HW_TLS_TX
)) {
1046 goto release_netdev
;
1049 /* Avoid offloading if the device is down
1050 * We don't want to offload new flows after
1051 * the NETDEV_DOWN event
1053 * device_offload_lock is taken in tls_devices's NETDEV_DOWN
1054 * handler thus protecting from the device going down before
1055 * ctx was added to tls_device_list.
1057 down_read(&device_offload_lock
);
1058 if (!(netdev
->flags
& IFF_UP
)) {
1063 ctx
->priv_ctx_tx
= offload_ctx
;
1064 rc
= netdev
->tlsdev_ops
->tls_dev_add(netdev
, sk
, TLS_OFFLOAD_CTX_DIR_TX
,
1065 &ctx
->crypto_send
.info
,
1066 tcp_sk(sk
)->write_seq
);
1067 trace_tls_device_offload_set(sk
, TLS_OFFLOAD_CTX_DIR_TX
,
1068 tcp_sk(sk
)->write_seq
, rec_seq
, rc
);
1072 tls_device_attach(ctx
, sk
, netdev
);
1073 up_read(&device_offload_lock
);
1075 /* following this assignment tls_is_sk_tx_device_offloaded
1076 * will return true and the context might be accessed
1077 * by the netdev's xmit function.
1079 smp_store_release(&sk
->sk_validate_xmit_skb
, tls_validate_xmit_skb
);
1085 up_read(&device_offload_lock
);
1089 clean_acked_data_disable(inet_csk(sk
));
1090 crypto_free_aead(offload_ctx
->aead_send
);
1092 kfree(ctx
->tx
.rec_seq
);
1097 ctx
->priv_ctx_tx
= NULL
;
1099 kfree(start_marker_record
);
1103 int tls_set_device_offload_rx(struct sock
*sk
, struct tls_context
*ctx
)
1105 struct tls12_crypto_info_aes_gcm_128
*info
;
1106 struct tls_offload_context_rx
*context
;
1107 struct net_device
*netdev
;
1110 if (ctx
->crypto_recv
.info
.version
!= TLS_1_2_VERSION
)
1113 netdev
= get_netdev_for_sock(sk
);
1115 pr_err_ratelimited("%s: netdev not found\n", __func__
);
1119 if (!(netdev
->features
& NETIF_F_HW_TLS_RX
)) {
1121 goto release_netdev
;
1124 /* Avoid offloading if the device is down
1125 * We don't want to offload new flows after
1126 * the NETDEV_DOWN event
1128 * device_offload_lock is taken in tls_devices's NETDEV_DOWN
1129 * handler thus protecting from the device going down before
1130 * ctx was added to tls_device_list.
1132 down_read(&device_offload_lock
);
1133 if (!(netdev
->flags
& IFF_UP
)) {
1138 context
= kzalloc(TLS_OFFLOAD_CONTEXT_SIZE_RX
, GFP_KERNEL
);
1143 context
->resync_nh_reset
= 1;
1145 ctx
->priv_ctx_rx
= context
;
1146 rc
= tls_set_sw_offload(sk
, ctx
, 0);
1150 rc
= netdev
->tlsdev_ops
->tls_dev_add(netdev
, sk
, TLS_OFFLOAD_CTX_DIR_RX
,
1151 &ctx
->crypto_recv
.info
,
1152 tcp_sk(sk
)->copied_seq
);
1153 info
= (void *)&ctx
->crypto_recv
.info
;
1154 trace_tls_device_offload_set(sk
, TLS_OFFLOAD_CTX_DIR_RX
,
1155 tcp_sk(sk
)->copied_seq
, info
->rec_seq
, rc
);
1157 goto free_sw_resources
;
1159 tls_device_attach(ctx
, sk
, netdev
);
1160 up_read(&device_offload_lock
);
1167 up_read(&device_offload_lock
);
1168 tls_sw_free_resources_rx(sk
);
1169 down_read(&device_offload_lock
);
1171 ctx
->priv_ctx_rx
= NULL
;
1173 up_read(&device_offload_lock
);
1179 void tls_device_offload_cleanup_rx(struct sock
*sk
)
1181 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
1182 struct net_device
*netdev
;
1184 down_read(&device_offload_lock
);
1185 netdev
= tls_ctx
->netdev
;
1189 netdev
->tlsdev_ops
->tls_dev_del(netdev
, tls_ctx
,
1190 TLS_OFFLOAD_CTX_DIR_RX
);
1192 if (tls_ctx
->tx_conf
!= TLS_HW
) {
1194 tls_ctx
->netdev
= NULL
;
1197 up_read(&device_offload_lock
);
1198 tls_sw_release_resources_rx(sk
);
1201 static int tls_device_down(struct net_device
*netdev
)
1203 struct tls_context
*ctx
, *tmp
;
1204 unsigned long flags
;
1207 /* Request a write lock to block new offload attempts */
1208 down_write(&device_offload_lock
);
1210 spin_lock_irqsave(&tls_device_lock
, flags
);
1211 list_for_each_entry_safe(ctx
, tmp
, &tls_device_list
, list
) {
1212 if (ctx
->netdev
!= netdev
||
1213 !refcount_inc_not_zero(&ctx
->refcount
))
1216 list_move(&ctx
->list
, &list
);
1218 spin_unlock_irqrestore(&tls_device_lock
, flags
);
1220 list_for_each_entry_safe(ctx
, tmp
, &list
, list
) {
1221 if (ctx
->tx_conf
== TLS_HW
)
1222 netdev
->tlsdev_ops
->tls_dev_del(netdev
, ctx
,
1223 TLS_OFFLOAD_CTX_DIR_TX
);
1224 if (ctx
->rx_conf
== TLS_HW
)
1225 netdev
->tlsdev_ops
->tls_dev_del(netdev
, ctx
,
1226 TLS_OFFLOAD_CTX_DIR_RX
);
1227 WRITE_ONCE(ctx
->netdev
, NULL
);
1228 smp_mb__before_atomic(); /* pairs with test_and_set_bit() */
1229 while (test_bit(TLS_RX_SYNC_RUNNING
, &ctx
->flags
))
1230 usleep_range(10, 200);
1232 list_del_init(&ctx
->list
);
1234 if (refcount_dec_and_test(&ctx
->refcount
))
1235 tls_device_free_ctx(ctx
);
1238 up_write(&device_offload_lock
);
1240 flush_work(&tls_device_gc_work
);
1245 static int tls_dev_event(struct notifier_block
*this, unsigned long event
,
1248 struct net_device
*dev
= netdev_notifier_info_to_dev(ptr
);
1250 if (!dev
->tlsdev_ops
&&
1251 !(dev
->features
& (NETIF_F_HW_TLS_RX
| NETIF_F_HW_TLS_TX
)))
1255 case NETDEV_REGISTER
:
1256 case NETDEV_FEAT_CHANGE
:
1257 if ((dev
->features
& NETIF_F_HW_TLS_RX
) &&
1258 !dev
->tlsdev_ops
->tls_dev_resync
)
1261 if (dev
->tlsdev_ops
&&
1262 dev
->tlsdev_ops
->tls_dev_add
&&
1263 dev
->tlsdev_ops
->tls_dev_del
)
1268 return tls_device_down(dev
);
1273 static struct notifier_block tls_dev_notifier
= {
1274 .notifier_call
= tls_dev_event
,
1277 void __init
tls_device_init(void)
1279 register_netdevice_notifier(&tls_dev_notifier
);
1282 void __exit
tls_device_cleanup(void)
1284 unregister_netdevice_notifier(&tls_dev_notifier
);
1285 flush_work(&tls_device_gc_work
);
1286 clean_acked_data_flush();