1 /* Copyright (c) 2018, Mellanox Technologies All rights reserved.
3 * This software is available to you under a choice of one of two
4 * licenses. You may choose to be licensed under the terms of the GNU
5 * General Public License (GPL) Version 2, available from the file
6 * COPYING in the main directory of this source tree, or the
7 * OpenIB.org BSD license below:
9 * Redistribution and use in source and binary forms, with or
10 * without modification, are permitted provided that the following
13 * - Redistributions of source code must retain the above
14 * copyright notice, this list of conditions and the following
17 * - Redistributions in binary form must reproduce the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer in the documentation and/or other materials
20 * provided with the distribution.
22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
23 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
25 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
26 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
27 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
28 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 #include <crypto/aead.h>
33 #include <linux/highmem.h>
34 #include <linux/module.h>
35 #include <linux/netdevice.h>
37 #include <net/inet_connection_sock.h>
43 /* device_offload_lock is used to synchronize tls_dev_add
44 * against NETDEV_DOWN notifications.
46 static DECLARE_RWSEM(device_offload_lock
);
48 static void tls_device_gc_task(struct work_struct
*work
);
50 static DECLARE_WORK(tls_device_gc_work
, tls_device_gc_task
);
51 static LIST_HEAD(tls_device_gc_list
);
52 static LIST_HEAD(tls_device_list
);
53 static LIST_HEAD(tls_device_down_list
);
54 static DEFINE_SPINLOCK(tls_device_lock
);
56 static void tls_device_free_ctx(struct tls_context
*ctx
)
58 if (ctx
->tx_conf
== TLS_HW
) {
59 kfree(tls_offload_ctx_tx(ctx
));
60 kfree(ctx
->tx
.rec_seq
);
64 if (ctx
->rx_conf
== TLS_HW
)
65 kfree(tls_offload_ctx_rx(ctx
));
67 tls_ctx_free(NULL
, ctx
);
70 static void tls_device_gc_task(struct work_struct
*work
)
72 struct tls_context
*ctx
, *tmp
;
76 spin_lock_irqsave(&tls_device_lock
, flags
);
77 list_splice_init(&tls_device_gc_list
, &gc_list
);
78 spin_unlock_irqrestore(&tls_device_lock
, flags
);
80 list_for_each_entry_safe(ctx
, tmp
, &gc_list
, list
) {
81 struct net_device
*netdev
= ctx
->netdev
;
83 if (netdev
&& ctx
->tx_conf
== TLS_HW
) {
84 netdev
->tlsdev_ops
->tls_dev_del(netdev
, ctx
,
85 TLS_OFFLOAD_CTX_DIR_TX
);
91 tls_device_free_ctx(ctx
);
95 static void tls_device_queue_ctx_destruction(struct tls_context
*ctx
)
99 spin_lock_irqsave(&tls_device_lock
, flags
);
100 list_move_tail(&ctx
->list
, &tls_device_gc_list
);
102 /* schedule_work inside the spinlock
103 * to make sure tls_device_down waits for that work.
105 schedule_work(&tls_device_gc_work
);
107 spin_unlock_irqrestore(&tls_device_lock
, flags
);
110 /* We assume that the socket is already connected */
111 static struct net_device
*get_netdev_for_sock(struct sock
*sk
)
113 struct dst_entry
*dst
= sk_dst_get(sk
);
114 struct net_device
*netdev
= NULL
;
117 netdev
= netdev_sk_get_lowest_dev(dst
->dev
, sk
);
126 static void destroy_record(struct tls_record_info
*record
)
130 for (i
= 0; i
< record
->num_frags
; i
++)
131 __skb_frag_unref(&record
->frags
[i
], false);
135 static void delete_all_records(struct tls_offload_context_tx
*offload_ctx
)
137 struct tls_record_info
*info
, *temp
;
139 list_for_each_entry_safe(info
, temp
, &offload_ctx
->records_list
, list
) {
140 list_del(&info
->list
);
141 destroy_record(info
);
144 offload_ctx
->retransmit_hint
= NULL
;
147 static void tls_icsk_clean_acked(struct sock
*sk
, u32 acked_seq
)
149 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
150 struct tls_record_info
*info
, *temp
;
151 struct tls_offload_context_tx
*ctx
;
152 u64 deleted_records
= 0;
158 ctx
= tls_offload_ctx_tx(tls_ctx
);
160 spin_lock_irqsave(&ctx
->lock
, flags
);
161 info
= ctx
->retransmit_hint
;
162 if (info
&& !before(acked_seq
, info
->end_seq
))
163 ctx
->retransmit_hint
= NULL
;
165 list_for_each_entry_safe(info
, temp
, &ctx
->records_list
, list
) {
166 if (before(acked_seq
, info
->end_seq
))
168 list_del(&info
->list
);
170 destroy_record(info
);
174 ctx
->unacked_record_sn
+= deleted_records
;
175 spin_unlock_irqrestore(&ctx
->lock
, flags
);
178 /* At this point, there should be no references on this
179 * socket and no in-flight SKBs associated with this
180 * socket, so it is safe to free all the resources.
182 void tls_device_sk_destruct(struct sock
*sk
)
184 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
185 struct tls_offload_context_tx
*ctx
= tls_offload_ctx_tx(tls_ctx
);
187 tls_ctx
->sk_destruct(sk
);
189 if (tls_ctx
->tx_conf
== TLS_HW
) {
190 if (ctx
->open_record
)
191 destroy_record(ctx
->open_record
);
192 delete_all_records(ctx
);
193 crypto_free_aead(ctx
->aead_send
);
194 clean_acked_data_disable(inet_csk(sk
));
197 if (refcount_dec_and_test(&tls_ctx
->refcount
))
198 tls_device_queue_ctx_destruction(tls_ctx
);
200 EXPORT_SYMBOL_GPL(tls_device_sk_destruct
);
202 void tls_device_free_resources_tx(struct sock
*sk
)
204 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
206 tls_free_partial_record(sk
, tls_ctx
);
209 void tls_offload_tx_resync_request(struct sock
*sk
, u32 got_seq
, u32 exp_seq
)
211 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
213 trace_tls_device_tx_resync_req(sk
, got_seq
, exp_seq
);
214 WARN_ON(test_and_set_bit(TLS_TX_SYNC_SCHED
, &tls_ctx
->flags
));
216 EXPORT_SYMBOL_GPL(tls_offload_tx_resync_request
);
218 static void tls_device_resync_tx(struct sock
*sk
, struct tls_context
*tls_ctx
,
221 struct net_device
*netdev
;
226 skb
= tcp_write_queue_tail(sk
);
228 TCP_SKB_CB(skb
)->eor
= 1;
230 rcd_sn
= tls_ctx
->tx
.rec_seq
;
232 trace_tls_device_tx_resync_send(sk
, seq
, rcd_sn
);
233 down_read(&device_offload_lock
);
234 netdev
= tls_ctx
->netdev
;
236 err
= netdev
->tlsdev_ops
->tls_dev_resync(netdev
, sk
, seq
,
238 TLS_OFFLOAD_CTX_DIR_TX
);
239 up_read(&device_offload_lock
);
243 clear_bit_unlock(TLS_TX_SYNC_SCHED
, &tls_ctx
->flags
);
246 static void tls_append_frag(struct tls_record_info
*record
,
247 struct page_frag
*pfrag
,
252 frag
= &record
->frags
[record
->num_frags
- 1];
253 if (skb_frag_page(frag
) == pfrag
->page
&&
254 skb_frag_off(frag
) + skb_frag_size(frag
) == pfrag
->offset
) {
255 skb_frag_size_add(frag
, size
);
258 __skb_frag_set_page(frag
, pfrag
->page
);
259 skb_frag_off_set(frag
, pfrag
->offset
);
260 skb_frag_size_set(frag
, size
);
262 get_page(pfrag
->page
);
265 pfrag
->offset
+= size
;
269 static int tls_push_record(struct sock
*sk
,
270 struct tls_context
*ctx
,
271 struct tls_offload_context_tx
*offload_ctx
,
272 struct tls_record_info
*record
,
275 struct tls_prot_info
*prot
= &ctx
->prot_info
;
276 struct tcp_sock
*tp
= tcp_sk(sk
);
280 record
->end_seq
= tp
->write_seq
+ record
->len
;
281 list_add_tail_rcu(&record
->list
, &offload_ctx
->records_list
);
282 offload_ctx
->open_record
= NULL
;
284 if (test_bit(TLS_TX_SYNC_SCHED
, &ctx
->flags
))
285 tls_device_resync_tx(sk
, ctx
, tp
->write_seq
);
287 tls_advance_record_sn(sk
, prot
, &ctx
->tx
);
289 for (i
= 0; i
< record
->num_frags
; i
++) {
290 frag
= &record
->frags
[i
];
291 sg_unmark_end(&offload_ctx
->sg_tx_data
[i
]);
292 sg_set_page(&offload_ctx
->sg_tx_data
[i
], skb_frag_page(frag
),
293 skb_frag_size(frag
), skb_frag_off(frag
));
294 sk_mem_charge(sk
, skb_frag_size(frag
));
295 get_page(skb_frag_page(frag
));
297 sg_mark_end(&offload_ctx
->sg_tx_data
[record
->num_frags
- 1]);
299 /* all ready, send */
300 return tls_push_sg(sk
, ctx
, offload_ctx
->sg_tx_data
, 0, flags
);
303 static int tls_device_record_close(struct sock
*sk
,
304 struct tls_context
*ctx
,
305 struct tls_record_info
*record
,
306 struct page_frag
*pfrag
,
307 unsigned char record_type
)
309 struct tls_prot_info
*prot
= &ctx
->prot_info
;
313 * device will fill in the tag, we just need to append a placeholder
314 * use socket memory to improve coalescing (re-using a single buffer
315 * increases frag count)
316 * if we can't allocate memory now, steal some back from data
318 if (likely(skb_page_frag_refill(prot
->tag_size
, pfrag
,
319 sk
->sk_allocation
))) {
321 tls_append_frag(record
, pfrag
, prot
->tag_size
);
323 ret
= prot
->tag_size
;
324 if (record
->len
<= prot
->overhead_size
)
329 tls_fill_prepend(ctx
, skb_frag_address(&record
->frags
[0]),
330 record
->len
- prot
->overhead_size
,
335 static int tls_create_new_record(struct tls_offload_context_tx
*offload_ctx
,
336 struct page_frag
*pfrag
,
339 struct tls_record_info
*record
;
342 record
= kmalloc(sizeof(*record
), GFP_KERNEL
);
346 frag
= &record
->frags
[0];
347 __skb_frag_set_page(frag
, pfrag
->page
);
348 skb_frag_off_set(frag
, pfrag
->offset
);
349 skb_frag_size_set(frag
, prepend_size
);
351 get_page(pfrag
->page
);
352 pfrag
->offset
+= prepend_size
;
354 record
->num_frags
= 1;
355 record
->len
= prepend_size
;
356 offload_ctx
->open_record
= record
;
360 static int tls_do_allocation(struct sock
*sk
,
361 struct tls_offload_context_tx
*offload_ctx
,
362 struct page_frag
*pfrag
,
367 if (!offload_ctx
->open_record
) {
368 if (unlikely(!skb_page_frag_refill(prepend_size
, pfrag
,
369 sk
->sk_allocation
))) {
370 READ_ONCE(sk
->sk_prot
)->enter_memory_pressure(sk
);
371 sk_stream_moderate_sndbuf(sk
);
375 ret
= tls_create_new_record(offload_ctx
, pfrag
, prepend_size
);
379 if (pfrag
->size
> pfrag
->offset
)
383 if (!sk_page_frag_refill(sk
, pfrag
))
389 static int tls_device_copy_data(void *addr
, size_t bytes
, struct iov_iter
*i
)
391 size_t pre_copy
, nocache
;
393 pre_copy
= ~((unsigned long)addr
- 1) & (SMP_CACHE_BYTES
- 1);
395 pre_copy
= min(pre_copy
, bytes
);
396 if (copy_from_iter(addr
, pre_copy
, i
) != pre_copy
)
402 nocache
= round_down(bytes
, SMP_CACHE_BYTES
);
403 if (copy_from_iter_nocache(addr
, nocache
, i
) != nocache
)
408 if (bytes
&& copy_from_iter(addr
, bytes
, i
) != bytes
)
414 static int tls_push_data(struct sock
*sk
,
415 struct iov_iter
*msg_iter
,
416 size_t size
, int flags
,
417 unsigned char record_type
)
419 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
420 struct tls_prot_info
*prot
= &tls_ctx
->prot_info
;
421 struct tls_offload_context_tx
*ctx
= tls_offload_ctx_tx(tls_ctx
);
422 struct tls_record_info
*record
;
423 int tls_push_record_flags
;
424 struct page_frag
*pfrag
;
425 size_t orig_size
= size
;
426 u32 max_open_record_len
;
433 ~(MSG_MORE
| MSG_DONTWAIT
| MSG_NOSIGNAL
| MSG_SENDPAGE_NOTLAST
))
436 if (unlikely(sk
->sk_err
))
439 flags
|= MSG_SENDPAGE_DECRYPTED
;
440 tls_push_record_flags
= flags
| MSG_SENDPAGE_NOTLAST
;
442 timeo
= sock_sndtimeo(sk
, flags
& MSG_DONTWAIT
);
443 if (tls_is_partially_sent_record(tls_ctx
)) {
444 rc
= tls_push_partial_record(sk
, tls_ctx
, flags
);
449 pfrag
= sk_page_frag(sk
);
451 /* TLS_HEADER_SIZE is not counted as part of the TLS record, and
452 * we need to leave room for an authentication tag.
454 max_open_record_len
= TLS_MAX_PAYLOAD_SIZE
+
457 rc
= tls_do_allocation(sk
, ctx
, pfrag
, prot
->prepend_size
);
459 rc
= sk_stream_wait_memory(sk
, &timeo
);
463 record
= ctx
->open_record
;
467 if (record_type
!= TLS_RECORD_TYPE_DATA
) {
468 /* avoid sending partial
469 * record with type !=
473 destroy_record(record
);
474 ctx
->open_record
= NULL
;
475 } else if (record
->len
> prot
->prepend_size
) {
482 record
= ctx
->open_record
;
483 copy
= min_t(size_t, size
, (pfrag
->size
- pfrag
->offset
));
484 copy
= min_t(size_t, copy
, (max_open_record_len
- record
->len
));
486 rc
= tls_device_copy_data(page_address(pfrag
->page
) +
487 pfrag
->offset
, copy
, msg_iter
);
490 tls_append_frag(record
, pfrag
, copy
);
495 tls_push_record_flags
= flags
;
496 if (flags
& (MSG_SENDPAGE_NOTLAST
| MSG_MORE
)) {
504 if (done
|| record
->len
>= max_open_record_len
||
505 (record
->num_frags
>= MAX_SKB_FRAGS
- 1)) {
506 rc
= tls_device_record_close(sk
, tls_ctx
, record
,
513 destroy_record(record
);
514 ctx
->open_record
= NULL
;
519 rc
= tls_push_record(sk
,
523 tls_push_record_flags
);
529 tls_ctx
->pending_open_record_frags
= more
;
531 if (orig_size
- size
> 0)
532 rc
= orig_size
- size
;
537 int tls_device_sendmsg(struct sock
*sk
, struct msghdr
*msg
, size_t size
)
539 unsigned char record_type
= TLS_RECORD_TYPE_DATA
;
540 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
543 mutex_lock(&tls_ctx
->tx_lock
);
546 if (unlikely(msg
->msg_controllen
)) {
547 rc
= tls_proccess_cmsg(sk
, msg
, &record_type
);
552 rc
= tls_push_data(sk
, &msg
->msg_iter
, size
,
553 msg
->msg_flags
, record_type
);
557 mutex_unlock(&tls_ctx
->tx_lock
);
561 int tls_device_sendpage(struct sock
*sk
, struct page
*page
,
562 int offset
, size_t size
, int flags
)
564 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
565 struct iov_iter msg_iter
;
570 if (flags
& MSG_SENDPAGE_NOTLAST
)
573 mutex_lock(&tls_ctx
->tx_lock
);
576 if (flags
& MSG_OOB
) {
582 iov
.iov_base
= kaddr
+ offset
;
584 iov_iter_kvec(&msg_iter
, WRITE
, &iov
, 1, size
);
585 rc
= tls_push_data(sk
, &msg_iter
, size
,
586 flags
, TLS_RECORD_TYPE_DATA
);
591 mutex_unlock(&tls_ctx
->tx_lock
);
595 struct tls_record_info
*tls_get_record(struct tls_offload_context_tx
*context
,
596 u32 seq
, u64
*p_record_sn
)
598 u64 record_sn
= context
->hint_record_sn
;
599 struct tls_record_info
*info
, *last
;
601 info
= context
->retransmit_hint
;
603 before(seq
, info
->end_seq
- info
->len
)) {
604 /* if retransmit_hint is irrelevant start
605 * from the beginning of the list
607 info
= list_first_entry_or_null(&context
->records_list
,
608 struct tls_record_info
, list
);
611 /* send the start_marker record if seq number is before the
612 * tls offload start marker sequence number. This record is
613 * required to handle TCP packets which are before TLS offload
615 * And if it's not start marker, look if this seq number
616 * belongs to the list.
618 if (likely(!tls_record_is_start_marker(info
))) {
619 /* we have the first record, get the last record to see
620 * if this seq number belongs to the list.
622 last
= list_last_entry(&context
->records_list
,
623 struct tls_record_info
, list
);
625 if (!between(seq
, tls_record_start_seq(info
),
629 record_sn
= context
->unacked_record_sn
;
632 /* We just need the _rcu for the READ_ONCE() */
634 list_for_each_entry_from_rcu(info
, &context
->records_list
, list
) {
635 if (before(seq
, info
->end_seq
)) {
636 if (!context
->retransmit_hint
||
638 context
->retransmit_hint
->end_seq
)) {
639 context
->hint_record_sn
= record_sn
;
640 context
->retransmit_hint
= info
;
642 *p_record_sn
= record_sn
;
643 goto exit_rcu_unlock
;
653 EXPORT_SYMBOL(tls_get_record
);
655 static int tls_device_push_pending_record(struct sock
*sk
, int flags
)
657 struct iov_iter msg_iter
;
659 iov_iter_kvec(&msg_iter
, WRITE
, NULL
, 0, 0);
660 return tls_push_data(sk
, &msg_iter
, 0, flags
, TLS_RECORD_TYPE_DATA
);
663 void tls_device_write_space(struct sock
*sk
, struct tls_context
*ctx
)
665 if (tls_is_partially_sent_record(ctx
)) {
666 gfp_t sk_allocation
= sk
->sk_allocation
;
668 WARN_ON_ONCE(sk
->sk_write_pending
);
670 sk
->sk_allocation
= GFP_ATOMIC
;
671 tls_push_partial_record(sk
, ctx
,
672 MSG_DONTWAIT
| MSG_NOSIGNAL
|
673 MSG_SENDPAGE_DECRYPTED
);
674 sk
->sk_allocation
= sk_allocation
;
678 static void tls_device_resync_rx(struct tls_context
*tls_ctx
,
679 struct sock
*sk
, u32 seq
, u8
*rcd_sn
)
681 struct tls_offload_context_rx
*rx_ctx
= tls_offload_ctx_rx(tls_ctx
);
682 struct net_device
*netdev
;
684 trace_tls_device_rx_resync_send(sk
, seq
, rcd_sn
, rx_ctx
->resync_type
);
686 netdev
= READ_ONCE(tls_ctx
->netdev
);
688 netdev
->tlsdev_ops
->tls_dev_resync(netdev
, sk
, seq
, rcd_sn
,
689 TLS_OFFLOAD_CTX_DIR_RX
);
691 TLS_INC_STATS(sock_net(sk
), LINUX_MIB_TLSRXDEVICERESYNC
);
695 tls_device_rx_resync_async(struct tls_offload_resync_async
*resync_async
,
696 s64 resync_req
, u32
*seq
, u16
*rcd_delta
)
698 u32 is_async
= resync_req
& RESYNC_REQ_ASYNC
;
699 u32 req_seq
= resync_req
>> 32;
700 u32 req_end
= req_seq
+ ((resync_req
>> 16) & 0xffff);
706 /* shouldn't get to wraparound:
707 * too long in async stage, something bad happened
709 if (WARN_ON_ONCE(resync_async
->rcd_delta
== USHRT_MAX
))
712 /* asynchronous stage: log all headers seq such that
713 * req_seq <= seq <= end_seq, and wait for real resync request
715 if (before(*seq
, req_seq
))
717 if (!after(*seq
, req_end
) &&
718 resync_async
->loglen
< TLS_DEVICE_RESYNC_ASYNC_LOGMAX
)
719 resync_async
->log
[resync_async
->loglen
++] = *seq
;
721 resync_async
->rcd_delta
++;
726 /* synchronous stage: check against the logged entries and
727 * proceed to check the next entries if no match was found
729 for (i
= 0; i
< resync_async
->loglen
; i
++)
730 if (req_seq
== resync_async
->log
[i
] &&
731 atomic64_try_cmpxchg(&resync_async
->req
, &resync_req
, 0)) {
732 *rcd_delta
= resync_async
->rcd_delta
- i
;
734 resync_async
->loglen
= 0;
735 resync_async
->rcd_delta
= 0;
739 resync_async
->loglen
= 0;
740 resync_async
->rcd_delta
= 0;
742 if (req_seq
== *seq
&&
743 atomic64_try_cmpxchg(&resync_async
->req
,
750 void tls_device_rx_resync_new_rec(struct sock
*sk
, u32 rcd_len
, u32 seq
)
752 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
753 struct tls_offload_context_rx
*rx_ctx
;
754 u8 rcd_sn
[TLS_MAX_REC_SEQ_SIZE
];
755 u32 sock_data
, is_req_pending
;
756 struct tls_prot_info
*prot
;
761 if (tls_ctx
->rx_conf
!= TLS_HW
)
763 if (unlikely(test_bit(TLS_RX_DEV_DEGRADED
, &tls_ctx
->flags
)))
766 prot
= &tls_ctx
->prot_info
;
767 rx_ctx
= tls_offload_ctx_rx(tls_ctx
);
768 memcpy(rcd_sn
, tls_ctx
->rx
.rec_seq
, prot
->rec_seq_size
);
770 switch (rx_ctx
->resync_type
) {
771 case TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ
:
772 resync_req
= atomic64_read(&rx_ctx
->resync_req
);
773 req_seq
= resync_req
>> 32;
774 seq
+= TLS_HEADER_SIZE
- 1;
775 is_req_pending
= resync_req
;
777 if (likely(!is_req_pending
) || req_seq
!= seq
||
778 !atomic64_try_cmpxchg(&rx_ctx
->resync_req
, &resync_req
, 0))
781 case TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT
:
782 if (likely(!rx_ctx
->resync_nh_do_now
))
785 /* head of next rec is already in, note that the sock_inq will
786 * include the currently parsed message when called from parser
788 sock_data
= tcp_inq(sk
);
789 if (sock_data
> rcd_len
) {
790 trace_tls_device_rx_resync_nh_delay(sk
, sock_data
,
795 rx_ctx
->resync_nh_do_now
= 0;
797 tls_bigint_increment(rcd_sn
, prot
->rec_seq_size
);
799 case TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC
:
800 resync_req
= atomic64_read(&rx_ctx
->resync_async
->req
);
801 is_req_pending
= resync_req
;
802 if (likely(!is_req_pending
))
805 if (!tls_device_rx_resync_async(rx_ctx
->resync_async
,
806 resync_req
, &seq
, &rcd_delta
))
808 tls_bigint_subtract(rcd_sn
, rcd_delta
);
812 tls_device_resync_rx(tls_ctx
, sk
, seq
, rcd_sn
);
815 static void tls_device_core_ctrl_rx_resync(struct tls_context
*tls_ctx
,
816 struct tls_offload_context_rx
*ctx
,
817 struct sock
*sk
, struct sk_buff
*skb
)
819 struct strp_msg
*rxm
;
821 /* device will request resyncs by itself based on stream scan */
822 if (ctx
->resync_type
!= TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT
)
824 /* already scheduled */
825 if (ctx
->resync_nh_do_now
)
827 /* seen decrypted fragments since last fully-failed record */
828 if (ctx
->resync_nh_reset
) {
829 ctx
->resync_nh_reset
= 0;
830 ctx
->resync_nh
.decrypted_failed
= 1;
831 ctx
->resync_nh
.decrypted_tgt
= TLS_DEVICE_RESYNC_NH_START_IVAL
;
835 if (++ctx
->resync_nh
.decrypted_failed
<= ctx
->resync_nh
.decrypted_tgt
)
838 /* doing resync, bump the next target in case it fails */
839 if (ctx
->resync_nh
.decrypted_tgt
< TLS_DEVICE_RESYNC_NH_MAX_IVAL
)
840 ctx
->resync_nh
.decrypted_tgt
*= 2;
842 ctx
->resync_nh
.decrypted_tgt
+= TLS_DEVICE_RESYNC_NH_MAX_IVAL
;
846 /* head of next rec is already in, parser will sync for us */
847 if (tcp_inq(sk
) > rxm
->full_len
) {
848 trace_tls_device_rx_resync_nh_schedule(sk
);
849 ctx
->resync_nh_do_now
= 1;
851 struct tls_prot_info
*prot
= &tls_ctx
->prot_info
;
852 u8 rcd_sn
[TLS_MAX_REC_SEQ_SIZE
];
854 memcpy(rcd_sn
, tls_ctx
->rx
.rec_seq
, prot
->rec_seq_size
);
855 tls_bigint_increment(rcd_sn
, prot
->rec_seq_size
);
857 tls_device_resync_rx(tls_ctx
, sk
, tcp_sk(sk
)->copied_seq
,
862 static int tls_device_reencrypt(struct sock
*sk
, struct sk_buff
*skb
)
864 struct strp_msg
*rxm
= strp_msg(skb
);
865 int err
= 0, offset
= rxm
->offset
, copy
, nsg
, data_len
, pos
;
866 struct sk_buff
*skb_iter
, *unused
;
867 struct scatterlist sg
[1];
868 char *orig_buf
, *buf
;
870 orig_buf
= kmalloc(rxm
->full_len
+ TLS_HEADER_SIZE
+
871 TLS_CIPHER_AES_GCM_128_IV_SIZE
, sk
->sk_allocation
);
876 nsg
= skb_cow_data(skb
, 0, &unused
);
877 if (unlikely(nsg
< 0)) {
882 sg_init_table(sg
, 1);
883 sg_set_buf(&sg
[0], buf
,
884 rxm
->full_len
+ TLS_HEADER_SIZE
+
885 TLS_CIPHER_AES_GCM_128_IV_SIZE
);
886 err
= skb_copy_bits(skb
, offset
, buf
,
887 TLS_HEADER_SIZE
+ TLS_CIPHER_AES_GCM_128_IV_SIZE
);
891 /* We are interested only in the decrypted data not the auth */
892 err
= decrypt_skb(sk
, skb
, sg
);
898 data_len
= rxm
->full_len
- TLS_CIPHER_AES_GCM_128_TAG_SIZE
;
900 if (skb_pagelen(skb
) > offset
) {
901 copy
= min_t(int, skb_pagelen(skb
) - offset
, data_len
);
903 if (skb
->decrypted
) {
904 err
= skb_store_bits(skb
, offset
, buf
, copy
);
913 pos
= skb_pagelen(skb
);
914 skb_walk_frags(skb
, skb_iter
) {
917 /* Practically all frags must belong to msg if reencrypt
918 * is needed with current strparser and coalescing logic,
919 * but strparser may "get optimized", so let's be safe.
921 if (pos
+ skb_iter
->len
<= offset
)
923 if (pos
>= data_len
+ rxm
->offset
)
926 frag_pos
= offset
- pos
;
927 copy
= min_t(int, skb_iter
->len
- frag_pos
,
928 data_len
+ rxm
->offset
- offset
);
930 if (skb_iter
->decrypted
) {
931 err
= skb_store_bits(skb_iter
, frag_pos
, buf
, copy
);
939 pos
+= skb_iter
->len
;
947 int tls_device_decrypted(struct sock
*sk
, struct tls_context
*tls_ctx
,
948 struct sk_buff
*skb
, struct strp_msg
*rxm
)
950 struct tls_offload_context_rx
*ctx
= tls_offload_ctx_rx(tls_ctx
);
951 int is_decrypted
= skb
->decrypted
;
952 int is_encrypted
= !is_decrypted
;
953 struct sk_buff
*skb_iter
;
955 /* Check if all the data is decrypted already */
956 skb_walk_frags(skb
, skb_iter
) {
957 is_decrypted
&= skb_iter
->decrypted
;
958 is_encrypted
&= !skb_iter
->decrypted
;
961 trace_tls_device_decrypted(sk
, tcp_sk(sk
)->copied_seq
- rxm
->full_len
,
962 tls_ctx
->rx
.rec_seq
, rxm
->full_len
,
963 is_encrypted
, is_decrypted
);
965 ctx
->sw
.decrypted
|= is_decrypted
;
967 if (unlikely(test_bit(TLS_RX_DEV_DEGRADED
, &tls_ctx
->flags
))) {
968 if (likely(is_encrypted
|| is_decrypted
))
971 /* After tls_device_down disables the offload, the next SKB will
972 * likely have initial fragments decrypted, and final ones not
973 * decrypted. We need to reencrypt that single SKB.
975 return tls_device_reencrypt(sk
, skb
);
978 /* Return immediately if the record is either entirely plaintext or
979 * entirely ciphertext. Otherwise handle reencrypt partially decrypted
983 ctx
->resync_nh_reset
= 1;
987 tls_device_core_ctrl_rx_resync(tls_ctx
, ctx
, sk
, skb
);
991 ctx
->resync_nh_reset
= 1;
992 return tls_device_reencrypt(sk
, skb
);
995 static void tls_device_attach(struct tls_context
*ctx
, struct sock
*sk
,
996 struct net_device
*netdev
)
998 if (sk
->sk_destruct
!= tls_device_sk_destruct
) {
999 refcount_set(&ctx
->refcount
, 1);
1001 ctx
->netdev
= netdev
;
1002 spin_lock_irq(&tls_device_lock
);
1003 list_add_tail(&ctx
->list
, &tls_device_list
);
1004 spin_unlock_irq(&tls_device_lock
);
1006 ctx
->sk_destruct
= sk
->sk_destruct
;
1007 smp_store_release(&sk
->sk_destruct
, tls_device_sk_destruct
);
1011 int tls_set_device_offload(struct sock
*sk
, struct tls_context
*ctx
)
1013 u16 nonce_size
, tag_size
, iv_size
, rec_seq_size
, salt_size
;
1014 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
1015 struct tls_prot_info
*prot
= &tls_ctx
->prot_info
;
1016 struct tls_record_info
*start_marker_record
;
1017 struct tls_offload_context_tx
*offload_ctx
;
1018 struct tls_crypto_info
*crypto_info
;
1019 struct net_device
*netdev
;
1021 struct sk_buff
*skb
;
1028 if (ctx
->priv_ctx_tx
)
1031 start_marker_record
= kmalloc(sizeof(*start_marker_record
), GFP_KERNEL
);
1032 if (!start_marker_record
)
1035 offload_ctx
= kzalloc(TLS_OFFLOAD_CONTEXT_SIZE_TX
, GFP_KERNEL
);
1038 goto free_marker_record
;
1041 crypto_info
= &ctx
->crypto_send
.info
;
1042 if (crypto_info
->version
!= TLS_1_2_VERSION
) {
1044 goto free_offload_ctx
;
1047 switch (crypto_info
->cipher_type
) {
1048 case TLS_CIPHER_AES_GCM_128
:
1049 nonce_size
= TLS_CIPHER_AES_GCM_128_IV_SIZE
;
1050 tag_size
= TLS_CIPHER_AES_GCM_128_TAG_SIZE
;
1051 iv_size
= TLS_CIPHER_AES_GCM_128_IV_SIZE
;
1052 iv
= ((struct tls12_crypto_info_aes_gcm_128
*)crypto_info
)->iv
;
1053 rec_seq_size
= TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE
;
1054 salt_size
= TLS_CIPHER_AES_GCM_128_SALT_SIZE
;
1056 ((struct tls12_crypto_info_aes_gcm_128
*)crypto_info
)->rec_seq
;
1060 goto free_offload_ctx
;
1063 /* Sanity-check the rec_seq_size for stack allocations */
1064 if (rec_seq_size
> TLS_MAX_REC_SEQ_SIZE
) {
1066 goto free_offload_ctx
;
1069 prot
->version
= crypto_info
->version
;
1070 prot
->cipher_type
= crypto_info
->cipher_type
;
1071 prot
->prepend_size
= TLS_HEADER_SIZE
+ nonce_size
;
1072 prot
->tag_size
= tag_size
;
1073 prot
->overhead_size
= prot
->prepend_size
+ prot
->tag_size
;
1074 prot
->iv_size
= iv_size
;
1075 prot
->salt_size
= salt_size
;
1076 ctx
->tx
.iv
= kmalloc(iv_size
+ TLS_CIPHER_AES_GCM_128_SALT_SIZE
,
1080 goto free_offload_ctx
;
1083 memcpy(ctx
->tx
.iv
+ TLS_CIPHER_AES_GCM_128_SALT_SIZE
, iv
, iv_size
);
1085 prot
->rec_seq_size
= rec_seq_size
;
1086 ctx
->tx
.rec_seq
= kmemdup(rec_seq
, rec_seq_size
, GFP_KERNEL
);
1087 if (!ctx
->tx
.rec_seq
) {
1092 rc
= tls_sw_fallback_init(sk
, offload_ctx
, crypto_info
);
1096 /* start at rec_seq - 1 to account for the start marker record */
1097 memcpy(&rcd_sn
, ctx
->tx
.rec_seq
, sizeof(rcd_sn
));
1098 offload_ctx
->unacked_record_sn
= be64_to_cpu(rcd_sn
) - 1;
1100 start_marker_record
->end_seq
= tcp_sk(sk
)->write_seq
;
1101 start_marker_record
->len
= 0;
1102 start_marker_record
->num_frags
= 0;
1104 INIT_LIST_HEAD(&offload_ctx
->records_list
);
1105 list_add_tail(&start_marker_record
->list
, &offload_ctx
->records_list
);
1106 spin_lock_init(&offload_ctx
->lock
);
1107 sg_init_table(offload_ctx
->sg_tx_data
,
1108 ARRAY_SIZE(offload_ctx
->sg_tx_data
));
1110 clean_acked_data_enable(inet_csk(sk
), &tls_icsk_clean_acked
);
1111 ctx
->push_pending_record
= tls_device_push_pending_record
;
1113 /* TLS offload is greatly simplified if we don't send
1114 * SKBs where only part of the payload needs to be encrypted.
1115 * So mark the last skb in the write queue as end of record.
1117 skb
= tcp_write_queue_tail(sk
);
1119 TCP_SKB_CB(skb
)->eor
= 1;
1121 netdev
= get_netdev_for_sock(sk
);
1123 pr_err_ratelimited("%s: netdev not found\n", __func__
);
1128 if (!(netdev
->features
& NETIF_F_HW_TLS_TX
)) {
1130 goto release_netdev
;
1133 /* Avoid offloading if the device is down
1134 * We don't want to offload new flows after
1135 * the NETDEV_DOWN event
1137 * device_offload_lock is taken in tls_devices's NETDEV_DOWN
1138 * handler thus protecting from the device going down before
1139 * ctx was added to tls_device_list.
1141 down_read(&device_offload_lock
);
1142 if (!(netdev
->flags
& IFF_UP
)) {
1147 ctx
->priv_ctx_tx
= offload_ctx
;
1148 rc
= netdev
->tlsdev_ops
->tls_dev_add(netdev
, sk
, TLS_OFFLOAD_CTX_DIR_TX
,
1149 &ctx
->crypto_send
.info
,
1150 tcp_sk(sk
)->write_seq
);
1151 trace_tls_device_offload_set(sk
, TLS_OFFLOAD_CTX_DIR_TX
,
1152 tcp_sk(sk
)->write_seq
, rec_seq
, rc
);
1156 tls_device_attach(ctx
, sk
, netdev
);
1157 up_read(&device_offload_lock
);
1159 /* following this assignment tls_is_sk_tx_device_offloaded
1160 * will return true and the context might be accessed
1161 * by the netdev's xmit function.
1163 smp_store_release(&sk
->sk_validate_xmit_skb
, tls_validate_xmit_skb
);
1169 up_read(&device_offload_lock
);
1173 clean_acked_data_disable(inet_csk(sk
));
1174 crypto_free_aead(offload_ctx
->aead_send
);
1176 kfree(ctx
->tx
.rec_seq
);
1181 ctx
->priv_ctx_tx
= NULL
;
1183 kfree(start_marker_record
);
1187 int tls_set_device_offload_rx(struct sock
*sk
, struct tls_context
*ctx
)
1189 struct tls12_crypto_info_aes_gcm_128
*info
;
1190 struct tls_offload_context_rx
*context
;
1191 struct net_device
*netdev
;
1194 if (ctx
->crypto_recv
.info
.version
!= TLS_1_2_VERSION
)
1197 netdev
= get_netdev_for_sock(sk
);
1199 pr_err_ratelimited("%s: netdev not found\n", __func__
);
1203 if (!(netdev
->features
& NETIF_F_HW_TLS_RX
)) {
1205 goto release_netdev
;
1208 /* Avoid offloading if the device is down
1209 * We don't want to offload new flows after
1210 * the NETDEV_DOWN event
1212 * device_offload_lock is taken in tls_devices's NETDEV_DOWN
1213 * handler thus protecting from the device going down before
1214 * ctx was added to tls_device_list.
1216 down_read(&device_offload_lock
);
1217 if (!(netdev
->flags
& IFF_UP
)) {
1222 context
= kzalloc(TLS_OFFLOAD_CONTEXT_SIZE_RX
, GFP_KERNEL
);
1227 context
->resync_nh_reset
= 1;
1229 ctx
->priv_ctx_rx
= context
;
1230 rc
= tls_set_sw_offload(sk
, ctx
, 0);
1234 rc
= netdev
->tlsdev_ops
->tls_dev_add(netdev
, sk
, TLS_OFFLOAD_CTX_DIR_RX
,
1235 &ctx
->crypto_recv
.info
,
1236 tcp_sk(sk
)->copied_seq
);
1237 info
= (void *)&ctx
->crypto_recv
.info
;
1238 trace_tls_device_offload_set(sk
, TLS_OFFLOAD_CTX_DIR_RX
,
1239 tcp_sk(sk
)->copied_seq
, info
->rec_seq
, rc
);
1241 goto free_sw_resources
;
1243 tls_device_attach(ctx
, sk
, netdev
);
1244 up_read(&device_offload_lock
);
1251 up_read(&device_offload_lock
);
1252 tls_sw_free_resources_rx(sk
);
1253 down_read(&device_offload_lock
);
1255 ctx
->priv_ctx_rx
= NULL
;
1257 up_read(&device_offload_lock
);
1263 void tls_device_offload_cleanup_rx(struct sock
*sk
)
1265 struct tls_context
*tls_ctx
= tls_get_ctx(sk
);
1266 struct net_device
*netdev
;
1268 down_read(&device_offload_lock
);
1269 netdev
= tls_ctx
->netdev
;
1273 netdev
->tlsdev_ops
->tls_dev_del(netdev
, tls_ctx
,
1274 TLS_OFFLOAD_CTX_DIR_RX
);
1276 if (tls_ctx
->tx_conf
!= TLS_HW
) {
1278 tls_ctx
->netdev
= NULL
;
1280 set_bit(TLS_RX_DEV_CLOSED
, &tls_ctx
->flags
);
1283 up_read(&device_offload_lock
);
1284 tls_sw_release_resources_rx(sk
);
1287 static int tls_device_down(struct net_device
*netdev
)
1289 struct tls_context
*ctx
, *tmp
;
1290 unsigned long flags
;
1293 /* Request a write lock to block new offload attempts */
1294 down_write(&device_offload_lock
);
1296 spin_lock_irqsave(&tls_device_lock
, flags
);
1297 list_for_each_entry_safe(ctx
, tmp
, &tls_device_list
, list
) {
1298 if (ctx
->netdev
!= netdev
||
1299 !refcount_inc_not_zero(&ctx
->refcount
))
1302 list_move(&ctx
->list
, &list
);
1304 spin_unlock_irqrestore(&tls_device_lock
, flags
);
1306 list_for_each_entry_safe(ctx
, tmp
, &list
, list
) {
1307 /* Stop offloaded TX and switch to the fallback.
1308 * tls_is_sk_tx_device_offloaded will return false.
1310 WRITE_ONCE(ctx
->sk
->sk_validate_xmit_skb
, tls_validate_xmit_skb_sw
);
1312 /* Stop the RX and TX resync.
1313 * tls_dev_resync must not be called after tls_dev_del.
1315 WRITE_ONCE(ctx
->netdev
, NULL
);
1317 /* Start skipping the RX resync logic completely. */
1318 set_bit(TLS_RX_DEV_DEGRADED
, &ctx
->flags
);
1320 /* Sync with inflight packets. After this point:
1321 * TX: no non-encrypted packets will be passed to the driver.
1322 * RX: resync requests from the driver will be ignored.
1326 /* Release the offload context on the driver side. */
1327 if (ctx
->tx_conf
== TLS_HW
)
1328 netdev
->tlsdev_ops
->tls_dev_del(netdev
, ctx
,
1329 TLS_OFFLOAD_CTX_DIR_TX
);
1330 if (ctx
->rx_conf
== TLS_HW
&&
1331 !test_bit(TLS_RX_DEV_CLOSED
, &ctx
->flags
))
1332 netdev
->tlsdev_ops
->tls_dev_del(netdev
, ctx
,
1333 TLS_OFFLOAD_CTX_DIR_RX
);
1337 /* Move the context to a separate list for two reasons:
1338 * 1. When the context is deallocated, list_del is called.
1339 * 2. It's no longer an offloaded context, so we don't want to
1340 * run offload-specific code on this context.
1342 spin_lock_irqsave(&tls_device_lock
, flags
);
1343 list_move_tail(&ctx
->list
, &tls_device_down_list
);
1344 spin_unlock_irqrestore(&tls_device_lock
, flags
);
1346 /* Device contexts for RX and TX will be freed in on sk_destruct
1347 * by tls_device_free_ctx. rx_conf and tx_conf stay in TLS_HW.
1351 up_write(&device_offload_lock
);
1353 flush_work(&tls_device_gc_work
);
1358 static int tls_dev_event(struct notifier_block
*this, unsigned long event
,
1361 struct net_device
*dev
= netdev_notifier_info_to_dev(ptr
);
1363 if (!dev
->tlsdev_ops
&&
1364 !(dev
->features
& (NETIF_F_HW_TLS_RX
| NETIF_F_HW_TLS_TX
)))
1368 case NETDEV_REGISTER
:
1369 case NETDEV_FEAT_CHANGE
:
1370 if (netif_is_bond_master(dev
))
1372 if ((dev
->features
& NETIF_F_HW_TLS_RX
) &&
1373 !dev
->tlsdev_ops
->tls_dev_resync
)
1376 if (dev
->tlsdev_ops
&&
1377 dev
->tlsdev_ops
->tls_dev_add
&&
1378 dev
->tlsdev_ops
->tls_dev_del
)
1383 return tls_device_down(dev
);
1388 static struct notifier_block tls_dev_notifier
= {
1389 .notifier_call
= tls_dev_event
,
1392 void __init
tls_device_init(void)
1394 register_netdevice_notifier(&tls_dev_notifier
);
1397 void __exit
tls_device_cleanup(void)
1399 unregister_netdevice_notifier(&tls_dev_notifier
);
1400 flush_work(&tls_device_gc_work
);
1401 clean_acked_data_flush();