2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/module.h>
37 #include <net/inet_common.h>
38 #include <linux/highmem.h>
39 #include <linux/netdevice.h>
40 #include <linux/sched/signal.h>
41 #include <linux/inetdevice.h>
45 MODULE_AUTHOR("Mellanox Technologies");
46 MODULE_DESCRIPTION("Transport Layer Security Support");
47 MODULE_LICENSE("Dual BSD/GPL");
64 static struct proto
*saved_tcpv6_prot
;
65 static DEFINE_MUTEX(tcpv6_prot_mutex
);
66 static LIST_HEAD(device_list
);
67 static DEFINE_MUTEX(device_mutex
);
68 static struct proto tls_prots
[TLS_NUM_PROTS
][TLS_NUM_CONFIG
];
69 static struct proto_ops tls_sw_proto_ops
;
71 static inline void update_sk_prot(struct sock
*sk
, struct tls_context
*ctx
)
73 int ip_ver
= sk
->sk_family
== AF_INET6
? TLSV6
: TLSV4
;
75 sk
->sk_prot
= &tls_prots
[ip_ver
][ctx
->conf
];
78 int wait_on_pending_writer(struct sock
*sk
, long *timeo
)
81 DEFINE_WAIT_FUNC(wait
, woken_wake_function
);
83 add_wait_queue(sk_sleep(sk
), &wait
);
90 if (signal_pending(current
)) {
91 rc
= sock_intr_errno(*timeo
);
95 if (sk_wait_event(sk
, timeo
, !sk
->sk_write_pending
, &wait
))
98 remove_wait_queue(sk_sleep(sk
), &wait
);
102 int tls_push_sg(struct sock
*sk
,
103 struct tls_context
*ctx
,
104 struct scatterlist
*sg
,
108 int sendpage_flags
= flags
| MSG_SENDPAGE_NOTLAST
;
112 int offset
= first_offset
;
114 size
= sg
->length
- offset
;
115 offset
+= sg
->offset
;
119 sendpage_flags
= flags
;
121 /* is sending application-limited? */
122 tcp_rate_check_app_limited(sk
);
125 ret
= do_tcp_sendpages(sk
, p
, offset
, size
, sendpage_flags
);
134 offset
-= sg
->offset
;
135 ctx
->partially_sent_offset
= offset
;
136 ctx
->partially_sent_record
= (void *)sg
;
141 sk_mem_uncharge(sk
, sg
->length
);
150 clear_bit(TLS_PENDING_CLOSED_RECORD
, &ctx
->flags
);
155 static int tls_handle_open_record(struct sock
*sk
, int flags
)
157 struct tls_context
*ctx
= tls_get_ctx(sk
);
159 if (tls_is_pending_open_record(ctx
))
160 return ctx
->push_pending_record(sk
, flags
);
165 int tls_proccess_cmsg(struct sock
*sk
, struct msghdr
*msg
,
166 unsigned char *record_type
)
168 struct cmsghdr
*cmsg
;
171 for_each_cmsghdr(cmsg
, msg
) {
172 if (!CMSG_OK(msg
, cmsg
))
174 if (cmsg
->cmsg_level
!= SOL_TLS
)
177 switch (cmsg
->cmsg_type
) {
178 case TLS_SET_RECORD_TYPE
:
179 if (cmsg
->cmsg_len
< CMSG_LEN(sizeof(*record_type
)))
182 if (msg
->msg_flags
& MSG_MORE
)
185 rc
= tls_handle_open_record(sk
, msg
->msg_flags
);
189 *record_type
= *(unsigned char *)CMSG_DATA(cmsg
);
200 int tls_push_pending_closed_record(struct sock
*sk
, struct tls_context
*ctx
,
201 int flags
, long *timeo
)
203 struct scatterlist
*sg
;
206 if (!tls_is_partially_sent_record(ctx
))
207 return ctx
->push_pending_record(sk
, flags
);
209 sg
= ctx
->partially_sent_record
;
210 offset
= ctx
->partially_sent_offset
;
212 ctx
->partially_sent_record
= NULL
;
213 return tls_push_sg(sk
, ctx
, sg
, offset
, flags
);
216 static void tls_write_space(struct sock
*sk
)
218 struct tls_context
*ctx
= tls_get_ctx(sk
);
220 if (!sk
->sk_write_pending
&& tls_is_pending_closed_record(ctx
)) {
221 gfp_t sk_allocation
= sk
->sk_allocation
;
225 sk
->sk_allocation
= GFP_ATOMIC
;
226 rc
= tls_push_pending_closed_record(sk
, ctx
,
230 sk
->sk_allocation
= sk_allocation
;
236 ctx
->sk_write_space(sk
);
239 static void tls_sk_proto_close(struct sock
*sk
, long timeout
)
241 struct tls_context
*ctx
= tls_get_ctx(sk
);
242 long timeo
= sock_sndtimeo(sk
, 0);
243 void (*sk_proto_close
)(struct sock
*sk
, long timeout
);
246 sk_proto_close
= ctx
->sk_proto_close
;
248 if (ctx
->conf
== TLS_HW_RECORD
)
249 goto skip_tx_cleanup
;
251 if (ctx
->conf
== TLS_BASE
) {
254 goto skip_tx_cleanup
;
257 if (!tls_complete_pending_work(sk
, ctx
, 0, &timeo
))
258 tls_handle_open_record(sk
, 0);
260 if (ctx
->partially_sent_record
) {
261 struct scatterlist
*sg
= ctx
->partially_sent_record
;
264 put_page(sg_page(sg
));
265 sk_mem_uncharge(sk
, sg
->length
);
273 kfree(ctx
->tx
.rec_seq
);
275 kfree(ctx
->rx
.rec_seq
);
278 if (ctx
->conf
== TLS_SW_TX
||
279 ctx
->conf
== TLS_SW_RX
||
280 ctx
->conf
== TLS_SW_RXTX
) {
281 tls_sw_free_resources(sk
);
286 sk_proto_close(sk
, timeout
);
287 /* free ctx for TLS_HW_RECORD, used by tcp_set_state
288 * for sk->sk_prot->unhash [tls_hw_unhash]
290 if (ctx
&& ctx
->conf
== TLS_HW_RECORD
)
294 static int do_tls_getsockopt_tx(struct sock
*sk
, char __user
*optval
,
298 struct tls_context
*ctx
= tls_get_ctx(sk
);
299 struct tls_crypto_info
*crypto_info
;
302 if (get_user(len
, optlen
))
305 if (!optval
|| (len
< sizeof(*crypto_info
))) {
315 /* get user crypto info */
316 crypto_info
= &ctx
->crypto_send
;
318 if (!TLS_CRYPTO_INFO_READY(crypto_info
)) {
323 if (len
== sizeof(*crypto_info
)) {
324 if (copy_to_user(optval
, crypto_info
, sizeof(*crypto_info
)))
329 switch (crypto_info
->cipher_type
) {
330 case TLS_CIPHER_AES_GCM_128
: {
331 struct tls12_crypto_info_aes_gcm_128
*
332 crypto_info_aes_gcm_128
=
333 container_of(crypto_info
,
334 struct tls12_crypto_info_aes_gcm_128
,
337 if (len
!= sizeof(*crypto_info_aes_gcm_128
)) {
342 memcpy(crypto_info_aes_gcm_128
->iv
,
343 ctx
->tx
.iv
+ TLS_CIPHER_AES_GCM_128_SALT_SIZE
,
344 TLS_CIPHER_AES_GCM_128_IV_SIZE
);
345 memcpy(crypto_info_aes_gcm_128
->rec_seq
, ctx
->tx
.rec_seq
,
346 TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE
);
348 if (copy_to_user(optval
,
349 crypto_info_aes_gcm_128
,
350 sizeof(*crypto_info_aes_gcm_128
)))
362 static int do_tls_getsockopt(struct sock
*sk
, int optname
,
363 char __user
*optval
, int __user
*optlen
)
369 rc
= do_tls_getsockopt_tx(sk
, optval
, optlen
);
378 static int tls_getsockopt(struct sock
*sk
, int level
, int optname
,
379 char __user
*optval
, int __user
*optlen
)
381 struct tls_context
*ctx
= tls_get_ctx(sk
);
383 if (level
!= SOL_TLS
)
384 return ctx
->getsockopt(sk
, level
, optname
, optval
, optlen
);
386 return do_tls_getsockopt(sk
, optname
, optval
, optlen
);
389 static int do_tls_setsockopt_conf(struct sock
*sk
, char __user
*optval
,
390 unsigned int optlen
, int tx
)
392 struct tls_crypto_info
*crypto_info
;
393 struct tls_context
*ctx
= tls_get_ctx(sk
);
397 if (!optval
|| (optlen
< sizeof(*crypto_info
))) {
403 crypto_info
= &ctx
->crypto_send
;
405 crypto_info
= &ctx
->crypto_recv
;
407 /* Currently we don't support set crypto info more than one time */
408 if (TLS_CRYPTO_INFO_READY(crypto_info
)) {
413 rc
= copy_from_user(crypto_info
, optval
, sizeof(*crypto_info
));
416 goto err_crypto_info
;
420 if (crypto_info
->version
!= TLS_1_2_VERSION
) {
422 goto err_crypto_info
;
425 switch (crypto_info
->cipher_type
) {
426 case TLS_CIPHER_AES_GCM_128
: {
427 if (optlen
!= sizeof(struct tls12_crypto_info_aes_gcm_128
)) {
429 goto err_crypto_info
;
431 rc
= copy_from_user(crypto_info
+ 1, optval
+ sizeof(*crypto_info
),
432 optlen
- sizeof(*crypto_info
));
435 goto err_crypto_info
;
441 goto err_crypto_info
;
444 /* currently SW is default, we will have ethtool in future */
446 rc
= tls_set_sw_offload(sk
, ctx
, 1);
447 if (ctx
->conf
== TLS_SW_RX
)
452 rc
= tls_set_sw_offload(sk
, ctx
, 0);
453 if (ctx
->conf
== TLS_SW_TX
)
460 goto err_crypto_info
;
463 update_sk_prot(sk
, ctx
);
465 ctx
->sk_write_space
= sk
->sk_write_space
;
466 sk
->sk_write_space
= tls_write_space
;
468 sk
->sk_socket
->ops
= &tls_sw_proto_ops
;
473 memset(crypto_info
, 0, sizeof(*crypto_info
));
478 static int do_tls_setsockopt(struct sock
*sk
, int optname
,
479 char __user
*optval
, unsigned int optlen
)
487 rc
= do_tls_setsockopt_conf(sk
, optval
, optlen
,
498 static int tls_setsockopt(struct sock
*sk
, int level
, int optname
,
499 char __user
*optval
, unsigned int optlen
)
501 struct tls_context
*ctx
= tls_get_ctx(sk
);
503 if (level
!= SOL_TLS
)
504 return ctx
->setsockopt(sk
, level
, optname
, optval
, optlen
);
506 return do_tls_setsockopt(sk
, optname
, optval
, optlen
);
509 static struct tls_context
*create_ctx(struct sock
*sk
)
511 struct inet_connection_sock
*icsk
= inet_csk(sk
);
512 struct tls_context
*ctx
;
514 ctx
= kzalloc(sizeof(*ctx
), GFP_KERNEL
);
518 icsk
->icsk_ulp_data
= ctx
;
522 static int tls_hw_prot(struct sock
*sk
)
524 struct tls_context
*ctx
;
525 struct tls_device
*dev
;
528 mutex_lock(&device_mutex
);
529 list_for_each_entry(dev
, &device_list
, dev_list
) {
530 if (dev
->feature
&& dev
->feature(dev
)) {
531 ctx
= create_ctx(sk
);
535 ctx
->hash
= sk
->sk_prot
->hash
;
536 ctx
->unhash
= sk
->sk_prot
->unhash
;
537 ctx
->sk_proto_close
= sk
->sk_prot
->close
;
538 ctx
->conf
= TLS_HW_RECORD
;
539 update_sk_prot(sk
, ctx
);
545 mutex_unlock(&device_mutex
);
549 static void tls_hw_unhash(struct sock
*sk
)
551 struct tls_context
*ctx
= tls_get_ctx(sk
);
552 struct tls_device
*dev
;
554 mutex_lock(&device_mutex
);
555 list_for_each_entry(dev
, &device_list
, dev_list
) {
557 dev
->unhash(dev
, sk
);
559 mutex_unlock(&device_mutex
);
563 static int tls_hw_hash(struct sock
*sk
)
565 struct tls_context
*ctx
= tls_get_ctx(sk
);
566 struct tls_device
*dev
;
570 mutex_lock(&device_mutex
);
571 list_for_each_entry(dev
, &device_list
, dev_list
) {
573 err
|= dev
->hash(dev
, sk
);
575 mutex_unlock(&device_mutex
);
582 static void build_protos(struct proto
*prot
, struct proto
*base
)
584 prot
[TLS_BASE
] = *base
;
585 prot
[TLS_BASE
].setsockopt
= tls_setsockopt
;
586 prot
[TLS_BASE
].getsockopt
= tls_getsockopt
;
587 prot
[TLS_BASE
].close
= tls_sk_proto_close
;
589 prot
[TLS_SW_TX
] = prot
[TLS_BASE
];
590 prot
[TLS_SW_TX
].sendmsg
= tls_sw_sendmsg
;
591 prot
[TLS_SW_TX
].sendpage
= tls_sw_sendpage
;
593 prot
[TLS_SW_RX
] = prot
[TLS_BASE
];
594 prot
[TLS_SW_RX
].recvmsg
= tls_sw_recvmsg
;
595 prot
[TLS_SW_RX
].close
= tls_sk_proto_close
;
597 prot
[TLS_SW_RXTX
] = prot
[TLS_SW_TX
];
598 prot
[TLS_SW_RXTX
].recvmsg
= tls_sw_recvmsg
;
599 prot
[TLS_SW_RXTX
].close
= tls_sk_proto_close
;
601 prot
[TLS_HW_RECORD
] = *base
;
602 prot
[TLS_HW_RECORD
].hash
= tls_hw_hash
;
603 prot
[TLS_HW_RECORD
].unhash
= tls_hw_unhash
;
604 prot
[TLS_HW_RECORD
].close
= tls_sk_proto_close
;
607 static int tls_init(struct sock
*sk
)
609 int ip_ver
= sk
->sk_family
== AF_INET6
? TLSV6
: TLSV4
;
610 struct tls_context
*ctx
;
616 /* The TLS ulp is currently supported only for TCP sockets
617 * in ESTABLISHED state.
618 * Supporting sockets in LISTEN state will require us
619 * to modify the accept implementation to clone rather then
620 * share the ulp context.
622 if (sk
->sk_state
!= TCP_ESTABLISHED
)
625 /* allocate tls context */
626 ctx
= create_ctx(sk
);
631 ctx
->setsockopt
= sk
->sk_prot
->setsockopt
;
632 ctx
->getsockopt
= sk
->sk_prot
->getsockopt
;
633 ctx
->sk_proto_close
= sk
->sk_prot
->close
;
635 /* Build IPv6 TLS whenever the address of tcpv6_prot changes */
636 if (ip_ver
== TLSV6
&&
637 unlikely(sk
->sk_prot
!= smp_load_acquire(&saved_tcpv6_prot
))) {
638 mutex_lock(&tcpv6_prot_mutex
);
639 if (likely(sk
->sk_prot
!= saved_tcpv6_prot
)) {
640 build_protos(tls_prots
[TLSV6
], sk
->sk_prot
);
641 smp_store_release(&saved_tcpv6_prot
, sk
->sk_prot
);
643 mutex_unlock(&tcpv6_prot_mutex
);
646 ctx
->conf
= TLS_BASE
;
647 update_sk_prot(sk
, ctx
);
652 void tls_register_device(struct tls_device
*device
)
654 mutex_lock(&device_mutex
);
655 list_add_tail(&device
->dev_list
, &device_list
);
656 mutex_unlock(&device_mutex
);
658 EXPORT_SYMBOL(tls_register_device
);
660 void tls_unregister_device(struct tls_device
*device
)
662 mutex_lock(&device_mutex
);
663 list_del(&device
->dev_list
);
664 mutex_unlock(&device_mutex
);
666 EXPORT_SYMBOL(tls_unregister_device
);
668 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly
= {
671 .user_visible
= true,
672 .owner
= THIS_MODULE
,
676 static int __init
tls_register(void)
678 build_protos(tls_prots
[TLSV4
], &tcp_prot
);
680 tls_sw_proto_ops
= inet_stream_ops
;
681 tls_sw_proto_ops
.poll
= tls_sw_poll
;
682 tls_sw_proto_ops
.splice_read
= tls_sw_splice_read
;
684 tcp_register_ulp(&tcp_tls_ulp_ops
);
689 static void __exit
tls_unregister(void)
691 tcp_unregister_ulp(&tcp_tls_ulp_ops
);
694 module_init(tls_register
);
695 module_exit(tls_unregister
);