[mirror_ubuntu-bionic-kernel.git] / net / tls / tls_main.c

/*
 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/module.h>

#include <net/tcp.h>
#include <net/inet_common.h>
#include <linux/highmem.h>
#include <linux/netdevice.h>
#include <linux/sched/signal.h>

#include <net/tls.h>

MODULE_AUTHOR("Mellanox Technologies");
MODULE_DESCRIPTION("Transport Layer Security Support");
MODULE_LICENSE("Dual BSD/GPL");
MODULE_ALIAS_TCP_ULP("tls");

enum {
	TLSV4,
	TLSV6,
	TLS_NUM_PROTS,
};

enum {
	TLS_BASE_TX,
	TLS_SW_TX,
	TLS_NUM_CONFIG,
};

static struct proto *saved_tcpv6_prot;
static DEFINE_MUTEX(tcpv6_prot_mutex);
static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG];

static inline void update_sk_prot(struct sock *sk, struct tls_context *ctx)
{
	int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;

	sk->sk_prot = &tls_prots[ip_ver][ctx->tx_conf];
}

int wait_on_pending_writer(struct sock *sk, long *timeo)
{
	int rc = 0;
	DEFINE_WAIT_FUNC(wait, woken_wake_function);

	add_wait_queue(sk_sleep(sk), &wait);
	while (1) {
		if (!*timeo) {
			rc = -EAGAIN;
			break;
		}

		if (signal_pending(current)) {
			rc = sock_intr_errno(*timeo);
			break;
		}

		if (sk_wait_event(sk, timeo, !sk->sk_write_pending, &wait))
			break;
	}
	remove_wait_queue(sk_sleep(sk), &wait);
	return rc;
}

int tls_push_sg(struct sock *sk,
		struct tls_context *ctx,
		struct scatterlist *sg,
		u16 first_offset,
		int flags)
{
	int sendpage_flags = flags | MSG_SENDPAGE_NOTLAST;
	int ret = 0;
	struct page *p;
	size_t size;
	int offset = first_offset;

	size = sg->length - offset;
	offset += sg->offset;

	ctx->in_tcp_sendpages = true;
	while (1) {
		if (sg_is_last(sg))
			sendpage_flags = flags;

		/* is sending application-limited? */
		tcp_rate_check_app_limited(sk);
		p = sg_page(sg);
retry:
		ret = do_tcp_sendpages(sk, p, offset, size, sendpage_flags);

		if (ret != size) {
			if (ret > 0) {
				offset += ret;
				size -= ret;
				goto retry;
			}

			offset -= sg->offset;
			ctx->partially_sent_offset = offset;
			ctx->partially_sent_record = (void *)sg;
			ctx->in_tcp_sendpages = false;
			return ret;
		}

		put_page(p);
		sk_mem_uncharge(sk, sg->length);
		sg = sg_next(sg);
		if (!sg)
			break;

		offset = sg->offset;
		size = sg->length;
	}

	clear_bit(TLS_PENDING_CLOSED_RECORD, &ctx->flags);
	ctx->in_tcp_sendpages = false;
	ctx->sk_write_space(sk);

	return 0;
}

static int tls_handle_open_record(struct sock *sk, int flags)
{
	struct tls_context *ctx = tls_get_ctx(sk);

	if (tls_is_pending_open_record(ctx))
		return ctx->push_pending_record(sk, flags);

	return 0;
}

int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
		      unsigned char *record_type)
{
	struct cmsghdr *cmsg;
	int rc = -EINVAL;

	for_each_cmsghdr(cmsg, msg) {
		if (!CMSG_OK(msg, cmsg))
			return -EINVAL;
		if (cmsg->cmsg_level != SOL_TLS)
			continue;

		switch (cmsg->cmsg_type) {
		case TLS_SET_RECORD_TYPE:
			if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type)))
				return -EINVAL;

			if (msg->msg_flags & MSG_MORE)
				return -EINVAL;

			rc = tls_handle_open_record(sk, msg->msg_flags);
			if (rc)
				return rc;

			*record_type = *(unsigned char *)CMSG_DATA(cmsg);
			rc = 0;
			break;
		default:
			return -EINVAL;
		}
	}

	return rc;
}

int tls_push_pending_closed_record(struct sock *sk, struct tls_context *ctx,
				   int flags, long *timeo)
{
	struct scatterlist *sg;
	u16 offset;

	if (!tls_is_partially_sent_record(ctx))
		return ctx->push_pending_record(sk, flags);

	sg = ctx->partially_sent_record;
	offset = ctx->partially_sent_offset;

	ctx->partially_sent_record = NULL;
	return tls_push_sg(sk, ctx, sg, offset, flags);
}

static void tls_write_space(struct sock *sk)
{
	struct tls_context *ctx = tls_get_ctx(sk);

	/* We are already sending pages, ignore notification */
	if (ctx->in_tcp_sendpages)
		return;

	if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) {
		gfp_t sk_allocation = sk->sk_allocation;
		int rc;
		long timeo = 0;

		sk->sk_allocation = GFP_ATOMIC;
		rc = tls_push_pending_closed_record(sk, ctx,
						    MSG_DONTWAIT |
						    MSG_NOSIGNAL,
						    &timeo);
		sk->sk_allocation = sk_allocation;

		if (rc < 0)
			return;
	}

	ctx->sk_write_space(sk);
}

static void tls_ctx_free(struct tls_context *ctx)
{
	if (!ctx)
		return;

	memzero_explicit(&ctx->crypto_send, sizeof(ctx->crypto_send));
	kfree(ctx);
}

static void tls_sk_proto_close(struct sock *sk, long timeout)
{
	struct tls_context *ctx = tls_get_ctx(sk);
	long timeo = sock_sndtimeo(sk, 0);
	void (*sk_proto_close)(struct sock *sk, long timeout);

	lock_sock(sk);
	sk_proto_close = ctx->sk_proto_close;

	if (ctx->tx_conf == TLS_BASE_TX) {
		tls_ctx_free(ctx);
		goto skip_tx_cleanup;
	}

	if (!tls_complete_pending_work(sk, ctx, 0, &timeo))
		tls_handle_open_record(sk, 0);

	if (ctx->partially_sent_record) {
		struct scatterlist *sg = ctx->partially_sent_record;

		while (1) {
			put_page(sg_page(sg));
			sk_mem_uncharge(sk, sg->length);

			if (sg_is_last(sg))
				break;
			sg++;
		}
	}

	kfree(ctx->rec_seq);
	kfree(ctx->iv);

	if (ctx->tx_conf == TLS_SW_TX)
		tls_sw_free_tx_resources(sk);

skip_tx_cleanup:
	release_sock(sk);
	sk_proto_close(sk, timeout);
}

static int do_tls_getsockopt_tx(struct sock *sk, char __user *optval,
				int __user *optlen)
{
	int rc = 0;
	struct tls_context *ctx = tls_get_ctx(sk);
	struct tls_crypto_info *crypto_info;
	int len;

	if (get_user(len, optlen))
		return -EFAULT;

	if (!optval || (len < sizeof(*crypto_info))) {
		rc = -EINVAL;
		goto out;
	}

	if (!ctx) {
		rc = -EBUSY;
		goto out;
	}

	/* get user crypto info */
	crypto_info = &ctx->crypto_send.info;

	if (!TLS_CRYPTO_INFO_READY(crypto_info)) {
		rc = -EBUSY;
		goto out;
	}

	if (len == sizeof(*crypto_info)) {
		if (copy_to_user(optval, crypto_info, sizeof(*crypto_info)))
			rc = -EFAULT;
		goto out;
	}

	switch (crypto_info->cipher_type) {
	case TLS_CIPHER_AES_GCM_128: {
		struct tls12_crypto_info_aes_gcm_128 *
		  crypto_info_aes_gcm_128 =
		  container_of(crypto_info,
			       struct tls12_crypto_info_aes_gcm_128,
			       info);

		if (len != sizeof(*crypto_info_aes_gcm_128)) {
			rc = -EINVAL;
			goto out;
		}
		lock_sock(sk);
		memcpy(crypto_info_aes_gcm_128->iv,
		       ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
		       TLS_CIPHER_AES_GCM_128_IV_SIZE);
		release_sock(sk);
		if (copy_to_user(optval,
				 crypto_info_aes_gcm_128,
				 sizeof(*crypto_info_aes_gcm_128)))
			rc = -EFAULT;
		break;
	}
	default:
		rc = -EINVAL;
	}

out:
	return rc;
}

static int do_tls_getsockopt(struct sock *sk, int optname,
			     char __user *optval, int __user *optlen)
{
	int rc = 0;

	switch (optname) {
	case TLS_TX:
		rc = do_tls_getsockopt_tx(sk, optval, optlen);
		break;
	default:
		rc = -ENOPROTOOPT;
		break;
	}
	return rc;
}

static int tls_getsockopt(struct sock *sk, int level, int optname,
			  char __user *optval, int __user *optlen)
{
	struct tls_context *ctx = tls_get_ctx(sk);

	if (level != SOL_TLS)
		return ctx->getsockopt(sk, level, optname, optval, optlen);

	return do_tls_getsockopt(sk, optname, optval, optlen);
}

static int do_tls_setsockopt_tx(struct sock *sk, char __user *optval,
				unsigned int optlen)
{
	struct tls_crypto_info *crypto_info;
	struct tls_context *ctx = tls_get_ctx(sk);
	int rc = 0;
	int tx_conf;

	if (!optval || (optlen < sizeof(*crypto_info))) {
		rc = -EINVAL;
		goto out;
	}

	/* get user crypto info */
	crypto_info = &ctx->crypto_send.info;

	/* Currently we don't support set crypto info more than one time */
	if (TLS_CRYPTO_INFO_READY(crypto_info)) {
		rc = -EBUSY;
		goto out;
	}

	rc = copy_from_user(crypto_info, optval, sizeof(*crypto_info));
	if (rc) {
		rc = -EFAULT;
		goto out;
	}

	/* check version */
	if (crypto_info->version != TLS_1_2_VERSION) {
		rc = -ENOTSUPP;
		goto err_crypto_info;
	}

	switch (crypto_info->cipher_type) {
	case TLS_CIPHER_AES_GCM_128: {
		if (optlen != sizeof(struct tls12_crypto_info_aes_gcm_128)) {
			rc = -EINVAL;
			goto err_crypto_info;
		}
		rc = copy_from_user(crypto_info + 1, optval + sizeof(*crypto_info),
				    optlen - sizeof(*crypto_info));
		if (rc) {
			rc = -EFAULT;
			goto err_crypto_info;
		}
		break;
	}
	default:
		rc = -EINVAL;
		goto err_crypto_info;
	}

	/* currently SW is default, we will have ethtool in future */
	rc = tls_set_sw_offload(sk, ctx);
	tx_conf = TLS_SW_TX;
	if (rc)
		goto err_crypto_info;

	ctx->tx_conf = tx_conf;
	update_sk_prot(sk, ctx);
	ctx->sk_write_space = sk->sk_write_space;
	sk->sk_write_space = tls_write_space;
	goto out;

err_crypto_info:
	memset(crypto_info, 0, sizeof(*crypto_info));
out:
	return rc;
}

static int do_tls_setsockopt(struct sock *sk, int optname,
			     char __user *optval, unsigned int optlen)
{
	int rc = 0;

	switch (optname) {
	case TLS_TX:
		lock_sock(sk);
		rc = do_tls_setsockopt_tx(sk, optval, optlen);
		release_sock(sk);
		break;
	default:
		rc = -ENOPROTOOPT;
		break;
	}
	return rc;
}

static int tls_setsockopt(struct sock *sk, int level, int optname,
			  char __user *optval, unsigned int optlen)
{
	struct tls_context *ctx = tls_get_ctx(sk);

	if (level != SOL_TLS)
		return ctx->setsockopt(sk, level, optname, optval, optlen);

	return do_tls_setsockopt(sk, optname, optval, optlen);
}

static void build_protos(struct proto *prot, struct proto *base)
{
	prot[TLS_BASE_TX] = *base;
	prot[TLS_BASE_TX].setsockopt	= tls_setsockopt;
	prot[TLS_BASE_TX].getsockopt	= tls_getsockopt;
	prot[TLS_BASE_TX].close		= tls_sk_proto_close;

	prot[TLS_SW_TX] = prot[TLS_BASE_TX];
	prot[TLS_SW_TX].sendmsg		= tls_sw_sendmsg;
	prot[TLS_SW_TX].sendpage	= tls_sw_sendpage;
}

static int tls_init(struct sock *sk)
{
	int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
	struct inet_connection_sock *icsk = inet_csk(sk);
	struct tls_context *ctx;
	int rc = 0;

	/* The TLS ulp is currently supported only for TCP sockets
	 * in ESTABLISHED state.
	 * Supporting sockets in LISTEN state will require us
	 * to modify the accept implementation to clone rather then
	 * share the ulp context.
	 */
	if (sk->sk_state != TCP_ESTABLISHED)
		return -ENOTSUPP;

	/* allocate tls context */
	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	if (!ctx) {
		rc = -ENOMEM;
		goto out;
	}
	icsk->icsk_ulp_data = ctx;
	ctx->setsockopt = sk->sk_prot->setsockopt;
	ctx->getsockopt = sk->sk_prot->getsockopt;
	ctx->sk_proto_close = sk->sk_prot->close;

	/* Build IPv6 TLS whenever the address of tcpv6_prot changes */
	if (ip_ver == TLSV6 &&
	    unlikely(sk->sk_prot != smp_load_acquire(&saved_tcpv6_prot))) {
		mutex_lock(&tcpv6_prot_mutex);
		if (likely(sk->sk_prot != saved_tcpv6_prot)) {
			build_protos(tls_prots[TLSV6], sk->sk_prot);
			smp_store_release(&saved_tcpv6_prot, sk->sk_prot);
		}
		mutex_unlock(&tcpv6_prot_mutex);
	}

	ctx->tx_conf = TLS_BASE_TX;
	update_sk_prot(sk, ctx);
out:
	return rc;
}

static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = {
	.name			= "tls",
	.owner			= THIS_MODULE,
	.init			= tls_init,
};

static int __init tls_register(void)
{
	build_protos(tls_prots[TLSV4], &tcp_prot);

	tcp_register_ulp(&tcp_tls_ulp_ops);

	return 0;
}

static void __exit tls_unregister(void)
{
	tcp_unregister_ulp(&tcp_tls_ulp_ops);
}

module_init(tls_register);
module_exit(tls_unregister);
Commit	Line	Data
3c4d7559 DW	1	/*
	2	* Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
	3	* Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
	4	*
	5	* This software is available to you under a choice of one of two
	6	* licenses. You may choose to be licensed under the terms of the GNU
	7	* General Public License (GPL) Version 2, available from the file
	8	* COPYING in the main directory of this source tree, or the
	9	* OpenIB.org BSD license below:
	10	*
	11	* Redistribution and use in source and binary forms, with or
	12	* without modification, are permitted provided that the following
	13	* conditions are met:
	14	*
	15	* - Redistributions of source code must retain the above
	16	* copyright notice, this list of conditions and the following
	17	* disclaimer.
	18	*
	19	* - Redistributions in binary form must reproduce the above
	20	* copyright notice, this list of conditions and the following
	21	* disclaimer in the documentation and/or other materials
	22	* provided with the distribution.
	23	*
	24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	31	* SOFTWARE.
	32	*/
	33
	34	#include <linux/module.h>
	35
	36	#include <net/tcp.h>
	37	#include <net/inet_common.h>
	38	#include <linux/highmem.h>
	39	#include <linux/netdevice.h>
	40	#include <linux/sched/signal.h>
	41
	42	#include <net/tls.h>
	43
	44	MODULE_AUTHOR("Mellanox Technologies");
	45	MODULE_DESCRIPTION("Transport Layer Security Support");
	46	MODULE_LICENSE("Dual BSD/GPL");
a9b71cc1	47	MODULE_ALIAS_TCP_ULP("tls");
3c4d7559	48
fa4baf63 BP	49	enum {
	50	TLSV4,
	51	TLSV6,
	52	TLS_NUM_PROTS,
	53	};
	54
6d88207f IL	55	enum {
	56	TLS_BASE_TX,
	57	TLS_SW_TX,
	58	TLS_NUM_CONFIG,
	59	};
	60
fa4baf63 BP	61	static struct proto *saved_tcpv6_prot;
	62	static DEFINE_MUTEX(tcpv6_prot_mutex);
	63	static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG];
6d88207f IL	64
	65	static inline void update_sk_prot(struct sock sk, struct tls_context ctx)
	66	{
fa4baf63 BP	67	int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
	68
	69	sk->sk_prot = &tls_prots[ip_ver][ctx->tx_conf];
6d88207f	70	}
3c4d7559 DW	71
	72	int wait_on_pending_writer(struct sock sk, long timeo)
	73	{
	74	int rc = 0;
	75	DEFINE_WAIT_FUNC(wait, woken_wake_function);
	76
	77	add_wait_queue(sk_sleep(sk), &wait);
	78	while (1) {
	79	if (!*timeo) {
	80	rc = -EAGAIN;
	81	break;
	82	}
	83
	84	if (signal_pending(current)) {
	85	rc = sock_intr_errno(*timeo);
	86	break;
	87	}
	88
	89	if (sk_wait_event(sk, timeo, !sk->sk_write_pending, &wait))
	90	break;
	91	}
	92	remove_wait_queue(sk_sleep(sk), &wait);
	93	return rc;
	94	}
	95
	96	int tls_push_sg(struct sock *sk,
	97	struct tls_context *ctx,
	98	struct scatterlist *sg,
	99	u16 first_offset,
	100	int flags)
	101	{
	102	int sendpage_flags = flags \| MSG_SENDPAGE_NOTLAST;
	103	int ret = 0;
	104	struct page *p;
	105	size_t size;
	106	int offset = first_offset;
	107
	108	size = sg->length - offset;
	109	offset += sg->offset;
	110
145c3455	111	ctx->in_tcp_sendpages = true;
3c4d7559 DW	112	while (1) {
	113	if (sg_is_last(sg))
	114	sendpage_flags = flags;
	115
	116	/* is sending application-limited? */
	117	tcp_rate_check_app_limited(sk);
	118	p = sg_page(sg);
	119	retry:
	120	ret = do_tcp_sendpages(sk, p, offset, size, sendpage_flags);
	121
	122	if (ret != size) {
	123	if (ret > 0) {
	124	offset += ret;
	125	size -= ret;
	126	goto retry;
	127	}
	128
	129	offset -= sg->offset;
	130	ctx->partially_sent_offset = offset;
	131	ctx->partially_sent_record = (void *)sg;
4dc89a1e	132	ctx->in_tcp_sendpages = false;
3c4d7559 DW	133	return ret;
	134	}
	135
	136	put_page(p);
	137	sk_mem_uncharge(sk, sg->length);
	138	sg = sg_next(sg);
	139	if (!sg)
	140	break;
	141
	142	offset = sg->offset;
	143	size = sg->length;
	144	}
	145
	146	clear_bit(TLS_PENDING_CLOSED_RECORD, &ctx->flags);
145c3455 DW	147	ctx->in_tcp_sendpages = false;
145c3455 DW	148	ctx->sk_write_space(sk);
3c4d7559 DW	149
	150	return 0;
	151	}
	152
	153	static int tls_handle_open_record(struct sock *sk, int flags)
	154	{
	155	struct tls_context *ctx = tls_get_ctx(sk);
	156
	157	if (tls_is_pending_open_record(ctx))
	158	return ctx->push_pending_record(sk, flags);
	159
	160	return 0;
	161	}
	162
	163	int tls_proccess_cmsg(struct sock sk, struct msghdr msg,
	164	unsigned char *record_type)
	165	{
	166	struct cmsghdr *cmsg;
	167	int rc = -EINVAL;
	168
	169	for_each_cmsghdr(cmsg, msg) {
	170	if (!CMSG_OK(msg, cmsg))
	171	return -EINVAL;
	172	if (cmsg->cmsg_level != SOL_TLS)
	173	continue;
	174
	175	switch (cmsg->cmsg_type) {
	176	case TLS_SET_RECORD_TYPE:
	177	if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type)))
	178	return -EINVAL;
	179
	180	if (msg->msg_flags & MSG_MORE)
	181	return -EINVAL;
	182
	183	rc = tls_handle_open_record(sk, msg->msg_flags);
	184	if (rc)
	185	return rc;
	186
	187	record_type = (unsigned char *)CMSG_DATA(cmsg);
	188	rc = 0;
	189	break;
	190	default:
	191	return -EINVAL;
	192	}
	193	}
	194
	195	return rc;
	196	}
	197
	198	int tls_push_pending_closed_record(struct sock sk, struct tls_context ctx,
	199	int flags, long *timeo)
	200	{
	201	struct scatterlist *sg;
	202	u16 offset;
	203
	204	if (!tls_is_partially_sent_record(ctx))
	205	return ctx->push_pending_record(sk, flags);
	206
	207	sg = ctx->partially_sent_record;
	208	offset = ctx->partially_sent_offset;
	209
	210	ctx->partially_sent_record = NULL;
	211	return tls_push_sg(sk, ctx, sg, offset, flags);
	212	}
213
214	static void tls_write_space(struct sock *sk)
215	{
216	struct tls_context *ctx = tls_get_ctx(sk);
217
145c3455 DW	218	/* We are already sending pages, ignore notification */
	219	if (ctx->in_tcp_sendpages)
	220	return;
	221
3c4d7559 DW	222	if (!sk->sk_write_pending && tls_is_pending_closed_record(ctx)) {
	223	gfp_t sk_allocation = sk->sk_allocation;
	224	int rc;
	225	long timeo = 0;
	226
	227	sk->sk_allocation = GFP_ATOMIC;
	228	rc = tls_push_pending_closed_record(sk, ctx,
	229	MSG_DONTWAIT \|
	230	MSG_NOSIGNAL,
	231	&timeo);
	232	sk->sk_allocation = sk_allocation;
	233
	234	if (rc < 0)
	235	return;
	236	}
	237
	238	ctx->sk_write_space(sk);
	239	}
	240
028e5727 SD	241	static void tls_ctx_free(struct tls_context *ctx)
	242	{
	243	if (!ctx)
	244	return;
	245
	246	memzero_explicit(&ctx->crypto_send, sizeof(ctx->crypto_send));
	247	kfree(ctx);
	248	}
	249
3c4d7559 DW	250	static void tls_sk_proto_close(struct sock *sk, long timeout)
	251	{
	252	struct tls_context *ctx = tls_get_ctx(sk);
	253	long timeo = sock_sndtimeo(sk, 0);
	254	void (sk_proto_close)(struct sock sk, long timeout);
	255
	256	lock_sock(sk);
ff45d820 IL	257	sk_proto_close = ctx->sk_proto_close;
	258
	259	if (ctx->tx_conf == TLS_BASE_TX) {
028e5727	260	tls_ctx_free(ctx);
ff45d820 IL	261	goto skip_tx_cleanup;
ff45d820 IL	262	}
3c4d7559 DW	263
	264	if (!tls_complete_pending_work(sk, ctx, 0, &timeo))
	265	tls_handle_open_record(sk, 0);
	266
	267	if (ctx->partially_sent_record) {
	268	struct scatterlist *sg = ctx->partially_sent_record;
	269
	270	while (1) {
	271	put_page(sg_page(sg));
	272	sk_mem_uncharge(sk, sg->length);
	273
	274	if (sg_is_last(sg))
	275	break;
	276	sg++;
	277	}
	278	}
ff45d820	279
3c4d7559 DW	280	kfree(ctx->rec_seq);
	281	kfree(ctx->iv);
	282
ff45d820 IL	283	if (ctx->tx_conf == TLS_SW_TX)
ff45d820 IL	284	tls_sw_free_tx_resources(sk);
3c4d7559	285
ff45d820	286	skip_tx_cleanup:
3c4d7559 DW	287	release_sock(sk);
	288	sk_proto_close(sk, timeout);
	289	}
	290
	291	static int do_tls_getsockopt_tx(struct sock sk, char __user optval,
	292	int __user *optlen)
	293	{
	294	int rc = 0;
	295	struct tls_context *ctx = tls_get_ctx(sk);
	296	struct tls_crypto_info *crypto_info;
	297	int len;
	298
	299	if (get_user(len, optlen))
	300	return -EFAULT;
	301
	302	if (!optval \|\| (len < sizeof(*crypto_info))) {
	303	rc = -EINVAL;
	304	goto out;
	305	}
	306
	307	if (!ctx) {
	308	rc = -EBUSY;
	309	goto out;
	310	}
	311
	312	/* get user crypto info */
028e5727	313	crypto_info = &ctx->crypto_send.info;
3c4d7559 DW	314
	315	if (!TLS_CRYPTO_INFO_READY(crypto_info)) {
	316	rc = -EBUSY;
	317	goto out;
	318	}
	319
5a3b886c	320	if (len == sizeof(*crypto_info)) {
ac55cd61 DC	321	if (copy_to_user(optval, crypto_info, sizeof(*crypto_info)))
ac55cd61 DC	322	rc = -EFAULT;
3c4d7559 DW	323	goto out;
	324	}
	325
	326	switch (crypto_info->cipher_type) {
	327	case TLS_CIPHER_AES_GCM_128: {
	328	struct tls12_crypto_info_aes_gcm_128 *
	329	crypto_info_aes_gcm_128 =
	330	container_of(crypto_info,
	331	struct tls12_crypto_info_aes_gcm_128,
	332	info);
	333
	334	if (len != sizeof(*crypto_info_aes_gcm_128)) {
	335	rc = -EINVAL;
	336	goto out;
	337	}
	338	lock_sock(sk);
5245eac4 BP	339	memcpy(crypto_info_aes_gcm_128->iv,
5245eac4 BP	340	ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
3c4d7559 DW	341	TLS_CIPHER_AES_GCM_128_IV_SIZE);
3c4d7559 DW	342	release_sock(sk);
ac55cd61 DC	343	if (copy_to_user(optval,
	344	crypto_info_aes_gcm_128,
	345	sizeof(*crypto_info_aes_gcm_128)))
	346	rc = -EFAULT;
3c4d7559 DW	347	break;
	348	}
	349	default:
	350	rc = -EINVAL;
	351	}
	352
	353	out:
	354	return rc;
	355	}
	356
	357	static int do_tls_getsockopt(struct sock *sk, int optname,
	358	char __user optval, int __user optlen)
	359	{
	360	int rc = 0;
	361
	362	switch (optname) {
	363	case TLS_TX:
	364	rc = do_tls_getsockopt_tx(sk, optval, optlen);
	365	break;
	366	default:
	367	rc = -ENOPROTOOPT;
	368	break;
	369	}
	370	return rc;
	371	}
	372
	373	static int tls_getsockopt(struct sock *sk, int level, int optname,
	374	char __user optval, int __user optlen)
	375	{
	376	struct tls_context *ctx = tls_get_ctx(sk);
	377
	378	if (level != SOL_TLS)
	379	return ctx->getsockopt(sk, level, optname, optval, optlen);
	380
	381	return do_tls_getsockopt(sk, optname, optval, optlen);
	382	}
	383
	384	static int do_tls_setsockopt_tx(struct sock sk, char __user optval,
	385	unsigned int optlen)
	386	{
196c31b4	387	struct tls_crypto_info *crypto_info;
3c4d7559	388	struct tls_context *ctx = tls_get_ctx(sk);
3c4d7559	389	int rc = 0;
6d88207f	390	int tx_conf;
3c4d7559 DW	391
	392	if (!optval \|\| (optlen < sizeof(*crypto_info))) {
	393	rc = -EINVAL;
	394	goto out;
	395	}
	396
028e5727 SD	397	/* get user crypto info */
	398	crypto_info = &ctx->crypto_send.info;
	399
196c31b4	400	/* Currently we don't support set crypto info more than one time */
877d17c7 SD	401	if (TLS_CRYPTO_INFO_READY(crypto_info)) {
877d17c7 SD	402	rc = -EBUSY;
196c31b4	403	goto out;
877d17c7	404	}
196c31b4 IL	405
196c31b4 IL	406	rc = copy_from_user(crypto_info, optval, sizeof(*crypto_info));
3c4d7559 DW	407	if (rc) {
	408	rc = -EFAULT;
	409	goto out;
	410	}
	411
	412	/* check version */
196c31b4	413	if (crypto_info->version != TLS_1_2_VERSION) {
3c4d7559	414	rc = -ENOTSUPP;
196c31b4	415	goto err_crypto_info;
3c4d7559 DW	416	}
3c4d7559 DW	417
196c31b4	418	switch (crypto_info->cipher_type) {
3c4d7559 DW	419	case TLS_CIPHER_AES_GCM_128: {
	420	if (optlen != sizeof(struct tls12_crypto_info_aes_gcm_128)) {
	421	rc = -EINVAL;
6db959c8	422	goto err_crypto_info;
3c4d7559	423	}
196c31b4 IL	424	rc = copy_from_user(crypto_info + 1, optval + sizeof(*crypto_info),
196c31b4 IL	425	optlen - sizeof(*crypto_info));
3c4d7559 DW	426	if (rc) {
	427	rc = -EFAULT;
	428	goto err_crypto_info;
	429	}
	430	break;
	431	}
	432	default:
	433	rc = -EINVAL;
6db959c8	434	goto err_crypto_info;
3c4d7559 DW	435	}
3c4d7559 DW	436
3c4d7559 DW	437	/* currently SW is default, we will have ethtool in future */
3c4d7559 DW	438	rc = tls_set_sw_offload(sk, ctx);
6d88207f	439	tx_conf = TLS_SW_TX;
3c4d7559 DW	440	if (rc)
	441	goto err_crypto_info;
	442
6d88207f IL	443	ctx->tx_conf = tx_conf;
6d88207f IL	444	update_sk_prot(sk, ctx);
ee181e52 IL	445	ctx->sk_write_space = sk->sk_write_space;
ee181e52 IL	446	sk->sk_write_space = tls_write_space;
3c4d7559 DW	447	goto out;
	448
	449	err_crypto_info:
	450	memset(crypto_info, 0, sizeof(*crypto_info));
	451	out:
	452	return rc;
	453	}
	454
	455	static int do_tls_setsockopt(struct sock *sk, int optname,
	456	char __user *optval, unsigned int optlen)
	457	{
	458	int rc = 0;
	459
	460	switch (optname) {
	461	case TLS_TX:
	462	lock_sock(sk);
	463	rc = do_tls_setsockopt_tx(sk, optval, optlen);
	464	release_sock(sk);
	465	break;
	466	default:
	467	rc = -ENOPROTOOPT;
	468	break;
	469	}
	470	return rc;
	471	}
	472
	473	static int tls_setsockopt(struct sock *sk, int level, int optname,
	474	char __user *optval, unsigned int optlen)
	475	{
	476	struct tls_context *ctx = tls_get_ctx(sk);
	477
	478	if (level != SOL_TLS)
	479	return ctx->setsockopt(sk, level, optname, optval, optlen);
	480
	481	return do_tls_setsockopt(sk, optname, optval, optlen);
	482	}
	483
fa4baf63 BP	484	static void build_protos(struct proto prot, struct proto base)
	485	{
	486	prot[TLS_BASE_TX] = *base;
	487	prot[TLS_BASE_TX].setsockopt = tls_setsockopt;
	488	prot[TLS_BASE_TX].getsockopt = tls_getsockopt;
	489	prot[TLS_BASE_TX].close = tls_sk_proto_close;
	490
	491	prot[TLS_SW_TX] = prot[TLS_BASE_TX];
	492	prot[TLS_SW_TX].sendmsg = tls_sw_sendmsg;
	493	prot[TLS_SW_TX].sendpage = tls_sw_sendpage;
	494	}
	495
3c4d7559 DW	496	static int tls_init(struct sock *sk)
3c4d7559 DW	497	{
fa4baf63	498	int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
3c4d7559 DW	499	struct inet_connection_sock *icsk = inet_csk(sk);
	500	struct tls_context *ctx;
	501	int rc = 0;
	502
d91c3e17 IL	503	/* The TLS ulp is currently supported only for TCP sockets
	504	* in ESTABLISHED state.
	505	* Supporting sockets in LISTEN state will require us
	506	* to modify the accept implementation to clone rather then
	507	* share the ulp context.
	508	*/
	509	if (sk->sk_state != TCP_ESTABLISHED)
	510	return -ENOTSUPP;
	511
3c4d7559 DW	512	/* allocate tls context */
	513	ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
	514	if (!ctx) {
	515	rc = -ENOMEM;
	516	goto out;
	517	}
	518	icsk->icsk_ulp_data = ctx;
	519	ctx->setsockopt = sk->sk_prot->setsockopt;
	520	ctx->getsockopt = sk->sk_prot->getsockopt;
ff45d820	521	ctx->sk_proto_close = sk->sk_prot->close;
6d88207f	522
fa4baf63 BP	523	/* Build IPv6 TLS whenever the address of tcpv6_prot changes */
	524	if (ip_ver == TLSV6 &&
	525	unlikely(sk->sk_prot != smp_load_acquire(&saved_tcpv6_prot))) {
	526	mutex_lock(&tcpv6_prot_mutex);
	527	if (likely(sk->sk_prot != saved_tcpv6_prot)) {
	528	build_protos(tls_prots[TLSV6], sk->sk_prot);
	529	smp_store_release(&saved_tcpv6_prot, sk->sk_prot);
	530	}
	531	mutex_unlock(&tcpv6_prot_mutex);
	532	}
	533
6d88207f IL	534	ctx->tx_conf = TLS_BASE_TX;
6d88207f IL	535	update_sk_prot(sk, ctx);
3c4d7559 DW	536	out:
	537	return rc;
	538	}
	539
	540	static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = {
	541	.name = "tls",
	542	.owner = THIS_MODULE,
	543	.init = tls_init,
	544	};
	545
	546	static int __init tls_register(void)
	547	{
fa4baf63	548	build_protos(tls_prots[TLSV4], &tcp_prot);
3c4d7559 DW	549
	550	tcp_register_ulp(&tcp_tls_ulp_ops);
	551
	552	return 0;
	553	}
	554
	555	static void __exit tls_unregister(void)
	556	{
	557	tcp_unregister_ulp(&tcp_tls_ulp_ops);
	558	}
	559
	560	module_init(tls_register);
	561	module_exit(tls_unregister);