[mirror_ubuntu-bionic-kernel.git] / net / smc / smc_tx.c

/*
 * Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 * Manage send buffer.
 * Producer:
 * Copy user space data into send buffer, if send buffer space available.
 * Consumer:
 * Trigger RDMA write into RMBE of peer and send CDC, if RMBE space available.
 *
 * Copyright IBM Corp. 2016
 *
 * Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
 */

#include <linux/net.h>
#include <linux/rcupdate.h>
#include <linux/workqueue.h>
#include <linux/sched/signal.h>

#include <net/sock.h>

#include "smc.h"
#include "smc_wr.h"
#include "smc_cdc.h"
#include "smc_tx.h"

#define SMC_TX_WORK_DELAY	HZ

/***************************** sndbuf producer *******************************/

/* callback implementation for sk.sk_write_space()
 * to wakeup sndbuf producers that blocked with smc_tx_wait_memory().
 * called under sk_socket lock.
 */
static void smc_tx_write_space(struct sock *sk)
{
	struct socket *sock = sk->sk_socket;
	struct smc_sock *smc = smc_sk(sk);
	struct socket_wq *wq;

	/* similar to sk_stream_write_space */
	if (atomic_read(&smc->conn.sndbuf_space) && sock) {
		clear_bit(SOCK_NOSPACE, &sock->flags);
		rcu_read_lock();
		wq = rcu_dereference(sk->sk_wq);
		if (skwq_has_sleeper(wq))
			wake_up_interruptible_poll(&wq->wait,
						   POLLOUT | POLLWRNORM |
						   POLLWRBAND);
		if (wq && wq->fasync_list && !(sk->sk_shutdown & SEND_SHUTDOWN))
			sock_wake_async(wq, SOCK_WAKE_SPACE, POLL_OUT);
		rcu_read_unlock();
	}
}

/* Wakeup sndbuf producers that blocked with smc_tx_wait_memory().
 * Cf. tcp_data_snd_check()=>tcp_check_space()=>tcp_new_space().
 */
void smc_tx_sndbuf_nonfull(struct smc_sock *smc)
{
	if (smc->sk.sk_socket &&
	    test_bit(SOCK_NOSPACE, &smc->sk.sk_socket->flags))
		smc->sk.sk_write_space(&smc->sk);
}

/* blocks sndbuf producer until at least one byte of free space available */
static int smc_tx_wait_memory(struct smc_sock *smc, int flags)
{
	DEFINE_WAIT_FUNC(wait, woken_wake_function);
	struct smc_connection *conn = &smc->conn;
	struct sock *sk = &smc->sk;
	bool noblock;
	long timeo;
	int rc = 0;

	/* similar to sk_stream_wait_memory */
	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
	noblock = timeo ? false : true;
	add_wait_queue(sk_sleep(sk), &wait);
	while (1) {
		sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
		if (sk->sk_err ||
		    (sk->sk_shutdown & SEND_SHUTDOWN) ||
		    conn->local_tx_ctrl.conn_state_flags.peer_done_writing) {
			rc = -EPIPE;
			break;
		}
		if (conn->local_rx_ctrl.conn_state_flags.peer_conn_abort) {
			rc = -ECONNRESET;
			break;
		}
		if (!timeo) {
			if (noblock)
				set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
			rc = -EAGAIN;
			break;
		}
		if (signal_pending(current)) {
			rc = sock_intr_errno(timeo);
			break;
		}
		sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
		if (atomic_read(&conn->sndbuf_space))
			break; /* at least 1 byte of free space available */
		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
		sk->sk_write_pending++;
		sk_wait_event(sk, &timeo,
			      sk->sk_err ||
			      (sk->sk_shutdown & SEND_SHUTDOWN) ||
			      smc_cdc_rxed_any_close_or_senddone(conn) ||
			      atomic_read(&conn->sndbuf_space),
			      &wait);
		sk->sk_write_pending--;
	}
	remove_wait_queue(sk_sleep(sk), &wait);
	return rc;
}

/* sndbuf producer: main API called by socket layer.
 * called under sock lock.
 */
int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len)
{
	size_t copylen, send_done = 0, send_remaining = len;
	size_t chunk_len, chunk_off, chunk_len_sum;
	struct smc_connection *conn = &smc->conn;
	union smc_host_cursor prep;
	struct sock *sk = &smc->sk;
	char *sndbuf_base;
	int tx_cnt_prep;
	int writespace;
	int rc, chunk;

	/* This should be in poll */
	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);

	if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) {
		rc = -EPIPE;
		goto out_err;
	}

	while (msg_data_left(msg)) {
		if (sk->sk_state == SMC_INIT)
			return -ENOTCONN;
		if (smc->sk.sk_shutdown & SEND_SHUTDOWN ||
		    (smc->sk.sk_err == ECONNABORTED) ||
		    conn->local_tx_ctrl.conn_state_flags.peer_conn_abort)
			return -EPIPE;
		if (smc_cdc_rxed_any_close(conn))
			return send_done ?: -ECONNRESET;

		if (!atomic_read(&conn->sndbuf_space)) {
			rc = smc_tx_wait_memory(smc, msg->msg_flags);
			if (rc) {
				if (send_done)
					return send_done;
				goto out_err;
			}
			continue;
		}

		/* initialize variables for 1st iteration of subsequent loop */
		/* could be just 1 byte, even after smc_tx_wait_memory above */
		writespace = atomic_read(&conn->sndbuf_space);
		/* not more than what user space asked for */
		copylen = min_t(size_t, send_remaining, writespace);
		/* determine start of sndbuf */
		sndbuf_base = conn->sndbuf_desc->cpu_addr;
		smc_curs_write(&prep,
			       smc_curs_read(&conn->tx_curs_prep, conn),
			       conn);
		tx_cnt_prep = prep.count;
		/* determine chunks where to write into sndbuf */
		/* either unwrapped case, or 1st chunk of wrapped case */
		chunk_len = min_t(size_t,
				  copylen, conn->sndbuf_size - tx_cnt_prep);
		chunk_len_sum = chunk_len;
		chunk_off = tx_cnt_prep;
		smc_sndbuf_sync_sg_for_cpu(conn);
		for (chunk = 0; chunk < 2; chunk++) {
			rc = memcpy_from_msg(sndbuf_base + chunk_off,
					     msg, chunk_len);
			if (rc) {
				smc_sndbuf_sync_sg_for_device(conn);
				if (send_done)
					return send_done;
				goto out_err;
			}
			send_done += chunk_len;
			send_remaining -= chunk_len;

			if (chunk_len_sum == copylen)
				break; /* either on 1st or 2nd iteration */
			/* prepare next (== 2nd) iteration */
			chunk_len = copylen - chunk_len; /* remainder */
			chunk_len_sum += chunk_len;
			chunk_off = 0; /* modulo offset in send ring buffer */
		}
		smc_sndbuf_sync_sg_for_device(conn);
		/* update cursors */
		smc_curs_add(conn->sndbuf_size, &prep, copylen);
		smc_curs_write(&conn->tx_curs_prep,
			       smc_curs_read(&prep, conn),
			       conn);
		/* increased in send tasklet smc_cdc_tx_handler() */
		smp_mb__before_atomic();
		atomic_sub(copylen, &conn->sndbuf_space);
		/* guarantee 0 <= sndbuf_space <= sndbuf_size */
		smp_mb__after_atomic();
		/* since we just produced more new data into sndbuf,
		 * trigger sndbuf consumer: RDMA write into peer RMBE and CDC
		 */
		smc_tx_sndbuf_nonempty(conn);
	} /* while (msg_data_left(msg)) */

	return send_done;

out_err:
	rc = sk_stream_error(sk, msg->msg_flags, rc);
	/* make sure we wake any epoll edge trigger waiter */
	if (unlikely(rc == -EAGAIN))
		sk->sk_write_space(sk);
	return rc;
}

/***************************** sndbuf consumer *******************************/

/* sndbuf consumer: actual data transfer of one target chunk with RDMA write */
static int smc_tx_rdma_write(struct smc_connection *conn, int peer_rmbe_offset,
			     int num_sges, struct ib_sge sges[])
{
	struct smc_link_group *lgr = conn->lgr;
	struct ib_send_wr *failed_wr = NULL;
	struct ib_rdma_wr rdma_wr;
	struct smc_link *link;
	int rc;

	memset(&rdma_wr, 0, sizeof(rdma_wr));
	link = &lgr->lnk[SMC_SINGLE_LINK];
	rdma_wr.wr.wr_id = smc_wr_tx_get_next_wr_id(link);
	rdma_wr.wr.sg_list = sges;
	rdma_wr.wr.num_sge = num_sges;
	rdma_wr.wr.opcode = IB_WR_RDMA_WRITE;
	rdma_wr.remote_addr =
		lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].dma_addr +
		/* RMBE within RMB */
		((conn->peer_conn_idx - 1) * conn->peer_rmbe_size) +
		/* offset within RMBE */
		peer_rmbe_offset;
	rdma_wr.rkey = lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].rkey;
	rc = ib_post_send(link->roce_qp, &rdma_wr.wr, &failed_wr);
	if (rc)
		conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;
	return rc;
}

/* sndbuf consumer */
static inline void smc_tx_advance_cursors(struct smc_connection *conn,
					  union smc_host_cursor *prod,
					  union smc_host_cursor *sent,
					  size_t len)
{
	smc_curs_add(conn->peer_rmbe_size, prod, len);
	/* increased in recv tasklet smc_cdc_msg_rcv() */
	smp_mb__before_atomic();
	/* data in flight reduces usable snd_wnd */
	atomic_sub(len, &conn->peer_rmbe_space);
	/* guarantee 0 <= peer_rmbe_space <= peer_rmbe_size */
	smp_mb__after_atomic();
	smc_curs_add(conn->sndbuf_size, sent, len);
}

/* sndbuf consumer: prepare all necessary (src&dst) chunks of data transmit;
 * usable snd_wnd as max transmit
 */
static int smc_tx_rdma_writes(struct smc_connection *conn)
{
	size_t src_off, src_len, dst_off, dst_len; /* current chunk values */
	size_t len, dst_len_sum, src_len_sum, dstchunk, srcchunk;
	union smc_host_cursor sent, prep, prod, cons;
	struct ib_sge sges[SMC_IB_MAX_SEND_SGE];
	struct smc_link_group *lgr = conn->lgr;
	int to_send, rmbespace;
	struct smc_link *link;
	dma_addr_t dma_addr;
	int num_sges;
	int rc;

	/* source: sndbuf */
	smc_curs_write(&sent, smc_curs_read(&conn->tx_curs_sent, conn), conn);
	smc_curs_write(&prep, smc_curs_read(&conn->tx_curs_prep, conn), conn);
	/* cf. wmem_alloc - (snd_max - snd_una) */
	to_send = smc_curs_diff(conn->sndbuf_size, &sent, &prep);
	if (to_send <= 0)
		return 0;

	/* destination: RMBE */
	/* cf. snd_wnd */
	rmbespace = atomic_read(&conn->peer_rmbe_space);
	if (rmbespace <= 0)
		return 0;
	smc_curs_write(&prod,
		       smc_curs_read(&conn->local_tx_ctrl.prod, conn),
		       conn);
	smc_curs_write(&cons,
		       smc_curs_read(&conn->local_rx_ctrl.cons, conn),
		       conn);

	/* if usable snd_wnd closes ask peer to advertise once it opens again */
	conn->local_tx_ctrl.prod_flags.write_blocked = (to_send >= rmbespace);
	/* cf. usable snd_wnd */
	len = min(to_send, rmbespace);

	/* initialize variables for first iteration of subsequent nested loop */
	link = &lgr->lnk[SMC_SINGLE_LINK];
	dst_off = prod.count;
	if (prod.wrap == cons.wrap) {
		/* the filled destination area is unwrapped,
		 * hence the available free destination space is wrapped
		 * and we need 2 destination chunks of sum len; start with 1st
		 * which is limited by what's available in sndbuf
		 */
		dst_len = min_t(size_t,
				conn->peer_rmbe_size - prod.count, len);
	} else {
		/* the filled destination area is wrapped,
		 * hence the available free destination space is unwrapped
		 * and we need a single destination chunk of entire len
		 */
		dst_len = len;
	}
	dst_len_sum = dst_len;
	src_off = sent.count;
	/* dst_len determines the maximum src_len */
	if (sent.count + dst_len <= conn->sndbuf_size) {
		/* unwrapped src case: single chunk of entire dst_len */
		src_len = dst_len;
	} else {
		/* wrapped src case: 2 chunks of sum dst_len; start with 1st: */
		src_len = conn->sndbuf_size - sent.count;
	}
	src_len_sum = src_len;
	dma_addr = sg_dma_address(conn->sndbuf_desc->sgt[SMC_SINGLE_LINK].sgl);
	for (dstchunk = 0; dstchunk < 2; dstchunk++) {
		num_sges = 0;
		for (srcchunk = 0; srcchunk < 2; srcchunk++) {
			sges[srcchunk].addr = dma_addr + src_off;
			sges[srcchunk].length = src_len;
			sges[srcchunk].lkey = link->roce_pd->local_dma_lkey;
			num_sges++;
			src_off += src_len;
			if (src_off >= conn->sndbuf_size)
				src_off -= conn->sndbuf_size;
						/* modulo in send ring */
			if (src_len_sum == dst_len)
				break; /* either on 1st or 2nd iteration */
			/* prepare next (== 2nd) iteration */
			src_len = dst_len - src_len; /* remainder */
			src_len_sum += src_len;
		}
		rc = smc_tx_rdma_write(conn, dst_off, num_sges, sges);
		if (rc)
			return rc;
		if (dst_len_sum == len)
			break; /* either on 1st or 2nd iteration */
		/* prepare next (== 2nd) iteration */
		dst_off = 0; /* modulo offset in RMBE ring buffer */
		dst_len = len - dst_len; /* remainder */
		dst_len_sum += dst_len;
		src_len = min_t(int,
				dst_len, conn->sndbuf_size - sent.count);
		src_len_sum = src_len;
	}

	smc_tx_advance_cursors(conn, &prod, &sent, len);
	/* update connection's cursors with advanced local cursors */
	smc_curs_write(&conn->local_tx_ctrl.prod,
		       smc_curs_read(&prod, conn),
		       conn);
							/* dst: peer RMBE */
	smc_curs_write(&conn->tx_curs_sent,
		       smc_curs_read(&sent, conn),
		       conn);
							/* src: local sndbuf */

	return 0;
}

/* Wakeup sndbuf consumers from any context (IRQ or process)
 * since there is more data to transmit; usable snd_wnd as max transmit
 */
int smc_tx_sndbuf_nonempty(struct smc_connection *conn)
{
	struct smc_cdc_tx_pend *pend;
	struct smc_wr_buf *wr_buf;
	int rc;

	spin_lock_bh(&conn->send_lock);
	rc = smc_cdc_get_free_slot(&conn->lgr->lnk[SMC_SINGLE_LINK], &wr_buf,
				   &pend);
	if (rc < 0) {
		if (rc == -EBUSY) {
			struct smc_sock *smc =
				container_of(conn, struct smc_sock, conn);

			if (smc->sk.sk_err == ECONNABORTED) {
				rc = sock_error(&smc->sk);
				goto out_unlock;
			}
			rc = 0;
			schedule_delayed_work(&conn->tx_work,
					      SMC_TX_WORK_DELAY);
		}
		goto out_unlock;
	}

	rc = smc_tx_rdma_writes(conn);
	if (rc) {
		smc_wr_tx_put_slot(&conn->lgr->lnk[SMC_SINGLE_LINK],
				   (struct smc_wr_tx_pend_priv *)pend);
		goto out_unlock;
	}

	rc = smc_cdc_msg_send(conn, wr_buf, pend);

out_unlock:
	spin_unlock_bh(&conn->send_lock);
	return rc;
}

/* Wakeup sndbuf consumers from process context
 * since there is more data to transmit
 */
static void smc_tx_work(struct work_struct *work)
{
	struct smc_connection *conn = container_of(to_delayed_work(work),
						   struct smc_connection,
						   tx_work);
	struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
	int rc;

	lock_sock(&smc->sk);
	rc = smc_tx_sndbuf_nonempty(conn);
	if (!rc && conn->local_rx_ctrl.prod_flags.write_blocked &&
	    !atomic_read(&conn->bytes_to_rcv))
		conn->local_rx_ctrl.prod_flags.write_blocked = 0;
	release_sock(&smc->sk);
}

void smc_tx_consumer_update(struct smc_connection *conn)
{
	union smc_host_cursor cfed, cons;
	struct smc_cdc_tx_pend *pend;
	struct smc_wr_buf *wr_buf;
	int to_confirm, rc;

	smc_curs_write(&cons,
		       smc_curs_read(&conn->local_tx_ctrl.cons, conn),
		       conn);
	smc_curs_write(&cfed,
		       smc_curs_read(&conn->rx_curs_confirmed, conn),
		       conn);
	to_confirm = smc_curs_diff(conn->rmbe_size, &cfed, &cons);

	if (conn->local_rx_ctrl.prod_flags.cons_curs_upd_req ||
	    ((to_confirm > conn->rmbe_update_limit) &&
	     ((to_confirm > (conn->rmbe_size / 2)) ||
	      conn->local_rx_ctrl.prod_flags.write_blocked))) {
		rc = smc_cdc_get_free_slot(&conn->lgr->lnk[SMC_SINGLE_LINK],
					   &wr_buf, &pend);
		if (!rc)
			rc = smc_cdc_msg_send(conn, wr_buf, pend);
		if (rc < 0) {
			schedule_delayed_work(&conn->tx_work,
					      SMC_TX_WORK_DELAY);
			return;
		}
		smc_curs_write(&conn->rx_curs_confirmed,
			       smc_curs_read(&conn->local_tx_ctrl.cons, conn),
			       conn);
		conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0;
	}
	if (conn->local_rx_ctrl.prod_flags.write_blocked &&
	    !atomic_read(&conn->bytes_to_rcv))
		conn->local_rx_ctrl.prod_flags.write_blocked = 0;
}

/***************************** send initialize *******************************/

/* Initialize send properties on connection establishment. NB: not __init! */
void smc_tx_init(struct smc_sock *smc)
{
	smc->sk.sk_write_space = smc_tx_write_space;
	INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
	spin_lock_init(&smc->conn.send_lock);
}
Commit	Line	Data
e6727f39 UB	1	/*
	2	* Shared Memory Communications over RDMA (SMC-R) and RoCE
	3	*
	4	* Manage send buffer.
	5	* Producer:
	6	* Copy user space data into send buffer, if send buffer space available.
	7	* Consumer:
	8	* Trigger RDMA write into RMBE of peer and send CDC, if RMBE space available.
	9	*
	10	* Copyright IBM Corp. 2016
	11	*
	12	* Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
	13	*/
	14
	15	#include <linux/net.h>
	16	#include <linux/rcupdate.h>
	17	#include <linux/workqueue.h>
c3edc401 IM	18	#include <linux/sched/signal.h>
c3edc401 IM	19
e6727f39 UB	20	#include <net/sock.h>
	21
	22	#include "smc.h"
	23	#include "smc_wr.h"
	24	#include "smc_cdc.h"
	25	#include "smc_tx.h"
	26
18e537cd UB	27	#define SMC_TX_WORK_DELAY HZ
18e537cd UB	28
e6727f39 UB	29	/*************************** sndbuf producer *****************************/
	30
	31	/* callback implementation for sk.sk_write_space()
	32	* to wakeup sndbuf producers that blocked with smc_tx_wait_memory().
	33	* called under sk_socket lock.
	34	*/
	35	static void smc_tx_write_space(struct sock *sk)
	36	{
	37	struct socket *sock = sk->sk_socket;
	38	struct smc_sock *smc = smc_sk(sk);
	39	struct socket_wq *wq;
	40
	41	/* similar to sk_stream_write_space */
	42	if (atomic_read(&smc->conn.sndbuf_space) && sock) {
	43	clear_bit(SOCK_NOSPACE, &sock->flags);
	44	rcu_read_lock();
	45	wq = rcu_dereference(sk->sk_wq);
	46	if (skwq_has_sleeper(wq))
	47	wake_up_interruptible_poll(&wq->wait,
	48	POLLOUT \| POLLWRNORM \|
	49	POLLWRBAND);
	50	if (wq && wq->fasync_list && !(sk->sk_shutdown & SEND_SHUTDOWN))
	51	sock_wake_async(wq, SOCK_WAKE_SPACE, POLL_OUT);
	52	rcu_read_unlock();
	53	}
	54	}
	55
	56	/* Wakeup sndbuf producers that blocked with smc_tx_wait_memory().
	57	* Cf. tcp_data_snd_check()=>tcp_check_space()=>tcp_new_space().
	58	*/
	59	void smc_tx_sndbuf_nonfull(struct smc_sock *smc)
	60	{
	61	if (smc->sk.sk_socket &&
	62	test_bit(SOCK_NOSPACE, &smc->sk.sk_socket->flags))
	63	smc->sk.sk_write_space(&smc->sk);
	64	}
	65
	66	/* blocks sndbuf producer until at least one byte of free space available */
	67	static int smc_tx_wait_memory(struct smc_sock *smc, int flags)
	68	{
	69	DEFINE_WAIT_FUNC(wait, woken_wake_function);
	70	struct smc_connection *conn = &smc->conn;
	71	struct sock *sk = &smc->sk;
	72	bool noblock;
	73	long timeo;
	74	int rc = 0;
	75
	76	/* similar to sk_stream_wait_memory */
	77	timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
	78	noblock = timeo ? false : true;
	79	add_wait_queue(sk_sleep(sk), &wait);
	80	while (1) {
	81	sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
	82	if (sk->sk_err \|\|
	83	(sk->sk_shutdown & SEND_SHUTDOWN) \|\|
	84	conn->local_tx_ctrl.conn_state_flags.peer_done_writing) {
	85	rc = -EPIPE;
	86	break;
	87	}
	88	if (conn->local_rx_ctrl.conn_state_flags.peer_conn_abort) {
	89	rc = -ECONNRESET;
	90	break;
	91	}
	92	if (!timeo) {
93	if (noblock)
94	set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
95	rc = -EAGAIN;
96	break;
97	}
98	if (signal_pending(current)) {
99	rc = sock_intr_errno(timeo);
100	break;
101	}
102	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
103	if (atomic_read(&conn->sndbuf_space))
104	break; /* at least 1 byte of free space available */
105	set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
106	sk->sk_write_pending++;
107	sk_wait_event(sk, &timeo,
108	sk->sk_err \|\|
109	(sk->sk_shutdown & SEND_SHUTDOWN) \|\|
110	smc_cdc_rxed_any_close_or_senddone(conn) \|\|
111	atomic_read(&conn->sndbuf_space),
112	&wait);
113	sk->sk_write_pending--;
114	}
115	remove_wait_queue(sk_sleep(sk), &wait);
116	return rc;
117	}
118
119	/* sndbuf producer: main API called by socket layer.
120	* called under sock lock.
121	*/
122	int smc_tx_sendmsg(struct smc_sock smc, struct msghdr msg, size_t len)
123	{
124	size_t copylen, send_done = 0, send_remaining = len;
125	size_t chunk_len, chunk_off, chunk_len_sum;
126	struct smc_connection *conn = &smc->conn;
127	union smc_host_cursor prep;
128	struct sock *sk = &smc->sk;
129	char *sndbuf_base;
130	int tx_cnt_prep;
131	int writespace;
132	int rc, chunk;
133
134	/* This should be in poll */
135	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
136
137	if (sk->sk_err \|\| (sk->sk_shutdown & SEND_SHUTDOWN)) {
138	rc = -EPIPE;
139	goto out_err;
140	}
141
142	while (msg_data_left(msg)) {
143	if (sk->sk_state == SMC_INIT)
144	return -ENOTCONN;
145	if (smc->sk.sk_shutdown & SEND_SHUTDOWN \|\|
b38d7324	146	(smc->sk.sk_err == ECONNABORTED) \|\|
e6727f39 UB	147	conn->local_tx_ctrl.conn_state_flags.peer_conn_abort)
	148	return -EPIPE;
	149	if (smc_cdc_rxed_any_close(conn))
	150	return send_done ?: -ECONNRESET;
	151
	152	if (!atomic_read(&conn->sndbuf_space)) {
	153	rc = smc_tx_wait_memory(smc, msg->msg_flags);
	154	if (rc) {
	155	if (send_done)
	156	return send_done;
	157	goto out_err;
	158	}
	159	continue;
	160	}
	161
	162	/* initialize variables for 1st iteration of subsequent loop */
	163	/* could be just 1 byte, even after smc_tx_wait_memory above */
	164	writespace = atomic_read(&conn->sndbuf_space);
	165	/* not more than what user space asked for */
	166	copylen = min_t(size_t, send_remaining, writespace);
	167	/* determine start of sndbuf */
	168	sndbuf_base = conn->sndbuf_desc->cpu_addr;
	169	smc_curs_write(&prep,
	170	smc_curs_read(&conn->tx_curs_prep, conn),
	171	conn);
	172	tx_cnt_prep = prep.count;
	173	/* determine chunks where to write into sndbuf */
	174	/* either unwrapped case, or 1st chunk of wrapped case */
	175	chunk_len = min_t(size_t,
	176	copylen, conn->sndbuf_size - tx_cnt_prep);
	177	chunk_len_sum = chunk_len;
	178	chunk_off = tx_cnt_prep;
10428dd8	179	smc_sndbuf_sync_sg_for_cpu(conn);
e6727f39 UB	180	for (chunk = 0; chunk < 2; chunk++) {
	181	rc = memcpy_from_msg(sndbuf_base + chunk_off,
	182	msg, chunk_len);
	183	if (rc) {
10428dd8	184	smc_sndbuf_sync_sg_for_device(conn);
e6727f39 UB	185	if (send_done)
	186	return send_done;
	187	goto out_err;
	188	}
	189	send_done += chunk_len;
	190	send_remaining -= chunk_len;
	191
	192	if (chunk_len_sum == copylen)
	193	break; /* either on 1st or 2nd iteration */
	194	/* prepare next (== 2nd) iteration */
	195	chunk_len = copylen - chunk_len; /* remainder */
	196	chunk_len_sum += chunk_len;
	197	chunk_off = 0; /* modulo offset in send ring buffer */
	198	}
10428dd8	199	smc_sndbuf_sync_sg_for_device(conn);
e6727f39 UB	200	/* update cursors */
	201	smc_curs_add(conn->sndbuf_size, &prep, copylen);
	202	smc_curs_write(&conn->tx_curs_prep,
	203	smc_curs_read(&prep, conn),
	204	conn);
	205	/* increased in send tasklet smc_cdc_tx_handler() */
	206	smp_mb__before_atomic();
	207	atomic_sub(copylen, &conn->sndbuf_space);
	208	/* guarantee 0 <= sndbuf_space <= sndbuf_size */
	209	smp_mb__after_atomic();
	210	/* since we just produced more new data into sndbuf,
	211	* trigger sndbuf consumer: RDMA write into peer RMBE and CDC
	212	*/
	213	smc_tx_sndbuf_nonempty(conn);
	214	} /* while (msg_data_left(msg)) */
	215
	216	return send_done;
	217
	218	out_err:
	219	rc = sk_stream_error(sk, msg->msg_flags, rc);
	220	/* make sure we wake any epoll edge trigger waiter */
	221	if (unlikely(rc == -EAGAIN))
	222	sk->sk_write_space(sk);
	223	return rc;
	224	}
	225
	226	/*************************** sndbuf consumer *****************************/
	227
	228	/* sndbuf consumer: actual data transfer of one target chunk with RDMA write */
	229	static int smc_tx_rdma_write(struct smc_connection *conn, int peer_rmbe_offset,
	230	int num_sges, struct ib_sge sges[])
	231	{
	232	struct smc_link_group *lgr = conn->lgr;
	233	struct ib_send_wr *failed_wr = NULL;
	234	struct ib_rdma_wr rdma_wr;
	235	struct smc_link *link;
	236	int rc;
	237
	238	memset(&rdma_wr, 0, sizeof(rdma_wr));
	239	link = &lgr->lnk[SMC_SINGLE_LINK];
	240	rdma_wr.wr.wr_id = smc_wr_tx_get_next_wr_id(link);
	241	rdma_wr.wr.sg_list = sges;
	242	rdma_wr.wr.num_sge = num_sges;
	243	rdma_wr.wr.opcode = IB_WR_RDMA_WRITE;
	244	rdma_wr.remote_addr =
	245	lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].dma_addr +
	246	/* RMBE within RMB */
	247	((conn->peer_conn_idx - 1) * conn->peer_rmbe_size) +
	248	/* offset within RMBE */
	249	peer_rmbe_offset;
	250	rdma_wr.rkey = lgr->rtokens[conn->rtoken_idx][SMC_SINGLE_LINK].rkey;
	251	rc = ib_post_send(link->roce_qp, &rdma_wr.wr, &failed_wr);
	252	if (rc)
	253	conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;
	254	return rc;
	255	}
	256
	257	/* sndbuf consumer */
	258	static inline void smc_tx_advance_cursors(struct smc_connection *conn,
	259	union smc_host_cursor *prod,
	260	union smc_host_cursor *sent,
	261	size_t len)
	262	{
	263	smc_curs_add(conn->peer_rmbe_size, prod, len);
264	/* increased in recv tasklet smc_cdc_msg_rcv() */
265	smp_mb__before_atomic();
266	/* data in flight reduces usable snd_wnd */
267	atomic_sub(len, &conn->peer_rmbe_space);
268	/* guarantee 0 <= peer_rmbe_space <= peer_rmbe_size */
269	smp_mb__after_atomic();
270	smc_curs_add(conn->sndbuf_size, sent, len);
271	}
272
273	/* sndbuf consumer: prepare all necessary (src&dst) chunks of data transmit;
274	* usable snd_wnd as max transmit
275	*/
276	static int smc_tx_rdma_writes(struct smc_connection *conn)
277	{
278	size_t src_off, src_len, dst_off, dst_len; /* current chunk values */
279	size_t len, dst_len_sum, src_len_sum, dstchunk, srcchunk;
280	union smc_host_cursor sent, prep, prod, cons;
281	struct ib_sge sges[SMC_IB_MAX_SEND_SGE];
282	struct smc_link_group *lgr = conn->lgr;
283	int to_send, rmbespace;
284	struct smc_link *link;
9d8fb617	285	dma_addr_t dma_addr;
e6727f39 UB	286	int num_sges;
	287	int rc;
	288
	289	/* source: sndbuf */
	290	smc_curs_write(&sent, smc_curs_read(&conn->tx_curs_sent, conn), conn);
	291	smc_curs_write(&prep, smc_curs_read(&conn->tx_curs_prep, conn), conn);
	292	/* cf. wmem_alloc - (snd_max - snd_una) */
	293	to_send = smc_curs_diff(conn->sndbuf_size, &sent, &prep);
	294	if (to_send <= 0)
	295	return 0;
	296
	297	/* destination: RMBE */
	298	/* cf. snd_wnd */
	299	rmbespace = atomic_read(&conn->peer_rmbe_space);
	300	if (rmbespace <= 0)
	301	return 0;
	302	smc_curs_write(&prod,
	303	smc_curs_read(&conn->local_tx_ctrl.prod, conn),
	304	conn);
	305	smc_curs_write(&cons,
	306	smc_curs_read(&conn->local_rx_ctrl.cons, conn),
	307	conn);
	308
	309	/* if usable snd_wnd closes ask peer to advertise once it opens again */
	310	conn->local_tx_ctrl.prod_flags.write_blocked = (to_send >= rmbespace);
	311	/* cf. usable snd_wnd */
	312	len = min(to_send, rmbespace);
	313
	314	/* initialize variables for first iteration of subsequent nested loop */
	315	link = &lgr->lnk[SMC_SINGLE_LINK];
	316	dst_off = prod.count;
	317	if (prod.wrap == cons.wrap) {
	318	/* the filled destination area is unwrapped,
	319	* hence the available free destination space is wrapped
	320	* and we need 2 destination chunks of sum len; start with 1st
	321	* which is limited by what's available in sndbuf
	322	*/
	323	dst_len = min_t(size_t,
	324	conn->peer_rmbe_size - prod.count, len);
	325	} else {
	326	/* the filled destination area is wrapped,
	327	* hence the available free destination space is unwrapped
	328	* and we need a single destination chunk of entire len
	329	*/
	330	dst_len = len;
	331	}
	332	dst_len_sum = dst_len;
	333	src_off = sent.count;
	334	/* dst_len determines the maximum src_len */
	335	if (sent.count + dst_len <= conn->sndbuf_size) {
	336	/* unwrapped src case: single chunk of entire dst_len */
	337	src_len = dst_len;
	338	} else {
	339	/* wrapped src case: 2 chunks of sum dst_len; start with 1st: */
	340	src_len = conn->sndbuf_size - sent.count;
	341	}
	342	src_len_sum = src_len;
9d8fb617	343	dma_addr = sg_dma_address(conn->sndbuf_desc->sgt[SMC_SINGLE_LINK].sgl);
e6727f39 UB	344	for (dstchunk = 0; dstchunk < 2; dstchunk++) {
	345	num_sges = 0;
	346	for (srcchunk = 0; srcchunk < 2; srcchunk++) {
9d8fb617	347	sges[srcchunk].addr = dma_addr + src_off;
e6727f39 UB	348	sges[srcchunk].length = src_len;
	349	sges[srcchunk].lkey = link->roce_pd->local_dma_lkey;
	350	num_sges++;
	351	src_off += src_len;
	352	if (src_off >= conn->sndbuf_size)
	353	src_off -= conn->sndbuf_size;
	354	/* modulo in send ring */
	355	if (src_len_sum == dst_len)
	356	break; /* either on 1st or 2nd iteration */
	357	/* prepare next (== 2nd) iteration */
	358	src_len = dst_len - src_len; /* remainder */
	359	src_len_sum += src_len;
	360	}
	361	rc = smc_tx_rdma_write(conn, dst_off, num_sges, sges);
	362	if (rc)
	363	return rc;
	364	if (dst_len_sum == len)
	365	break; /* either on 1st or 2nd iteration */
	366	/* prepare next (== 2nd) iteration */
	367	dst_off = 0; /* modulo offset in RMBE ring buffer */
	368	dst_len = len - dst_len; /* remainder */
	369	dst_len_sum += dst_len;
	370	src_len = min_t(int,
	371	dst_len, conn->sndbuf_size - sent.count);
	372	src_len_sum = src_len;
	373	}
	374
	375	smc_tx_advance_cursors(conn, &prod, &sent, len);
	376	/* update connection's cursors with advanced local cursors */
	377	smc_curs_write(&conn->local_tx_ctrl.prod,
	378	smc_curs_read(&prod, conn),
	379	conn);
	380	/* dst: peer RMBE */
	381	smc_curs_write(&conn->tx_curs_sent,
	382	smc_curs_read(&sent, conn),
	383	conn);
	384	/* src: local sndbuf */
	385
	386	return 0;
	387	}
	388
	389	/* Wakeup sndbuf consumers from any context (IRQ or process)
	390	* since there is more data to transmit; usable snd_wnd as max transmit
	391	*/
	392	int smc_tx_sndbuf_nonempty(struct smc_connection *conn)
	393	{
	394	struct smc_cdc_tx_pend *pend;
	395	struct smc_wr_buf *wr_buf;
	396	int rc;
	397
	398	spin_lock_bh(&conn->send_lock);
	399	rc = smc_cdc_get_free_slot(&conn->lgr->lnk[SMC_SINGLE_LINK], &wr_buf,
	400	&pend);
	401	if (rc < 0) {
	402	if (rc == -EBUSY) {
b38d7324 UB	403	struct smc_sock *smc =
	404	container_of(conn, struct smc_sock, conn);
	405
	406	if (smc->sk.sk_err == ECONNABORTED) {
	407	rc = sock_error(&smc->sk);
	408	goto out_unlock;
	409	}
e6727f39	410	rc = 0;
18e537cd UB	411	schedule_delayed_work(&conn->tx_work,
18e537cd UB	412	SMC_TX_WORK_DELAY);
e6727f39 UB	413	}
	414	goto out_unlock;
	415	}
	416
	417	rc = smc_tx_rdma_writes(conn);
	418	if (rc) {
	419	smc_wr_tx_put_slot(&conn->lgr->lnk[SMC_SINGLE_LINK],
	420	(struct smc_wr_tx_pend_priv *)pend);
	421	goto out_unlock;
	422	}
	423
	424	rc = smc_cdc_msg_send(conn, wr_buf, pend);
	425
	426	out_unlock:
	427	spin_unlock_bh(&conn->send_lock);
	428	return rc;
	429	}
	430
	431	/* Wakeup sndbuf consumers from process context
	432	* since there is more data to transmit
	433	*/
	434	static void smc_tx_work(struct work_struct *work)
	435	{
18e537cd	436	struct smc_connection *conn = container_of(to_delayed_work(work),
e6727f39 UB	437	struct smc_connection,
	438	tx_work);
	439	struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
90cacb2e	440	int rc;
e6727f39 UB	441
e6727f39 UB	442	lock_sock(&smc->sk);
90cacb2e UB	443	rc = smc_tx_sndbuf_nonempty(conn);
	444	if (!rc && conn->local_rx_ctrl.prod_flags.write_blocked &&
	445	!atomic_read(&conn->bytes_to_rcv))
	446	conn->local_rx_ctrl.prod_flags.write_blocked = 0;
e6727f39 UB	447	release_sock(&smc->sk);
	448	}
	449
952310cc UB	450	void smc_tx_consumer_update(struct smc_connection *conn)
	451	{
	452	union smc_host_cursor cfed, cons;
	453	struct smc_cdc_tx_pend *pend;
	454	struct smc_wr_buf *wr_buf;
	455	int to_confirm, rc;
	456
	457	smc_curs_write(&cons,
	458	smc_curs_read(&conn->local_tx_ctrl.cons, conn),
	459	conn);
	460	smc_curs_write(&cfed,
	461	smc_curs_read(&conn->rx_curs_confirmed, conn),
	462	conn);
	463	to_confirm = smc_curs_diff(conn->rmbe_size, &cfed, &cons);
	464
	465	if (conn->local_rx_ctrl.prod_flags.cons_curs_upd_req \|\|
	466	((to_confirm > conn->rmbe_update_limit) &&
	467	((to_confirm > (conn->rmbe_size / 2)) \|\|
	468	conn->local_rx_ctrl.prod_flags.write_blocked))) {
	469	rc = smc_cdc_get_free_slot(&conn->lgr->lnk[SMC_SINGLE_LINK],
	470	&wr_buf, &pend);
	471	if (!rc)
	472	rc = smc_cdc_msg_send(conn, wr_buf, pend);
	473	if (rc < 0) {
18e537cd UB	474	schedule_delayed_work(&conn->tx_work,
18e537cd UB	475	SMC_TX_WORK_DELAY);
952310cc UB	476	return;
	477	}
	478	smc_curs_write(&conn->rx_curs_confirmed,
	479	smc_curs_read(&conn->local_tx_ctrl.cons, conn),
	480	conn);
	481	conn->local_rx_ctrl.prod_flags.cons_curs_upd_req = 0;
	482	}
	483	if (conn->local_rx_ctrl.prod_flags.write_blocked &&
	484	!atomic_read(&conn->bytes_to_rcv))
	485	conn->local_rx_ctrl.prod_flags.write_blocked = 0;
	486	}
	487
e6727f39 UB	488	/*************************** send initialize *****************************/
	489
	490	/* Initialize send properties on connection establishment. NB: not __init! */
	491	void smc_tx_init(struct smc_sock *smc)
	492	{
	493	smc->sk.sk_write_space = smc_tx_write_space;
18e537cd	494	INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
e6727f39 UB	495	spin_lock_init(&smc->conn.send_lock);
e6727f39 UB	496	}