]>
git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blob - net/smc/smc_tx.c
1 // SPDX-License-Identifier: GPL-2.0
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
7 * Copy user space data into send buffer, if send buffer space available.
9 * Trigger RDMA write into RMBE of peer and send CDC, if RMBE space available.
11 * Copyright IBM Corp. 2016
13 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
16 #include <linux/net.h>
17 #include <linux/rcupdate.h>
18 #include <linux/workqueue.h>
19 #include <linux/sched/signal.h>
27 #include "smc_close.h"
30 #include "smc_stats.h"
32 #define SMC_TX_WORK_DELAY 0
33 #define SMC_TX_CORK_DELAY (HZ >> 2) /* 250 ms */
35 /***************************** sndbuf producer *******************************/
37 /* callback implementation for sk.sk_write_space()
38 * to wakeup sndbuf producers that blocked with smc_tx_wait().
39 * called under sk_socket lock.
41 static void smc_tx_write_space(struct sock
*sk
)
43 struct socket
*sock
= sk
->sk_socket
;
44 struct smc_sock
*smc
= smc_sk(sk
);
47 /* similar to sk_stream_write_space */
48 if (atomic_read(&smc
->conn
.sndbuf_space
) && sock
) {
49 if (test_bit(SOCK_NOSPACE
, &sock
->flags
))
50 SMC_STAT_RMB_TX_FULL(smc
, !smc
->conn
.lnk
);
51 clear_bit(SOCK_NOSPACE
, &sock
->flags
);
53 wq
= rcu_dereference(sk
->sk_wq
);
54 if (skwq_has_sleeper(wq
))
55 wake_up_interruptible_poll(&wq
->wait
,
56 EPOLLOUT
| EPOLLWRNORM
|
58 if (wq
&& wq
->fasync_list
&& !(sk
->sk_shutdown
& SEND_SHUTDOWN
))
59 sock_wake_async(wq
, SOCK_WAKE_SPACE
, POLL_OUT
);
64 /* Wakeup sndbuf producers that blocked with smc_tx_wait().
65 * Cf. tcp_data_snd_check()=>tcp_check_space()=>tcp_new_space().
67 void smc_tx_sndbuf_nonfull(struct smc_sock
*smc
)
69 if (smc
->sk
.sk_socket
&&
70 test_bit(SOCK_NOSPACE
, &smc
->sk
.sk_socket
->flags
))
71 smc
->sk
.sk_write_space(&smc
->sk
);
74 /* blocks sndbuf producer until at least one byte of free space available
75 * or urgent Byte was consumed
77 static int smc_tx_wait(struct smc_sock
*smc
, int flags
)
79 DEFINE_WAIT_FUNC(wait
, woken_wake_function
);
80 struct smc_connection
*conn
= &smc
->conn
;
81 struct sock
*sk
= &smc
->sk
;
85 /* similar to sk_stream_wait_memory */
86 timeo
= sock_sndtimeo(sk
, flags
& MSG_DONTWAIT
);
87 add_wait_queue(sk_sleep(sk
), &wait
);
89 sk_set_bit(SOCKWQ_ASYNC_NOSPACE
, sk
);
91 (sk
->sk_shutdown
& SEND_SHUTDOWN
) ||
93 conn
->local_tx_ctrl
.conn_state_flags
.peer_done_writing
) {
97 if (smc_cdc_rxed_any_close(conn
)) {
102 /* ensure EPOLLOUT is subsequently generated */
103 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
107 if (signal_pending(current
)) {
108 rc
= sock_intr_errno(timeo
);
111 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE
, sk
);
112 if (atomic_read(&conn
->sndbuf_space
) && !conn
->urg_tx_pend
)
113 break; /* at least 1 byte of free & no urgent data */
114 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
115 sk_wait_event(sk
, &timeo
,
117 (sk
->sk_shutdown
& SEND_SHUTDOWN
) ||
118 smc_cdc_rxed_any_close(conn
) ||
119 (atomic_read(&conn
->sndbuf_space
) &&
123 remove_wait_queue(sk_sleep(sk
), &wait
);
127 static bool smc_tx_is_corked(struct smc_sock
*smc
)
129 struct tcp_sock
*tp
= tcp_sk(smc
->clcsock
->sk
);
131 return (tp
->nonagle
& TCP_NAGLE_CORK
) ? true : false;
134 /* sndbuf producer: main API called by socket layer.
135 * called under sock lock.
137 int smc_tx_sendmsg(struct smc_sock
*smc
, struct msghdr
*msg
, size_t len
)
139 size_t copylen
, send_done
= 0, send_remaining
= len
;
140 size_t chunk_len
, chunk_off
, chunk_len_sum
;
141 struct smc_connection
*conn
= &smc
->conn
;
142 union smc_host_cursor prep
;
143 struct sock
*sk
= &smc
->sk
;
149 /* This should be in poll */
150 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE
, sk
);
152 if (sk
->sk_err
|| (sk
->sk_shutdown
& SEND_SHUTDOWN
)) {
157 if (sk
->sk_state
== SMC_INIT
)
160 if (len
> conn
->sndbuf_desc
->len
)
161 SMC_STAT_RMB_TX_SIZE_SMALL(smc
, !conn
->lnk
);
163 if (len
> conn
->peer_rmbe_size
)
164 SMC_STAT_RMB_TX_PEER_SIZE_SMALL(smc
, !conn
->lnk
);
166 if (msg
->msg_flags
& MSG_OOB
)
167 SMC_STAT_INC(smc
, urg_data_cnt
);
169 while (msg_data_left(msg
)) {
170 if (smc
->sk
.sk_shutdown
& SEND_SHUTDOWN
||
171 (smc
->sk
.sk_err
== ECONNABORTED
) ||
174 if (smc_cdc_rxed_any_close(conn
))
175 return send_done
?: -ECONNRESET
;
177 if (msg
->msg_flags
& MSG_OOB
)
178 conn
->local_tx_ctrl
.prod_flags
.urg_data_pending
= 1;
180 if (!atomic_read(&conn
->sndbuf_space
) || conn
->urg_tx_pend
) {
183 rc
= smc_tx_wait(smc
, msg
->msg_flags
);
189 /* initialize variables for 1st iteration of subsequent loop */
190 /* could be just 1 byte, even after smc_tx_wait above */
191 writespace
= atomic_read(&conn
->sndbuf_space
);
192 /* not more than what user space asked for */
193 copylen
= min_t(size_t, send_remaining
, writespace
);
194 /* determine start of sndbuf */
195 sndbuf_base
= conn
->sndbuf_desc
->cpu_addr
;
196 smc_curs_copy(&prep
, &conn
->tx_curs_prep
, conn
);
197 tx_cnt_prep
= prep
.count
;
198 /* determine chunks where to write into sndbuf */
199 /* either unwrapped case, or 1st chunk of wrapped case */
200 chunk_len
= min_t(size_t, copylen
, conn
->sndbuf_desc
->len
-
202 chunk_len_sum
= chunk_len
;
203 chunk_off
= tx_cnt_prep
;
204 smc_sndbuf_sync_sg_for_cpu(conn
);
205 for (chunk
= 0; chunk
< 2; chunk
++) {
206 rc
= memcpy_from_msg(sndbuf_base
+ chunk_off
,
209 smc_sndbuf_sync_sg_for_device(conn
);
214 send_done
+= chunk_len
;
215 send_remaining
-= chunk_len
;
217 if (chunk_len_sum
== copylen
)
218 break; /* either on 1st or 2nd iteration */
219 /* prepare next (== 2nd) iteration */
220 chunk_len
= copylen
- chunk_len
; /* remainder */
221 chunk_len_sum
+= chunk_len
;
222 chunk_off
= 0; /* modulo offset in send ring buffer */
224 smc_sndbuf_sync_sg_for_device(conn
);
226 smc_curs_add(conn
->sndbuf_desc
->len
, &prep
, copylen
);
227 smc_curs_copy(&conn
->tx_curs_prep
, &prep
, conn
);
228 /* increased in send tasklet smc_cdc_tx_handler() */
229 smp_mb__before_atomic();
230 atomic_sub(copylen
, &conn
->sndbuf_space
);
231 /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */
232 smp_mb__after_atomic();
233 /* since we just produced more new data into sndbuf,
234 * trigger sndbuf consumer: RDMA write into peer RMBE and CDC
236 if ((msg
->msg_flags
& MSG_OOB
) && !send_remaining
)
237 conn
->urg_tx_pend
= true;
238 if ((msg
->msg_flags
& MSG_MORE
|| smc_tx_is_corked(smc
)) &&
239 (atomic_read(&conn
->sndbuf_space
) >
240 (conn
->sndbuf_desc
->len
>> 1)))
241 /* for a corked socket defer the RDMA writes if there
242 * is still sufficient sndbuf_space available
244 queue_delayed_work(conn
->lgr
->tx_wq
, &conn
->tx_work
,
247 smc_tx_sndbuf_nonempty(conn
);
248 } /* while (msg_data_left(msg)) */
253 rc
= sk_stream_error(sk
, msg
->msg_flags
, rc
);
254 /* make sure we wake any epoll edge trigger waiter */
255 if (unlikely(rc
== -EAGAIN
))
256 sk
->sk_write_space(sk
);
260 /***************************** sndbuf consumer *******************************/
262 /* sndbuf consumer: actual data transfer of one target chunk with ISM write */
263 int smcd_tx_ism_write(struct smc_connection
*conn
, void *data
, size_t len
,
264 u32 offset
, int signal
)
266 struct smc_ism_position pos
;
269 memset(&pos
, 0, sizeof(pos
));
270 pos
.token
= conn
->peer_token
;
271 pos
.index
= conn
->peer_rmbe_idx
;
272 pos
.offset
= conn
->tx_off
+ offset
;
274 rc
= smc_ism_write(conn
->lgr
->smcd
, &pos
, data
, len
);
276 conn
->local_tx_ctrl
.conn_state_flags
.peer_conn_abort
= 1;
280 /* sndbuf consumer: actual data transfer of one target chunk with RDMA write */
281 static int smc_tx_rdma_write(struct smc_connection
*conn
, int peer_rmbe_offset
,
282 int num_sges
, struct ib_rdma_wr
*rdma_wr
)
284 struct smc_link_group
*lgr
= conn
->lgr
;
285 struct smc_link
*link
= conn
->lnk
;
288 rdma_wr
->wr
.wr_id
= smc_wr_tx_get_next_wr_id(link
);
289 rdma_wr
->wr
.num_sge
= num_sges
;
290 rdma_wr
->remote_addr
=
291 lgr
->rtokens
[conn
->rtoken_idx
][link
->link_idx
].dma_addr
+
292 /* RMBE within RMB */
294 /* offset within RMBE */
296 rdma_wr
->rkey
= lgr
->rtokens
[conn
->rtoken_idx
][link
->link_idx
].rkey
;
297 rc
= ib_post_send(link
->roce_qp
, &rdma_wr
->wr
, NULL
);
299 smcr_link_down_cond_sched(link
);
303 /* sndbuf consumer */
304 static inline void smc_tx_advance_cursors(struct smc_connection
*conn
,
305 union smc_host_cursor
*prod
,
306 union smc_host_cursor
*sent
,
309 smc_curs_add(conn
->peer_rmbe_size
, prod
, len
);
310 /* increased in recv tasklet smc_cdc_msg_rcv() */
311 smp_mb__before_atomic();
312 /* data in flight reduces usable snd_wnd */
313 atomic_sub(len
, &conn
->peer_rmbe_space
);
314 /* guarantee 0 <= peer_rmbe_space <= peer_rmbe_size */
315 smp_mb__after_atomic();
316 smc_curs_add(conn
->sndbuf_desc
->len
, sent
, len
);
319 /* SMC-R helper for smc_tx_rdma_writes() */
320 static int smcr_tx_rdma_writes(struct smc_connection
*conn
, size_t len
,
321 size_t src_off
, size_t src_len
,
322 size_t dst_off
, size_t dst_len
,
323 struct smc_rdma_wr
*wr_rdma_buf
)
325 struct smc_link
*link
= conn
->lnk
;
327 dma_addr_t dma_addr
=
328 sg_dma_address(conn
->sndbuf_desc
->sgt
[link
->link_idx
].sgl
);
329 int src_len_sum
= src_len
, dst_len_sum
= dst_len
;
330 int sent_count
= src_off
;
331 int srcchunk
, dstchunk
;
335 for (dstchunk
= 0; dstchunk
< 2; dstchunk
++) {
337 wr_rdma_buf
->wr_tx_rdma
[dstchunk
].wr
.sg_list
;
340 for (srcchunk
= 0; srcchunk
< 2; srcchunk
++) {
341 sge
[srcchunk
].addr
= dma_addr
+ src_off
;
342 sge
[srcchunk
].length
= src_len
;
346 if (src_off
>= conn
->sndbuf_desc
->len
)
347 src_off
-= conn
->sndbuf_desc
->len
;
348 /* modulo in send ring */
349 if (src_len_sum
== dst_len
)
350 break; /* either on 1st or 2nd iteration */
351 /* prepare next (== 2nd) iteration */
352 src_len
= dst_len
- src_len
; /* remainder */
353 src_len_sum
+= src_len
;
355 rc
= smc_tx_rdma_write(conn
, dst_off
, num_sges
,
356 &wr_rdma_buf
->wr_tx_rdma
[dstchunk
]);
359 if (dst_len_sum
== len
)
360 break; /* either on 1st or 2nd iteration */
361 /* prepare next (== 2nd) iteration */
362 dst_off
= 0; /* modulo offset in RMBE ring buffer */
363 dst_len
= len
- dst_len
; /* remainder */
364 dst_len_sum
+= dst_len
;
365 src_len
= min_t(int, dst_len
, conn
->sndbuf_desc
->len
-
367 src_len_sum
= src_len
;
372 /* SMC-D helper for smc_tx_rdma_writes() */
373 static int smcd_tx_rdma_writes(struct smc_connection
*conn
, size_t len
,
374 size_t src_off
, size_t src_len
,
375 size_t dst_off
, size_t dst_len
)
377 int src_len_sum
= src_len
, dst_len_sum
= dst_len
;
378 int srcchunk
, dstchunk
;
381 for (dstchunk
= 0; dstchunk
< 2; dstchunk
++) {
382 for (srcchunk
= 0; srcchunk
< 2; srcchunk
++) {
383 void *data
= conn
->sndbuf_desc
->cpu_addr
+ src_off
;
385 rc
= smcd_tx_ism_write(conn
, data
, src_len
, dst_off
+
386 sizeof(struct smcd_cdc_msg
), 0);
391 if (src_off
>= conn
->sndbuf_desc
->len
)
392 src_off
-= conn
->sndbuf_desc
->len
;
393 /* modulo in send ring */
394 if (src_len_sum
== dst_len
)
395 break; /* either on 1st or 2nd iteration */
396 /* prepare next (== 2nd) iteration */
397 src_len
= dst_len
- src_len
; /* remainder */
398 src_len_sum
+= src_len
;
400 if (dst_len_sum
== len
)
401 break; /* either on 1st or 2nd iteration */
402 /* prepare next (== 2nd) iteration */
403 dst_off
= 0; /* modulo offset in RMBE ring buffer */
404 dst_len
= len
- dst_len
; /* remainder */
405 dst_len_sum
+= dst_len
;
406 src_len
= min_t(int, dst_len
, conn
->sndbuf_desc
->len
- src_off
);
407 src_len_sum
= src_len
;
412 /* sndbuf consumer: prepare all necessary (src&dst) chunks of data transmit;
413 * usable snd_wnd as max transmit
415 static int smc_tx_rdma_writes(struct smc_connection
*conn
,
416 struct smc_rdma_wr
*wr_rdma_buf
)
418 size_t len
, src_len
, dst_off
, dst_len
; /* current chunk values */
419 union smc_host_cursor sent
, prep
, prod
, cons
;
420 struct smc_cdc_producer_flags
*pflags
;
421 int to_send
, rmbespace
;
425 smc_curs_copy(&sent
, &conn
->tx_curs_sent
, conn
);
426 smc_curs_copy(&prep
, &conn
->tx_curs_prep
, conn
);
427 /* cf. wmem_alloc - (snd_max - snd_una) */
428 to_send
= smc_curs_diff(conn
->sndbuf_desc
->len
, &sent
, &prep
);
432 /* destination: RMBE */
434 rmbespace
= atomic_read(&conn
->peer_rmbe_space
);
435 if (rmbespace
<= 0) {
436 struct smc_sock
*smc
= container_of(conn
, struct smc_sock
,
438 SMC_STAT_RMB_TX_PEER_FULL(smc
, !conn
->lnk
);
441 smc_curs_copy(&prod
, &conn
->local_tx_ctrl
.prod
, conn
);
442 smc_curs_copy(&cons
, &conn
->local_rx_ctrl
.cons
, conn
);
444 /* if usable snd_wnd closes ask peer to advertise once it opens again */
445 pflags
= &conn
->local_tx_ctrl
.prod_flags
;
446 pflags
->write_blocked
= (to_send
>= rmbespace
);
447 /* cf. usable snd_wnd */
448 len
= min(to_send
, rmbespace
);
450 /* initialize variables for first iteration of subsequent nested loop */
451 dst_off
= prod
.count
;
452 if (prod
.wrap
== cons
.wrap
) {
453 /* the filled destination area is unwrapped,
454 * hence the available free destination space is wrapped
455 * and we need 2 destination chunks of sum len; start with 1st
456 * which is limited by what's available in sndbuf
458 dst_len
= min_t(size_t,
459 conn
->peer_rmbe_size
- prod
.count
, len
);
461 /* the filled destination area is wrapped,
462 * hence the available free destination space is unwrapped
463 * and we need a single destination chunk of entire len
467 /* dst_len determines the maximum src_len */
468 if (sent
.count
+ dst_len
<= conn
->sndbuf_desc
->len
) {
469 /* unwrapped src case: single chunk of entire dst_len */
472 /* wrapped src case: 2 chunks of sum dst_len; start with 1st: */
473 src_len
= conn
->sndbuf_desc
->len
- sent
.count
;
476 if (conn
->lgr
->is_smcd
)
477 rc
= smcd_tx_rdma_writes(conn
, len
, sent
.count
, src_len
,
480 rc
= smcr_tx_rdma_writes(conn
, len
, sent
.count
, src_len
,
481 dst_off
, dst_len
, wr_rdma_buf
);
485 if (conn
->urg_tx_pend
&& len
== to_send
)
486 pflags
->urg_data_present
= 1;
487 smc_tx_advance_cursors(conn
, &prod
, &sent
, len
);
488 /* update connection's cursors with advanced local cursors */
489 smc_curs_copy(&conn
->local_tx_ctrl
.prod
, &prod
, conn
);
491 smc_curs_copy(&conn
->tx_curs_sent
, &sent
, conn
);/* src: local sndbuf */
496 /* Wakeup sndbuf consumers from any context (IRQ or process)
497 * since there is more data to transmit; usable snd_wnd as max transmit
499 static int smcr_tx_sndbuf_nonempty(struct smc_connection
*conn
)
501 struct smc_cdc_producer_flags
*pflags
= &conn
->local_tx_ctrl
.prod_flags
;
502 struct smc_link
*link
= conn
->lnk
;
503 struct smc_rdma_wr
*wr_rdma_buf
;
504 struct smc_cdc_tx_pend
*pend
;
505 struct smc_wr_buf
*wr_buf
;
508 if (!link
|| !smc_wr_tx_link_hold(link
))
510 rc
= smc_cdc_get_free_slot(conn
, link
, &wr_buf
, &wr_rdma_buf
, &pend
);
512 smc_wr_tx_link_put(link
);
514 struct smc_sock
*smc
=
515 container_of(conn
, struct smc_sock
, conn
);
517 if (smc
->sk
.sk_err
== ECONNABORTED
)
518 return sock_error(&smc
->sk
);
522 mod_delayed_work(conn
->lgr
->tx_wq
, &conn
->tx_work
,
528 spin_lock_bh(&conn
->send_lock
);
529 if (link
!= conn
->lnk
) {
530 /* link of connection changed, tx_work will restart */
531 smc_wr_tx_put_slot(link
,
532 (struct smc_wr_tx_pend_priv
*)pend
);
536 if (!pflags
->urg_data_present
) {
537 rc
= smc_tx_rdma_writes(conn
, wr_rdma_buf
);
539 smc_wr_tx_put_slot(link
,
540 (struct smc_wr_tx_pend_priv
*)pend
);
545 rc
= smc_cdc_msg_send(conn
, wr_buf
, pend
);
546 if (!rc
&& pflags
->urg_data_present
) {
547 pflags
->urg_data_pending
= 0;
548 pflags
->urg_data_present
= 0;
552 spin_unlock_bh(&conn
->send_lock
);
553 smc_wr_tx_link_put(link
);
557 static int smcd_tx_sndbuf_nonempty(struct smc_connection
*conn
)
559 struct smc_cdc_producer_flags
*pflags
= &conn
->local_tx_ctrl
.prod_flags
;
562 spin_lock_bh(&conn
->send_lock
);
563 if (!pflags
->urg_data_present
)
564 rc
= smc_tx_rdma_writes(conn
, NULL
);
566 rc
= smcd_cdc_msg_send(conn
);
568 if (!rc
&& pflags
->urg_data_present
) {
569 pflags
->urg_data_pending
= 0;
570 pflags
->urg_data_present
= 0;
572 spin_unlock_bh(&conn
->send_lock
);
576 int smc_tx_sndbuf_nonempty(struct smc_connection
*conn
)
581 conn
->local_rx_ctrl
.conn_state_flags
.peer_conn_abort
)
582 return -EPIPE
; /* connection being aborted */
583 if (conn
->lgr
->is_smcd
)
584 rc
= smcd_tx_sndbuf_nonempty(conn
);
586 rc
= smcr_tx_sndbuf_nonempty(conn
);
589 /* trigger socket release if connection is closing */
590 struct smc_sock
*smc
= container_of(conn
, struct smc_sock
,
592 smc_close_wake_tx_prepared(smc
);
597 void smc_tx_pending(struct smc_connection
*conn
)
599 struct smc_sock
*smc
= container_of(conn
, struct smc_sock
, conn
);
605 rc
= smc_tx_sndbuf_nonempty(conn
);
606 if (!rc
&& conn
->local_rx_ctrl
.prod_flags
.write_blocked
&&
607 !atomic_read(&conn
->bytes_to_rcv
))
608 conn
->local_rx_ctrl
.prod_flags
.write_blocked
= 0;
611 /* Wakeup sndbuf consumers from process context
612 * since there is more data to transmit
614 void smc_tx_work(struct work_struct
*work
)
616 struct smc_connection
*conn
= container_of(to_delayed_work(work
),
617 struct smc_connection
,
619 struct smc_sock
*smc
= container_of(conn
, struct smc_sock
, conn
);
622 smc_tx_pending(conn
);
623 release_sock(&smc
->sk
);
626 void smc_tx_consumer_update(struct smc_connection
*conn
, bool force
)
628 union smc_host_cursor cfed
, cons
, prod
;
629 int sender_free
= conn
->rmb_desc
->len
;
632 smc_curs_copy(&cons
, &conn
->local_tx_ctrl
.cons
, conn
);
633 smc_curs_copy(&cfed
, &conn
->rx_curs_confirmed
, conn
);
634 to_confirm
= smc_curs_diff(conn
->rmb_desc
->len
, &cfed
, &cons
);
635 if (to_confirm
> conn
->rmbe_update_limit
) {
636 smc_curs_copy(&prod
, &conn
->local_rx_ctrl
.prod
, conn
);
637 sender_free
= conn
->rmb_desc
->len
-
638 smc_curs_diff_large(conn
->rmb_desc
->len
,
642 if (conn
->local_rx_ctrl
.prod_flags
.cons_curs_upd_req
||
644 ((to_confirm
> conn
->rmbe_update_limit
) &&
645 ((sender_free
<= (conn
->rmb_desc
->len
/ 2)) ||
646 conn
->local_rx_ctrl
.prod_flags
.write_blocked
))) {
648 conn
->local_rx_ctrl
.conn_state_flags
.peer_conn_abort
)
650 if ((smc_cdc_get_slot_and_msg_send(conn
) < 0) &&
652 queue_delayed_work(conn
->lgr
->tx_wq
, &conn
->tx_work
,
657 if (conn
->local_rx_ctrl
.prod_flags
.write_blocked
&&
658 !atomic_read(&conn
->bytes_to_rcv
))
659 conn
->local_rx_ctrl
.prod_flags
.write_blocked
= 0;
662 /***************************** send initialize *******************************/
664 /* Initialize send properties on connection establishment. NB: not __init! */
665 void smc_tx_init(struct smc_sock
*smc
)
667 smc
->sk
.sk_write_space
= smc_tx_write_space
;