1 // SPDX-License-Identifier: GPL-2.0-only
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
5 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
6 * applies to SOCK_STREAM sockets only
7 * offers an alternative communication option for TCP-protocol sockets
8 * applicable with RoCE-cards only
10 * Initial restrictions:
11 * - support for alternate links postponed
13 * Copyright IBM Corp. 2016, 2018
15 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
16 * based on prototype from Frank Blaschka
19 #define KMSG_COMPONENT "smc"
20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
22 #include <linux/module.h>
23 #include <linux/socket.h>
24 #include <linux/workqueue.h>
26 #include <linux/sched/signal.h>
27 #include <linux/if_vlan.h>
28 #include <linux/rcupdate_wait.h>
33 #include <asm/ioctls.h>
35 #include <net/net_namespace.h>
36 #include <net/netns/generic.h>
37 #include "smc_netns.h"
49 #include "smc_close.h"
51 static DEFINE_MUTEX(smc_server_lgr_pending
); /* serialize link group
54 static DEFINE_MUTEX(smc_client_lgr_pending
); /* serialize link group
58 static void smc_tcp_listen_work(struct work_struct
*);
59 static void smc_connect_work(struct work_struct
*);
61 static void smc_set_keepalive(struct sock
*sk
, int val
)
63 struct smc_sock
*smc
= smc_sk(sk
);
65 smc
->clcsock
->sk
->sk_prot
->keepalive(smc
->clcsock
->sk
, val
);
68 static struct smc_hashinfo smc_v4_hashinfo
= {
69 .lock
= __RW_LOCK_UNLOCKED(smc_v4_hashinfo
.lock
),
72 static struct smc_hashinfo smc_v6_hashinfo
= {
73 .lock
= __RW_LOCK_UNLOCKED(smc_v6_hashinfo
.lock
),
76 int smc_hash_sk(struct sock
*sk
)
78 struct smc_hashinfo
*h
= sk
->sk_prot
->h
.smc_hash
;
79 struct hlist_head
*head
;
83 write_lock_bh(&h
->lock
);
84 sk_add_node(sk
, head
);
85 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, 1);
86 write_unlock_bh(&h
->lock
);
90 EXPORT_SYMBOL_GPL(smc_hash_sk
);
92 void smc_unhash_sk(struct sock
*sk
)
94 struct smc_hashinfo
*h
= sk
->sk_prot
->h
.smc_hash
;
96 write_lock_bh(&h
->lock
);
97 if (sk_del_node_init(sk
))
98 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, -1);
99 write_unlock_bh(&h
->lock
);
101 EXPORT_SYMBOL_GPL(smc_unhash_sk
);
103 struct proto smc_proto
= {
105 .owner
= THIS_MODULE
,
106 .keepalive
= smc_set_keepalive
,
108 .unhash
= smc_unhash_sk
,
109 .obj_size
= sizeof(struct smc_sock
),
110 .h
.smc_hash
= &smc_v4_hashinfo
,
111 .slab_flags
= SLAB_TYPESAFE_BY_RCU
,
113 EXPORT_SYMBOL_GPL(smc_proto
);
115 struct proto smc_proto6
= {
117 .owner
= THIS_MODULE
,
118 .keepalive
= smc_set_keepalive
,
120 .unhash
= smc_unhash_sk
,
121 .obj_size
= sizeof(struct smc_sock
),
122 .h
.smc_hash
= &smc_v6_hashinfo
,
123 .slab_flags
= SLAB_TYPESAFE_BY_RCU
,
125 EXPORT_SYMBOL_GPL(smc_proto6
);
127 static void smc_restore_fallback_changes(struct smc_sock
*smc
)
129 smc
->clcsock
->file
->private_data
= smc
->sk
.sk_socket
;
130 smc
->clcsock
->file
= NULL
;
133 static int __smc_release(struct smc_sock
*smc
)
135 struct sock
*sk
= &smc
->sk
;
138 if (!smc
->use_fallback
) {
139 rc
= smc_close_active(smc
);
140 sock_set_flag(sk
, SOCK_DEAD
);
141 sk
->sk_shutdown
|= SHUTDOWN_MASK
;
143 if (sk
->sk_state
!= SMC_LISTEN
&& sk
->sk_state
!= SMC_INIT
)
144 sock_put(sk
); /* passive closing */
145 if (sk
->sk_state
== SMC_LISTEN
) {
146 /* wake up clcsock accept */
147 rc
= kernel_sock_shutdown(smc
->clcsock
, SHUT_RDWR
);
149 sk
->sk_state
= SMC_CLOSED
;
150 sk
->sk_state_change(sk
);
151 smc_restore_fallback_changes(smc
);
154 sk
->sk_prot
->unhash(sk
);
156 if (sk
->sk_state
== SMC_CLOSED
) {
159 smc_clcsock_release(smc
);
162 if (!smc
->use_fallback
)
163 smc_conn_free(&smc
->conn
);
169 static int smc_release(struct socket
*sock
)
171 struct sock
*sk
= sock
->sk
;
172 struct smc_sock
*smc
;
178 sock_hold(sk
); /* sock_put below */
181 /* cleanup for a dangling non-blocking connect */
182 if (smc
->connect_nonblock
&& sk
->sk_state
== SMC_INIT
)
183 tcp_abort(smc
->clcsock
->sk
, ECONNABORTED
);
184 flush_work(&smc
->connect_work
);
186 if (sk
->sk_state
== SMC_LISTEN
)
187 /* smc_close_non_accepted() is called and acquires
188 * sock lock for child sockets again
190 lock_sock_nested(sk
, SINGLE_DEPTH_NESTING
);
194 rc
= __smc_release(smc
);
201 sock_put(sk
); /* sock_hold above */
202 sock_put(sk
); /* final sock_put */
207 static void smc_destruct(struct sock
*sk
)
209 if (sk
->sk_state
!= SMC_CLOSED
)
211 if (!sock_flag(sk
, SOCK_DEAD
))
214 sk_refcnt_debug_dec(sk
);
217 static struct sock
*smc_sock_alloc(struct net
*net
, struct socket
*sock
,
220 struct smc_sock
*smc
;
224 prot
= (protocol
== SMCPROTO_SMC6
) ? &smc_proto6
: &smc_proto
;
225 sk
= sk_alloc(net
, PF_SMC
, GFP_KERNEL
, prot
, 0);
229 sock_init_data(sock
, sk
); /* sets sk_refcnt to 1 */
230 sk
->sk_state
= SMC_INIT
;
231 sk
->sk_destruct
= smc_destruct
;
232 sk
->sk_protocol
= protocol
;
234 INIT_WORK(&smc
->tcp_listen_work
, smc_tcp_listen_work
);
235 INIT_WORK(&smc
->connect_work
, smc_connect_work
);
236 INIT_DELAYED_WORK(&smc
->conn
.tx_work
, smc_tx_work
);
237 INIT_LIST_HEAD(&smc
->accept_q
);
238 spin_lock_init(&smc
->accept_q_lock
);
239 spin_lock_init(&smc
->conn
.send_lock
);
240 sk
->sk_prot
->hash(sk
);
241 sk_refcnt_debug_inc(sk
);
242 mutex_init(&smc
->clcsock_release_lock
);
247 static int smc_bind(struct socket
*sock
, struct sockaddr
*uaddr
,
250 struct sockaddr_in
*addr
= (struct sockaddr_in
*)uaddr
;
251 struct sock
*sk
= sock
->sk
;
252 struct smc_sock
*smc
;
257 /* replicate tests from inet_bind(), to be safe wrt. future changes */
259 if (addr_len
< sizeof(struct sockaddr_in
))
263 if (addr
->sin_family
!= AF_INET
&&
264 addr
->sin_family
!= AF_INET6
&&
265 addr
->sin_family
!= AF_UNSPEC
)
267 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
268 if (addr
->sin_family
== AF_UNSPEC
&&
269 addr
->sin_addr
.s_addr
!= htonl(INADDR_ANY
))
274 /* Check if socket is already active */
276 if (sk
->sk_state
!= SMC_INIT
|| smc
->connect_nonblock
)
279 smc
->clcsock
->sk
->sk_reuse
= sk
->sk_reuse
;
280 rc
= kernel_bind(smc
->clcsock
, uaddr
, addr_len
);
288 static void smc_copy_sock_settings(struct sock
*nsk
, struct sock
*osk
,
291 /* options we don't get control via setsockopt for */
292 nsk
->sk_type
= osk
->sk_type
;
293 nsk
->sk_sndbuf
= osk
->sk_sndbuf
;
294 nsk
->sk_rcvbuf
= osk
->sk_rcvbuf
;
295 nsk
->sk_sndtimeo
= osk
->sk_sndtimeo
;
296 nsk
->sk_rcvtimeo
= osk
->sk_rcvtimeo
;
297 nsk
->sk_mark
= osk
->sk_mark
;
298 nsk
->sk_priority
= osk
->sk_priority
;
299 nsk
->sk_rcvlowat
= osk
->sk_rcvlowat
;
300 nsk
->sk_bound_dev_if
= osk
->sk_bound_dev_if
;
301 nsk
->sk_err
= osk
->sk_err
;
303 nsk
->sk_flags
&= ~mask
;
304 nsk
->sk_flags
|= osk
->sk_flags
& mask
;
307 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
308 (1UL << SOCK_KEEPOPEN) | \
309 (1UL << SOCK_LINGER) | \
310 (1UL << SOCK_BROADCAST) | \
311 (1UL << SOCK_TIMESTAMP) | \
312 (1UL << SOCK_DBG) | \
313 (1UL << SOCK_RCVTSTAMP) | \
314 (1UL << SOCK_RCVTSTAMPNS) | \
315 (1UL << SOCK_LOCALROUTE) | \
316 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
317 (1UL << SOCK_RXQ_OVFL) | \
318 (1UL << SOCK_WIFI_STATUS) | \
319 (1UL << SOCK_NOFCS) | \
320 (1UL << SOCK_FILTER_LOCKED) | \
321 (1UL << SOCK_TSTAMP_NEW))
322 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
323 * clc socket (since smc is not called for these options from net/core)
325 static void smc_copy_sock_settings_to_clc(struct smc_sock
*smc
)
327 smc_copy_sock_settings(smc
->clcsock
->sk
, &smc
->sk
, SK_FLAGS_SMC_TO_CLC
);
330 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
331 (1UL << SOCK_KEEPOPEN) | \
332 (1UL << SOCK_LINGER) | \
334 /* copy only settings and flags relevant for smc from clc to smc socket */
335 static void smc_copy_sock_settings_to_smc(struct smc_sock
*smc
)
337 smc_copy_sock_settings(&smc
->sk
, smc
->clcsock
->sk
, SK_FLAGS_CLC_TO_SMC
);
340 /* register the new rmb on all links */
341 static int smcr_lgr_reg_rmbs(struct smc_link
*link
,
342 struct smc_buf_desc
*rmb_desc
)
344 struct smc_link_group
*lgr
= link
->lgr
;
347 rc
= smc_llc_flow_initiate(lgr
, SMC_LLC_FLOW_RKEY
);
350 /* protect against parallel smc_llc_cli_rkey_exchange() and
351 * parallel smcr_link_reg_rmb()
353 mutex_lock(&lgr
->llc_conf_mutex
);
354 for (i
= 0; i
< SMC_LINKS_PER_LGR_MAX
; i
++) {
355 if (lgr
->lnk
[i
].state
!= SMC_LNK_ACTIVE
)
357 rc
= smcr_link_reg_rmb(&lgr
->lnk
[i
], rmb_desc
);
362 /* exchange confirm_rkey msg with peer */
363 rc
= smc_llc_do_confirm_rkey(link
, rmb_desc
);
368 rmb_desc
->is_conf_rkey
= true;
370 mutex_unlock(&lgr
->llc_conf_mutex
);
371 smc_llc_flow_stop(lgr
, &lgr
->llc_flow_lcl
);
375 static int smcr_clnt_conf_first_link(struct smc_sock
*smc
)
377 struct smc_link
*link
= smc
->conn
.lnk
;
378 struct smc_llc_qentry
*qentry
;
381 /* receive CONFIRM LINK request from server over RoCE fabric */
382 qentry
= smc_llc_wait(link
->lgr
, NULL
, SMC_LLC_WAIT_TIME
,
383 SMC_LLC_CONFIRM_LINK
);
385 struct smc_clc_msg_decline dclc
;
387 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
388 SMC_CLC_DECLINE
, CLC_WAIT_TIME_SHORT
);
389 return rc
== -EAGAIN
? SMC_CLC_DECL_TIMEOUT_CL
: rc
;
391 smc_llc_save_peer_uid(qentry
);
392 rc
= smc_llc_eval_conf_link(qentry
, SMC_LLC_REQ
);
393 smc_llc_flow_qentry_del(&link
->lgr
->llc_flow_lcl
);
395 return SMC_CLC_DECL_RMBE_EC
;
397 rc
= smc_ib_modify_qp_rts(link
);
399 return SMC_CLC_DECL_ERR_RDYLNK
;
401 smc_wr_remember_qp_attr(link
);
403 if (smcr_link_reg_rmb(link
, smc
->conn
.rmb_desc
))
404 return SMC_CLC_DECL_ERR_REGRMB
;
406 /* confirm_rkey is implicit on 1st contact */
407 smc
->conn
.rmb_desc
->is_conf_rkey
= true;
409 /* send CONFIRM LINK response over RoCE fabric */
410 rc
= smc_llc_send_confirm_link(link
, SMC_LLC_RESP
);
412 return SMC_CLC_DECL_TIMEOUT_CL
;
414 smc_llc_link_active(link
);
415 smcr_lgr_set_type(link
->lgr
, SMC_LGR_SINGLE
);
417 /* optional 2nd link, receive ADD LINK request from server */
418 qentry
= smc_llc_wait(link
->lgr
, NULL
, SMC_LLC_WAIT_TIME
,
421 struct smc_clc_msg_decline dclc
;
423 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
424 SMC_CLC_DECLINE
, CLC_WAIT_TIME_SHORT
);
426 rc
= 0; /* no DECLINE received, go with one link */
429 smc_llc_flow_qentry_clr(&link
->lgr
->llc_flow_lcl
);
430 smc_llc_cli_add_link(link
, qentry
);
434 static void smcr_conn_save_peer_info(struct smc_sock
*smc
,
435 struct smc_clc_msg_accept_confirm
*clc
)
437 int bufsize
= smc_uncompress_bufsize(clc
->rmbe_size
);
439 smc
->conn
.peer_rmbe_idx
= clc
->rmbe_idx
;
440 smc
->conn
.local_tx_ctrl
.token
= ntohl(clc
->rmbe_alert_token
);
441 smc
->conn
.peer_rmbe_size
= bufsize
;
442 atomic_set(&smc
->conn
.peer_rmbe_space
, smc
->conn
.peer_rmbe_size
);
443 smc
->conn
.tx_off
= bufsize
* (smc
->conn
.peer_rmbe_idx
- 1);
446 static void smcd_conn_save_peer_info(struct smc_sock
*smc
,
447 struct smc_clc_msg_accept_confirm
*clc
)
449 int bufsize
= smc_uncompress_bufsize(clc
->dmbe_size
);
451 smc
->conn
.peer_rmbe_idx
= clc
->dmbe_idx
;
452 smc
->conn
.peer_token
= clc
->token
;
453 /* msg header takes up space in the buffer */
454 smc
->conn
.peer_rmbe_size
= bufsize
- sizeof(struct smcd_cdc_msg
);
455 atomic_set(&smc
->conn
.peer_rmbe_space
, smc
->conn
.peer_rmbe_size
);
456 smc
->conn
.tx_off
= bufsize
* smc
->conn
.peer_rmbe_idx
;
459 static void smc_conn_save_peer_info(struct smc_sock
*smc
,
460 struct smc_clc_msg_accept_confirm
*clc
)
462 if (smc
->conn
.lgr
->is_smcd
)
463 smcd_conn_save_peer_info(smc
, clc
);
465 smcr_conn_save_peer_info(smc
, clc
);
468 static void smc_link_save_peer_info(struct smc_link
*link
,
469 struct smc_clc_msg_accept_confirm
*clc
)
471 link
->peer_qpn
= ntoh24(clc
->qpn
);
472 memcpy(link
->peer_gid
, clc
->lcl
.gid
, SMC_GID_SIZE
);
473 memcpy(link
->peer_mac
, clc
->lcl
.mac
, sizeof(link
->peer_mac
));
474 link
->peer_psn
= ntoh24(clc
->psn
);
475 link
->peer_mtu
= clc
->qp_mtu
;
478 static void smc_switch_to_fallback(struct smc_sock
*smc
)
480 smc
->use_fallback
= true;
481 if (smc
->sk
.sk_socket
&& smc
->sk
.sk_socket
->file
) {
482 smc
->clcsock
->file
= smc
->sk
.sk_socket
->file
;
483 smc
->clcsock
->file
->private_data
= smc
->clcsock
;
484 smc
->clcsock
->wq
.fasync_list
=
485 smc
->sk
.sk_socket
->wq
.fasync_list
;
489 /* fall back during connect */
490 static int smc_connect_fallback(struct smc_sock
*smc
, int reason_code
)
492 smc_switch_to_fallback(smc
);
493 smc
->fallback_rsn
= reason_code
;
494 smc_copy_sock_settings_to_clc(smc
);
495 smc
->connect_nonblock
= 0;
496 if (smc
->sk
.sk_state
== SMC_INIT
)
497 smc
->sk
.sk_state
= SMC_ACTIVE
;
501 /* decline and fall back during connect */
502 static int smc_connect_decline_fallback(struct smc_sock
*smc
, int reason_code
)
506 if (reason_code
< 0) { /* error, fallback is not possible */
507 if (smc
->sk
.sk_state
== SMC_INIT
)
508 sock_put(&smc
->sk
); /* passive closing */
511 if (reason_code
!= SMC_CLC_DECL_PEERDECL
) {
512 rc
= smc_clc_send_decline(smc
, reason_code
);
514 if (smc
->sk
.sk_state
== SMC_INIT
)
515 sock_put(&smc
->sk
); /* passive closing */
519 return smc_connect_fallback(smc
, reason_code
);
522 /* abort connecting */
523 static int smc_connect_abort(struct smc_sock
*smc
, int reason_code
,
526 bool is_smcd
= smc
->conn
.lgr
->is_smcd
;
528 if (local_contact
== SMC_FIRST_CONTACT
)
529 smc_lgr_cleanup_early(&smc
->conn
);
531 smc_conn_free(&smc
->conn
);
533 /* there is only one lgr role for SMC-D; use server lock */
534 mutex_unlock(&smc_server_lgr_pending
);
536 mutex_unlock(&smc_client_lgr_pending
);
538 smc
->connect_nonblock
= 0;
542 /* check if there is a rdma device available for this connection. */
543 /* called for connect and listen */
544 static int smc_find_rdma_device(struct smc_sock
*smc
, struct smc_init_info
*ini
)
546 /* PNET table look up: search active ib_device and port
547 * within same PNETID that also contains the ethernet device
548 * used for the internal TCP socket
550 smc_pnet_find_roce_resource(smc
->clcsock
->sk
, ini
);
552 return SMC_CLC_DECL_NOSMCRDEV
;
556 /* check if there is an ISM device available for this connection. */
557 /* called for connect and listen */
558 static int smc_find_ism_device(struct smc_sock
*smc
, struct smc_init_info
*ini
)
560 /* Find ISM device with same PNETID as connecting interface */
561 smc_pnet_find_ism_resource(smc
->clcsock
->sk
, ini
);
563 return SMC_CLC_DECL_NOSMCDDEV
;
567 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
568 static int smc_connect_ism_vlan_setup(struct smc_sock
*smc
,
569 struct smc_init_info
*ini
)
571 if (ini
->vlan_id
&& smc_ism_get_vlan(ini
->ism_dev
, ini
->vlan_id
))
572 return SMC_CLC_DECL_ISMVLANERR
;
576 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
577 * used, the VLAN ID will be registered again during the connection setup.
579 static int smc_connect_ism_vlan_cleanup(struct smc_sock
*smc
, bool is_smcd
,
580 struct smc_init_info
*ini
)
584 if (ini
->vlan_id
&& smc_ism_put_vlan(ini
->ism_dev
, ini
->vlan_id
))
585 return SMC_CLC_DECL_CNFERR
;
589 /* CLC handshake during connect */
590 static int smc_connect_clc(struct smc_sock
*smc
, int smc_type
,
591 struct smc_clc_msg_accept_confirm
*aclc
,
592 struct smc_init_info
*ini
)
596 /* do inband token exchange */
597 rc
= smc_clc_send_proposal(smc
, smc_type
, ini
);
600 /* receive SMC Accept CLC message */
601 return smc_clc_wait_msg(smc
, aclc
, sizeof(*aclc
), SMC_CLC_ACCEPT
,
605 /* setup for RDMA connection of client */
606 static int smc_connect_rdma(struct smc_sock
*smc
,
607 struct smc_clc_msg_accept_confirm
*aclc
,
608 struct smc_init_info
*ini
)
610 int i
, reason_code
= 0;
611 struct smc_link
*link
;
613 ini
->is_smcd
= false;
614 ini
->ib_lcl
= &aclc
->lcl
;
615 ini
->ib_clcqpn
= ntoh24(aclc
->qpn
);
616 ini
->srv_first_contact
= aclc
->hdr
.flag
;
618 mutex_lock(&smc_client_lgr_pending
);
619 reason_code
= smc_conn_create(smc
, ini
);
621 mutex_unlock(&smc_client_lgr_pending
);
625 smc_conn_save_peer_info(smc
, aclc
);
627 if (ini
->cln_first_contact
== SMC_FIRST_CONTACT
) {
628 link
= smc
->conn
.lnk
;
630 /* set link that was assigned by server */
632 for (i
= 0; i
< SMC_LINKS_PER_LGR_MAX
; i
++) {
633 struct smc_link
*l
= &smc
->conn
.lgr
->lnk
[i
];
635 if (l
->peer_qpn
== ntoh24(aclc
->qpn
)) {
641 return smc_connect_abort(smc
, SMC_CLC_DECL_NOSRVLINK
,
642 ini
->cln_first_contact
);
643 smc
->conn
.lnk
= link
;
646 /* create send buffer and rmb */
647 if (smc_buf_create(smc
, false))
648 return smc_connect_abort(smc
, SMC_CLC_DECL_MEM
,
649 ini
->cln_first_contact
);
651 if (ini
->cln_first_contact
== SMC_FIRST_CONTACT
)
652 smc_link_save_peer_info(link
, aclc
);
654 if (smc_rmb_rtoken_handling(&smc
->conn
, link
, aclc
))
655 return smc_connect_abort(smc
, SMC_CLC_DECL_ERR_RTOK
,
656 ini
->cln_first_contact
);
661 if (ini
->cln_first_contact
== SMC_FIRST_CONTACT
) {
662 if (smc_ib_ready_link(link
))
663 return smc_connect_abort(smc
, SMC_CLC_DECL_ERR_RDYLNK
,
664 ini
->cln_first_contact
);
666 if (smcr_lgr_reg_rmbs(link
, smc
->conn
.rmb_desc
))
667 return smc_connect_abort(smc
, SMC_CLC_DECL_ERR_REGRMB
,
668 ini
->cln_first_contact
);
670 smc_rmb_sync_sg_for_device(&smc
->conn
);
672 reason_code
= smc_clc_send_confirm(smc
);
674 return smc_connect_abort(smc
, reason_code
,
675 ini
->cln_first_contact
);
679 if (ini
->cln_first_contact
== SMC_FIRST_CONTACT
) {
680 /* QP confirmation over RoCE fabric */
681 smc_llc_flow_initiate(link
->lgr
, SMC_LLC_FLOW_ADD_LINK
);
682 reason_code
= smcr_clnt_conf_first_link(smc
);
683 smc_llc_flow_stop(link
->lgr
, &link
->lgr
->llc_flow_lcl
);
685 return smc_connect_abort(smc
, reason_code
,
686 ini
->cln_first_contact
);
688 mutex_unlock(&smc_client_lgr_pending
);
690 smc_copy_sock_settings_to_clc(smc
);
691 smc
->connect_nonblock
= 0;
692 if (smc
->sk
.sk_state
== SMC_INIT
)
693 smc
->sk
.sk_state
= SMC_ACTIVE
;
698 /* setup for ISM connection of client */
699 static int smc_connect_ism(struct smc_sock
*smc
,
700 struct smc_clc_msg_accept_confirm
*aclc
,
701 struct smc_init_info
*ini
)
706 ini
->ism_gid
= aclc
->gid
;
707 ini
->srv_first_contact
= aclc
->hdr
.flag
;
709 /* there is only one lgr role for SMC-D; use server lock */
710 mutex_lock(&smc_server_lgr_pending
);
711 rc
= smc_conn_create(smc
, ini
);
713 mutex_unlock(&smc_server_lgr_pending
);
717 /* Create send and receive buffers */
718 if (smc_buf_create(smc
, true))
719 return smc_connect_abort(smc
, SMC_CLC_DECL_MEM
,
720 ini
->cln_first_contact
);
722 smc_conn_save_peer_info(smc
, aclc
);
727 rc
= smc_clc_send_confirm(smc
);
729 return smc_connect_abort(smc
, rc
, ini
->cln_first_contact
);
730 mutex_unlock(&smc_server_lgr_pending
);
732 smc_copy_sock_settings_to_clc(smc
);
733 smc
->connect_nonblock
= 0;
734 if (smc
->sk
.sk_state
== SMC_INIT
)
735 smc
->sk
.sk_state
= SMC_ACTIVE
;
740 /* perform steps before actually connecting */
741 static int __smc_connect(struct smc_sock
*smc
)
743 bool ism_supported
= false, rdma_supported
= false;
744 struct smc_clc_msg_accept_confirm aclc
;
745 struct smc_init_info ini
= {0};
749 if (smc
->use_fallback
)
750 return smc_connect_fallback(smc
, smc
->fallback_rsn
);
752 /* if peer has not signalled SMC-capability, fall back */
753 if (!tcp_sk(smc
->clcsock
->sk
)->syn_smc
)
754 return smc_connect_fallback(smc
, SMC_CLC_DECL_PEERNOSMC
);
756 /* IPSec connections opt out of SMC-R optimizations */
757 if (using_ipsec(smc
))
758 return smc_connect_decline_fallback(smc
, SMC_CLC_DECL_IPSEC
);
760 /* get vlan id from IP device */
761 if (smc_vlan_by_tcpsk(smc
->clcsock
, &ini
))
762 return smc_connect_decline_fallback(smc
,
763 SMC_CLC_DECL_GETVLANERR
);
765 /* check if there is an ism device available */
766 if (!smc_find_ism_device(smc
, &ini
) &&
767 !smc_connect_ism_vlan_setup(smc
, &ini
)) {
768 /* ISM is supported for this connection */
769 ism_supported
= true;
770 smc_type
= SMC_TYPE_D
;
773 /* check if there is a rdma device available */
774 if (!smc_find_rdma_device(smc
, &ini
)) {
775 /* RDMA is supported for this connection */
776 rdma_supported
= true;
778 smc_type
= SMC_TYPE_B
; /* both */
780 smc_type
= SMC_TYPE_R
; /* only RDMA */
783 /* if neither ISM nor RDMA are supported, fallback */
784 if (!rdma_supported
&& !ism_supported
)
785 return smc_connect_decline_fallback(smc
, SMC_CLC_DECL_NOSMCDEV
);
787 /* perform CLC handshake */
788 rc
= smc_connect_clc(smc
, smc_type
, &aclc
, &ini
);
790 smc_connect_ism_vlan_cleanup(smc
, ism_supported
, &ini
);
791 return smc_connect_decline_fallback(smc
, rc
);
794 /* depending on previous steps, connect using rdma or ism */
795 if (rdma_supported
&& aclc
.hdr
.path
== SMC_TYPE_R
)
796 rc
= smc_connect_rdma(smc
, &aclc
, &ini
);
797 else if (ism_supported
&& aclc
.hdr
.path
== SMC_TYPE_D
)
798 rc
= smc_connect_ism(smc
, &aclc
, &ini
);
800 rc
= SMC_CLC_DECL_MODEUNSUPP
;
802 smc_connect_ism_vlan_cleanup(smc
, ism_supported
, &ini
);
803 return smc_connect_decline_fallback(smc
, rc
);
806 smc_connect_ism_vlan_cleanup(smc
, ism_supported
, &ini
);
810 static void smc_connect_work(struct work_struct
*work
)
812 struct smc_sock
*smc
= container_of(work
, struct smc_sock
,
814 long timeo
= smc
->sk
.sk_sndtimeo
;
818 timeo
= MAX_SCHEDULE_TIMEOUT
;
819 lock_sock(smc
->clcsock
->sk
);
820 if (smc
->clcsock
->sk
->sk_err
) {
821 smc
->sk
.sk_err
= smc
->clcsock
->sk
->sk_err
;
822 } else if ((1 << smc
->clcsock
->sk
->sk_state
) &
823 (TCPF_SYN_SENT
| TCP_SYN_RECV
)) {
824 rc
= sk_stream_wait_connect(smc
->clcsock
->sk
, &timeo
);
825 if ((rc
== -EPIPE
) &&
826 ((1 << smc
->clcsock
->sk
->sk_state
) &
827 (TCPF_ESTABLISHED
| TCPF_CLOSE_WAIT
)))
830 release_sock(smc
->clcsock
->sk
);
832 if (rc
!= 0 || smc
->sk
.sk_err
) {
833 smc
->sk
.sk_state
= SMC_CLOSED
;
834 if (rc
== -EPIPE
|| rc
== -EAGAIN
)
835 smc
->sk
.sk_err
= EPIPE
;
836 else if (signal_pending(current
))
837 smc
->sk
.sk_err
= -sock_intr_errno(timeo
);
838 sock_put(&smc
->sk
); /* passive closing */
842 rc
= __smc_connect(smc
);
844 smc
->sk
.sk_err
= -rc
;
847 if (!sock_flag(&smc
->sk
, SOCK_DEAD
)) {
848 if (smc
->sk
.sk_err
) {
849 smc
->sk
.sk_state_change(&smc
->sk
);
850 } else { /* allow polling before and after fallback decision */
851 smc
->clcsock
->sk
->sk_write_space(smc
->clcsock
->sk
);
852 smc
->sk
.sk_write_space(&smc
->sk
);
855 release_sock(&smc
->sk
);
858 static int smc_connect(struct socket
*sock
, struct sockaddr
*addr
,
861 struct sock
*sk
= sock
->sk
;
862 struct smc_sock
*smc
;
867 /* separate smc parameter checking to be safe */
868 if (alen
< sizeof(addr
->sa_family
))
870 if (addr
->sa_family
!= AF_INET
&& addr
->sa_family
!= AF_INET6
)
874 switch (sk
->sk_state
) {
885 smc_copy_sock_settings_to_clc(smc
);
886 tcp_sk(smc
->clcsock
->sk
)->syn_smc
= 1;
887 if (smc
->connect_nonblock
) {
891 rc
= kernel_connect(smc
->clcsock
, addr
, alen
, flags
);
892 if (rc
&& rc
!= -EINPROGRESS
)
895 sock_hold(&smc
->sk
); /* sock put in passive closing */
896 if (smc
->use_fallback
)
898 if (flags
& O_NONBLOCK
) {
899 if (schedule_work(&smc
->connect_work
))
900 smc
->connect_nonblock
= 1;
903 rc
= __smc_connect(smc
);
907 rc
= 0; /* success cases including fallback */
916 static int smc_clcsock_accept(struct smc_sock
*lsmc
, struct smc_sock
**new_smc
)
918 struct socket
*new_clcsock
= NULL
;
919 struct sock
*lsk
= &lsmc
->sk
;
924 new_sk
= smc_sock_alloc(sock_net(lsk
), NULL
, lsk
->sk_protocol
);
927 lsk
->sk_err
= ENOMEM
;
932 *new_smc
= smc_sk(new_sk
);
934 mutex_lock(&lsmc
->clcsock_release_lock
);
936 rc
= kernel_accept(lsmc
->clcsock
, &new_clcsock
, 0);
937 mutex_unlock(&lsmc
->clcsock_release_lock
);
941 if (rc
< 0 || lsk
->sk_state
== SMC_CLOSED
) {
942 new_sk
->sk_prot
->unhash(new_sk
);
944 sock_release(new_clcsock
);
945 new_sk
->sk_state
= SMC_CLOSED
;
946 sock_set_flag(new_sk
, SOCK_DEAD
);
947 sock_put(new_sk
); /* final */
952 (*new_smc
)->clcsock
= new_clcsock
;
957 /* add a just created sock to the accept queue of the listen sock as
958 * candidate for a following socket accept call from user space
960 static void smc_accept_enqueue(struct sock
*parent
, struct sock
*sk
)
962 struct smc_sock
*par
= smc_sk(parent
);
964 sock_hold(sk
); /* sock_put in smc_accept_unlink () */
965 spin_lock(&par
->accept_q_lock
);
966 list_add_tail(&smc_sk(sk
)->accept_q
, &par
->accept_q
);
967 spin_unlock(&par
->accept_q_lock
);
968 sk_acceptq_added(parent
);
971 /* remove a socket from the accept queue of its parental listening socket */
972 static void smc_accept_unlink(struct sock
*sk
)
974 struct smc_sock
*par
= smc_sk(sk
)->listen_smc
;
976 spin_lock(&par
->accept_q_lock
);
977 list_del_init(&smc_sk(sk
)->accept_q
);
978 spin_unlock(&par
->accept_q_lock
);
979 sk_acceptq_removed(&smc_sk(sk
)->listen_smc
->sk
);
980 sock_put(sk
); /* sock_hold in smc_accept_enqueue */
983 /* remove a sock from the accept queue to bind it to a new socket created
984 * for a socket accept call from user space
986 struct sock
*smc_accept_dequeue(struct sock
*parent
,
987 struct socket
*new_sock
)
989 struct smc_sock
*isk
, *n
;
992 list_for_each_entry_safe(isk
, n
, &smc_sk(parent
)->accept_q
, accept_q
) {
993 new_sk
= (struct sock
*)isk
;
995 smc_accept_unlink(new_sk
);
996 if (new_sk
->sk_state
== SMC_CLOSED
) {
997 new_sk
->sk_prot
->unhash(new_sk
);
999 sock_release(isk
->clcsock
);
1000 isk
->clcsock
= NULL
;
1002 sock_put(new_sk
); /* final */
1006 sock_graft(new_sk
, new_sock
);
1007 if (isk
->use_fallback
) {
1008 smc_sk(new_sk
)->clcsock
->file
= new_sock
->file
;
1009 isk
->clcsock
->file
->private_data
= isk
->clcsock
;
1017 /* clean up for a created but never accepted sock */
1018 void smc_close_non_accepted(struct sock
*sk
)
1020 struct smc_sock
*smc
= smc_sk(sk
);
1022 sock_hold(sk
); /* sock_put below */
1024 if (!sk
->sk_lingertime
)
1025 /* wait for peer closing */
1026 sk
->sk_lingertime
= SMC_MAX_STREAM_WAIT_TIMEOUT
;
1029 sock_put(sk
); /* sock_hold above */
1030 sock_put(sk
); /* final sock_put */
1033 static int smcr_serv_conf_first_link(struct smc_sock
*smc
)
1035 struct smc_link
*link
= smc
->conn
.lnk
;
1036 struct smc_llc_qentry
*qentry
;
1039 if (smcr_link_reg_rmb(link
, smc
->conn
.rmb_desc
))
1040 return SMC_CLC_DECL_ERR_REGRMB
;
1042 /* send CONFIRM LINK request to client over the RoCE fabric */
1043 rc
= smc_llc_send_confirm_link(link
, SMC_LLC_REQ
);
1045 return SMC_CLC_DECL_TIMEOUT_CL
;
1047 /* receive CONFIRM LINK response from client over the RoCE fabric */
1048 qentry
= smc_llc_wait(link
->lgr
, link
, SMC_LLC_WAIT_TIME
,
1049 SMC_LLC_CONFIRM_LINK
);
1051 struct smc_clc_msg_decline dclc
;
1053 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
1054 SMC_CLC_DECLINE
, CLC_WAIT_TIME_SHORT
);
1055 return rc
== -EAGAIN
? SMC_CLC_DECL_TIMEOUT_CL
: rc
;
1057 smc_llc_save_peer_uid(qentry
);
1058 rc
= smc_llc_eval_conf_link(qentry
, SMC_LLC_RESP
);
1059 smc_llc_flow_qentry_del(&link
->lgr
->llc_flow_lcl
);
1061 return SMC_CLC_DECL_RMBE_EC
;
1063 /* confirm_rkey is implicit on 1st contact */
1064 smc
->conn
.rmb_desc
->is_conf_rkey
= true;
1066 smc_llc_link_active(link
);
1067 smcr_lgr_set_type(link
->lgr
, SMC_LGR_SINGLE
);
1069 /* initial contact - try to establish second link */
1070 smc_llc_srv_add_link(link
);
1074 /* listen worker: finish */
1075 static void smc_listen_out(struct smc_sock
*new_smc
)
1077 struct smc_sock
*lsmc
= new_smc
->listen_smc
;
1078 struct sock
*newsmcsk
= &new_smc
->sk
;
1080 if (lsmc
->sk
.sk_state
== SMC_LISTEN
) {
1081 lock_sock_nested(&lsmc
->sk
, SINGLE_DEPTH_NESTING
);
1082 smc_accept_enqueue(&lsmc
->sk
, newsmcsk
);
1083 release_sock(&lsmc
->sk
);
1084 } else { /* no longer listening */
1085 smc_close_non_accepted(newsmcsk
);
1088 /* Wake up accept */
1089 lsmc
->sk
.sk_data_ready(&lsmc
->sk
);
1090 sock_put(&lsmc
->sk
); /* sock_hold in smc_tcp_listen_work */
1093 /* listen worker: finish in state connected */
1094 static void smc_listen_out_connected(struct smc_sock
*new_smc
)
1096 struct sock
*newsmcsk
= &new_smc
->sk
;
1098 sk_refcnt_debug_inc(newsmcsk
);
1099 if (newsmcsk
->sk_state
== SMC_INIT
)
1100 newsmcsk
->sk_state
= SMC_ACTIVE
;
1102 smc_listen_out(new_smc
);
1105 /* listen worker: finish in error state */
1106 static void smc_listen_out_err(struct smc_sock
*new_smc
)
1108 struct sock
*newsmcsk
= &new_smc
->sk
;
1110 if (newsmcsk
->sk_state
== SMC_INIT
)
1111 sock_put(&new_smc
->sk
); /* passive closing */
1112 newsmcsk
->sk_state
= SMC_CLOSED
;
1114 smc_listen_out(new_smc
);
1117 /* listen worker: decline and fall back if possible */
1118 static void smc_listen_decline(struct smc_sock
*new_smc
, int reason_code
,
1121 /* RDMA setup failed, switch back to TCP */
1122 if (local_contact
== SMC_FIRST_CONTACT
)
1123 smc_lgr_cleanup_early(&new_smc
->conn
);
1125 smc_conn_free(&new_smc
->conn
);
1126 if (reason_code
< 0) { /* error, no fallback possible */
1127 smc_listen_out_err(new_smc
);
1130 smc_switch_to_fallback(new_smc
);
1131 new_smc
->fallback_rsn
= reason_code
;
1132 if (reason_code
&& reason_code
!= SMC_CLC_DECL_PEERDECL
) {
1133 if (smc_clc_send_decline(new_smc
, reason_code
) < 0) {
1134 smc_listen_out_err(new_smc
);
1138 smc_listen_out_connected(new_smc
);
1141 /* listen worker: check prefixes */
1142 static int smc_listen_prfx_check(struct smc_sock
*new_smc
,
1143 struct smc_clc_msg_proposal
*pclc
)
1145 struct smc_clc_msg_proposal_prefix
*pclc_prfx
;
1146 struct socket
*newclcsock
= new_smc
->clcsock
;
1148 pclc_prfx
= smc_clc_proposal_get_prefix(pclc
);
1149 if (smc_clc_prfx_match(newclcsock
, pclc_prfx
))
1150 return SMC_CLC_DECL_DIFFPREFIX
;
1155 /* listen worker: initialize connection and buffers */
1156 static int smc_listen_rdma_init(struct smc_sock
*new_smc
,
1157 struct smc_init_info
*ini
)
1161 /* allocate connection / link group */
1162 rc
= smc_conn_create(new_smc
, ini
);
1166 /* create send buffer and rmb */
1167 if (smc_buf_create(new_smc
, false))
1168 return SMC_CLC_DECL_MEM
;
1173 /* listen worker: initialize connection and buffers for SMC-D */
1174 static int smc_listen_ism_init(struct smc_sock
*new_smc
,
1175 struct smc_clc_msg_proposal
*pclc
,
1176 struct smc_init_info
*ini
)
1178 struct smc_clc_msg_smcd
*pclc_smcd
;
1181 pclc_smcd
= smc_get_clc_msg_smcd(pclc
);
1182 ini
->ism_gid
= pclc_smcd
->gid
;
1183 rc
= smc_conn_create(new_smc
, ini
);
1187 /* Check if peer can be reached via ISM device */
1188 if (smc_ism_cantalk(new_smc
->conn
.lgr
->peer_gid
,
1189 new_smc
->conn
.lgr
->vlan_id
,
1190 new_smc
->conn
.lgr
->smcd
)) {
1191 if (ini
->cln_first_contact
== SMC_FIRST_CONTACT
)
1192 smc_lgr_cleanup_early(&new_smc
->conn
);
1194 smc_conn_free(&new_smc
->conn
);
1195 return SMC_CLC_DECL_SMCDNOTALK
;
1198 /* Create send and receive buffers */
1199 if (smc_buf_create(new_smc
, true)) {
1200 if (ini
->cln_first_contact
== SMC_FIRST_CONTACT
)
1201 smc_lgr_cleanup_early(&new_smc
->conn
);
1203 smc_conn_free(&new_smc
->conn
);
1204 return SMC_CLC_DECL_MEM
;
1210 /* listen worker: register buffers */
1211 static int smc_listen_rdma_reg(struct smc_sock
*new_smc
, int local_contact
)
1213 struct smc_connection
*conn
= &new_smc
->conn
;
1215 if (local_contact
!= SMC_FIRST_CONTACT
) {
1216 if (smcr_lgr_reg_rmbs(conn
->lnk
, conn
->rmb_desc
))
1217 return SMC_CLC_DECL_ERR_REGRMB
;
1219 smc_rmb_sync_sg_for_device(&new_smc
->conn
);
1224 /* listen worker: finish RDMA setup */
1225 static int smc_listen_rdma_finish(struct smc_sock
*new_smc
,
1226 struct smc_clc_msg_accept_confirm
*cclc
,
1229 struct smc_link
*link
= new_smc
->conn
.lnk
;
1230 int reason_code
= 0;
1232 if (local_contact
== SMC_FIRST_CONTACT
)
1233 smc_link_save_peer_info(link
, cclc
);
1235 if (smc_rmb_rtoken_handling(&new_smc
->conn
, link
, cclc
)) {
1236 reason_code
= SMC_CLC_DECL_ERR_RTOK
;
1240 if (local_contact
== SMC_FIRST_CONTACT
) {
1241 if (smc_ib_ready_link(link
)) {
1242 reason_code
= SMC_CLC_DECL_ERR_RDYLNK
;
1245 /* QP confirmation over RoCE fabric */
1246 smc_llc_flow_initiate(link
->lgr
, SMC_LLC_FLOW_ADD_LINK
);
1247 reason_code
= smcr_serv_conf_first_link(new_smc
);
1248 smc_llc_flow_stop(link
->lgr
, &link
->lgr
->llc_flow_lcl
);
1255 smc_listen_decline(new_smc
, reason_code
, local_contact
);
1259 /* setup for RDMA connection of server */
1260 static void smc_listen_work(struct work_struct
*work
)
1262 struct smc_sock
*new_smc
= container_of(work
, struct smc_sock
,
1264 struct socket
*newclcsock
= new_smc
->clcsock
;
1265 struct smc_clc_msg_accept_confirm cclc
;
1266 struct smc_clc_msg_proposal
*pclc
;
1267 struct smc_init_info ini
= {0};
1268 bool ism_supported
= false;
1269 u8 buf
[SMC_CLC_MAX_LEN
];
1272 if (new_smc
->listen_smc
->sk
.sk_state
!= SMC_LISTEN
)
1273 return smc_listen_out_err(new_smc
);
1275 if (new_smc
->use_fallback
) {
1276 smc_listen_out_connected(new_smc
);
1280 /* check if peer is smc capable */
1281 if (!tcp_sk(newclcsock
->sk
)->syn_smc
) {
1282 smc_switch_to_fallback(new_smc
);
1283 new_smc
->fallback_rsn
= SMC_CLC_DECL_PEERNOSMC
;
1284 smc_listen_out_connected(new_smc
);
1288 /* do inband token exchange -
1289 * wait for and receive SMC Proposal CLC message
1291 pclc
= (struct smc_clc_msg_proposal
*)&buf
;
1292 rc
= smc_clc_wait_msg(new_smc
, pclc
, SMC_CLC_MAX_LEN
,
1293 SMC_CLC_PROPOSAL
, CLC_WAIT_TIME
);
1297 /* IPSec connections opt out of SMC-R optimizations */
1298 if (using_ipsec(new_smc
)) {
1299 rc
= SMC_CLC_DECL_IPSEC
;
1303 /* check for matching IP prefix and subnet length */
1304 rc
= smc_listen_prfx_check(new_smc
, pclc
);
1308 /* get vlan id from IP device */
1309 if (smc_vlan_by_tcpsk(new_smc
->clcsock
, &ini
)) {
1310 rc
= SMC_CLC_DECL_GETVLANERR
;
1314 mutex_lock(&smc_server_lgr_pending
);
1315 smc_close_init(new_smc
);
1316 smc_rx_init(new_smc
);
1317 smc_tx_init(new_smc
);
1319 /* check if ISM is available */
1320 if (pclc
->hdr
.path
== SMC_TYPE_D
|| pclc
->hdr
.path
== SMC_TYPE_B
) {
1321 ini
.is_smcd
= true; /* prepare ISM check */
1322 rc
= smc_find_ism_device(new_smc
, &ini
);
1324 rc
= smc_listen_ism_init(new_smc
, pclc
, &ini
);
1326 ism_supported
= true;
1327 else if (pclc
->hdr
.path
== SMC_TYPE_D
)
1328 goto out_unlock
; /* skip RDMA and decline */
1331 /* check if RDMA is available */
1332 if (!ism_supported
) { /* SMC_TYPE_R or SMC_TYPE_B */
1333 /* prepare RDMA check */
1334 ini
.is_smcd
= false;
1336 ini
.ib_lcl
= &pclc
->lcl
;
1337 rc
= smc_find_rdma_device(new_smc
, &ini
);
1339 /* no RDMA device found */
1340 if (pclc
->hdr
.path
== SMC_TYPE_B
)
1341 /* neither ISM nor RDMA device found */
1342 rc
= SMC_CLC_DECL_NOSMCDEV
;
1345 rc
= smc_listen_rdma_init(new_smc
, &ini
);
1348 rc
= smc_listen_rdma_reg(new_smc
, ini
.cln_first_contact
);
1353 /* send SMC Accept CLC message */
1354 rc
= smc_clc_send_accept(new_smc
, ini
.cln_first_contact
);
1358 /* SMC-D does not need this lock any more */
1360 mutex_unlock(&smc_server_lgr_pending
);
1362 /* receive SMC Confirm CLC message */
1363 rc
= smc_clc_wait_msg(new_smc
, &cclc
, sizeof(cclc
),
1364 SMC_CLC_CONFIRM
, CLC_WAIT_TIME
);
1372 if (!ism_supported
) {
1373 rc
= smc_listen_rdma_finish(new_smc
, &cclc
,
1374 ini
.cln_first_contact
);
1375 mutex_unlock(&smc_server_lgr_pending
);
1379 smc_conn_save_peer_info(new_smc
, &cclc
);
1380 smc_listen_out_connected(new_smc
);
1384 mutex_unlock(&smc_server_lgr_pending
);
1386 smc_listen_decline(new_smc
, rc
, ini
.cln_first_contact
);
1389 static void smc_tcp_listen_work(struct work_struct
*work
)
1391 struct smc_sock
*lsmc
= container_of(work
, struct smc_sock
,
1393 struct sock
*lsk
= &lsmc
->sk
;
1394 struct smc_sock
*new_smc
;
1398 while (lsk
->sk_state
== SMC_LISTEN
) {
1399 rc
= smc_clcsock_accept(lsmc
, &new_smc
);
1405 new_smc
->listen_smc
= lsmc
;
1406 new_smc
->use_fallback
= lsmc
->use_fallback
;
1407 new_smc
->fallback_rsn
= lsmc
->fallback_rsn
;
1408 sock_hold(lsk
); /* sock_put in smc_listen_work */
1409 INIT_WORK(&new_smc
->smc_listen_work
, smc_listen_work
);
1410 smc_copy_sock_settings_to_smc(new_smc
);
1411 new_smc
->sk
.sk_sndbuf
= lsmc
->sk
.sk_sndbuf
;
1412 new_smc
->sk
.sk_rcvbuf
= lsmc
->sk
.sk_rcvbuf
;
1413 sock_hold(&new_smc
->sk
); /* sock_put in passive closing */
1414 if (!schedule_work(&new_smc
->smc_listen_work
))
1415 sock_put(&new_smc
->sk
);
1420 sock_put(&lsmc
->sk
); /* sock_hold in smc_listen */
1423 static int smc_listen(struct socket
*sock
, int backlog
)
1425 struct sock
*sk
= sock
->sk
;
1426 struct smc_sock
*smc
;
1433 if ((sk
->sk_state
!= SMC_INIT
&& sk
->sk_state
!= SMC_LISTEN
) ||
1434 smc
->connect_nonblock
)
1438 if (sk
->sk_state
== SMC_LISTEN
) {
1439 sk
->sk_max_ack_backlog
= backlog
;
1442 /* some socket options are handled in core, so we could not apply
1443 * them to the clc socket -- copy smc socket options to clc socket
1445 smc_copy_sock_settings_to_clc(smc
);
1446 if (!smc
->use_fallback
)
1447 tcp_sk(smc
->clcsock
->sk
)->syn_smc
= 1;
1449 rc
= kernel_listen(smc
->clcsock
, backlog
);
1452 sk
->sk_max_ack_backlog
= backlog
;
1453 sk
->sk_ack_backlog
= 0;
1454 sk
->sk_state
= SMC_LISTEN
;
1455 sock_hold(sk
); /* sock_hold in tcp_listen_worker */
1456 if (!schedule_work(&smc
->tcp_listen_work
))
1464 static int smc_accept(struct socket
*sock
, struct socket
*new_sock
,
1465 int flags
, bool kern
)
1467 struct sock
*sk
= sock
->sk
, *nsk
;
1468 DECLARE_WAITQUEUE(wait
, current
);
1469 struct smc_sock
*lsmc
;
1474 sock_hold(sk
); /* sock_put below */
1477 if (lsmc
->sk
.sk_state
!= SMC_LISTEN
) {
1483 /* Wait for an incoming connection */
1484 timeo
= sock_rcvtimeo(sk
, flags
& O_NONBLOCK
);
1485 add_wait_queue_exclusive(sk_sleep(sk
), &wait
);
1486 while (!(nsk
= smc_accept_dequeue(sk
, new_sock
))) {
1487 set_current_state(TASK_INTERRUPTIBLE
);
1493 timeo
= schedule_timeout(timeo
);
1494 /* wakeup by sk_data_ready in smc_listen_work() */
1495 sched_annotate_sleep();
1497 if (signal_pending(current
)) {
1498 rc
= sock_intr_errno(timeo
);
1502 set_current_state(TASK_RUNNING
);
1503 remove_wait_queue(sk_sleep(sk
), &wait
);
1506 rc
= sock_error(nsk
);
1511 if (lsmc
->sockopt_defer_accept
&& !(flags
& O_NONBLOCK
)) {
1512 /* wait till data arrives on the socket */
1513 timeo
= msecs_to_jiffies(lsmc
->sockopt_defer_accept
*
1515 if (smc_sk(nsk
)->use_fallback
) {
1516 struct sock
*clcsk
= smc_sk(nsk
)->clcsock
->sk
;
1519 if (skb_queue_empty(&clcsk
->sk_receive_queue
))
1520 sk_wait_data(clcsk
, &timeo
, NULL
);
1521 release_sock(clcsk
);
1522 } else if (!atomic_read(&smc_sk(nsk
)->conn
.bytes_to_rcv
)) {
1524 smc_rx_wait(smc_sk(nsk
), &timeo
, smc_rx_data_available
);
1530 sock_put(sk
); /* sock_hold above */
1534 static int smc_getname(struct socket
*sock
, struct sockaddr
*addr
,
1537 struct smc_sock
*smc
;
1539 if (peer
&& (sock
->sk
->sk_state
!= SMC_ACTIVE
) &&
1540 (sock
->sk
->sk_state
!= SMC_APPCLOSEWAIT1
))
1543 smc
= smc_sk(sock
->sk
);
1545 return smc
->clcsock
->ops
->getname(smc
->clcsock
, addr
, peer
);
1548 static int smc_sendmsg(struct socket
*sock
, struct msghdr
*msg
, size_t len
)
1550 struct sock
*sk
= sock
->sk
;
1551 struct smc_sock
*smc
;
1556 if ((sk
->sk_state
!= SMC_ACTIVE
) &&
1557 (sk
->sk_state
!= SMC_APPCLOSEWAIT1
) &&
1558 (sk
->sk_state
!= SMC_INIT
))
1561 if (msg
->msg_flags
& MSG_FASTOPEN
) {
1562 if (sk
->sk_state
== SMC_INIT
&& !smc
->connect_nonblock
) {
1563 smc_switch_to_fallback(smc
);
1564 smc
->fallback_rsn
= SMC_CLC_DECL_OPTUNSUPP
;
1571 if (smc
->use_fallback
)
1572 rc
= smc
->clcsock
->ops
->sendmsg(smc
->clcsock
, msg
, len
);
1574 rc
= smc_tx_sendmsg(smc
, msg
, len
);
1580 static int smc_recvmsg(struct socket
*sock
, struct msghdr
*msg
, size_t len
,
1583 struct sock
*sk
= sock
->sk
;
1584 struct smc_sock
*smc
;
1589 if (sk
->sk_state
== SMC_CLOSED
&& (sk
->sk_shutdown
& RCV_SHUTDOWN
)) {
1590 /* socket was connected before, no more data to read */
1594 if ((sk
->sk_state
== SMC_INIT
) ||
1595 (sk
->sk_state
== SMC_LISTEN
) ||
1596 (sk
->sk_state
== SMC_CLOSED
))
1599 if (sk
->sk_state
== SMC_PEERFINCLOSEWAIT
) {
1604 if (smc
->use_fallback
) {
1605 rc
= smc
->clcsock
->ops
->recvmsg(smc
->clcsock
, msg
, len
, flags
);
1607 msg
->msg_namelen
= 0;
1608 rc
= smc_rx_recvmsg(smc
, msg
, NULL
, len
, flags
);
1616 static __poll_t
smc_accept_poll(struct sock
*parent
)
1618 struct smc_sock
*isk
= smc_sk(parent
);
1621 spin_lock(&isk
->accept_q_lock
);
1622 if (!list_empty(&isk
->accept_q
))
1623 mask
= EPOLLIN
| EPOLLRDNORM
;
1624 spin_unlock(&isk
->accept_q_lock
);
1629 static __poll_t
smc_poll(struct file
*file
, struct socket
*sock
,
1632 struct sock
*sk
= sock
->sk
;
1633 struct smc_sock
*smc
;
1639 smc
= smc_sk(sock
->sk
);
1640 if (smc
->use_fallback
) {
1641 /* delegate to CLC child sock */
1642 mask
= smc
->clcsock
->ops
->poll(file
, smc
->clcsock
, wait
);
1643 sk
->sk_err
= smc
->clcsock
->sk
->sk_err
;
1645 if (sk
->sk_state
!= SMC_CLOSED
)
1646 sock_poll_wait(file
, sock
, wait
);
1649 if ((sk
->sk_shutdown
== SHUTDOWN_MASK
) ||
1650 (sk
->sk_state
== SMC_CLOSED
))
1652 if (sk
->sk_state
== SMC_LISTEN
) {
1653 /* woken up by sk_data_ready in smc_listen_work() */
1654 mask
|= smc_accept_poll(sk
);
1655 } else if (smc
->use_fallback
) { /* as result of connect_work()*/
1656 mask
|= smc
->clcsock
->ops
->poll(file
, smc
->clcsock
,
1658 sk
->sk_err
= smc
->clcsock
->sk
->sk_err
;
1660 if ((sk
->sk_state
!= SMC_INIT
&&
1661 atomic_read(&smc
->conn
.sndbuf_space
)) ||
1662 sk
->sk_shutdown
& SEND_SHUTDOWN
) {
1663 mask
|= EPOLLOUT
| EPOLLWRNORM
;
1665 sk_set_bit(SOCKWQ_ASYNC_NOSPACE
, sk
);
1666 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
1668 if (atomic_read(&smc
->conn
.bytes_to_rcv
))
1669 mask
|= EPOLLIN
| EPOLLRDNORM
;
1670 if (sk
->sk_shutdown
& RCV_SHUTDOWN
)
1671 mask
|= EPOLLIN
| EPOLLRDNORM
| EPOLLRDHUP
;
1672 if (sk
->sk_state
== SMC_APPCLOSEWAIT1
)
1674 if (smc
->conn
.urg_state
== SMC_URG_VALID
)
1682 static int smc_shutdown(struct socket
*sock
, int how
)
1684 struct sock
*sk
= sock
->sk
;
1685 struct smc_sock
*smc
;
1691 if ((how
< SHUT_RD
) || (how
> SHUT_RDWR
))
1697 if ((sk
->sk_state
!= SMC_ACTIVE
) &&
1698 (sk
->sk_state
!= SMC_PEERCLOSEWAIT1
) &&
1699 (sk
->sk_state
!= SMC_PEERCLOSEWAIT2
) &&
1700 (sk
->sk_state
!= SMC_APPCLOSEWAIT1
) &&
1701 (sk
->sk_state
!= SMC_APPCLOSEWAIT2
) &&
1702 (sk
->sk_state
!= SMC_APPFINCLOSEWAIT
))
1704 if (smc
->use_fallback
) {
1705 rc
= kernel_sock_shutdown(smc
->clcsock
, how
);
1706 sk
->sk_shutdown
= smc
->clcsock
->sk
->sk_shutdown
;
1707 if (sk
->sk_shutdown
== SHUTDOWN_MASK
)
1708 sk
->sk_state
= SMC_CLOSED
;
1712 case SHUT_RDWR
: /* shutdown in both directions */
1713 rc
= smc_close_active(smc
);
1716 rc
= smc_close_shutdown_write(smc
);
1720 /* nothing more to do because peer is not involved */
1724 rc1
= kernel_sock_shutdown(smc
->clcsock
, how
);
1725 /* map sock_shutdown_cmd constants to sk_shutdown value range */
1726 sk
->sk_shutdown
|= how
+ 1;
1730 return rc
? rc
: rc1
;
1733 static int smc_setsockopt(struct socket
*sock
, int level
, int optname
,
1734 char __user
*optval
, unsigned int optlen
)
1736 struct sock
*sk
= sock
->sk
;
1737 struct smc_sock
*smc
;
1742 /* generic setsockopts reaching us here always apply to the
1745 if (unlikely(!smc
->clcsock
->ops
->setsockopt
))
1748 rc
= smc
->clcsock
->ops
->setsockopt(smc
->clcsock
, level
, optname
,
1750 if (smc
->clcsock
->sk
->sk_err
) {
1751 sk
->sk_err
= smc
->clcsock
->sk
->sk_err
;
1752 sk
->sk_error_report(sk
);
1755 if (optlen
< sizeof(int))
1757 if (get_user(val
, (int __user
*)optval
))
1761 if (rc
|| smc
->use_fallback
)
1766 case TCP_FASTOPEN_CONNECT
:
1767 case TCP_FASTOPEN_KEY
:
1768 case TCP_FASTOPEN_NO_COOKIE
:
1769 /* option not supported by SMC */
1770 if (sk
->sk_state
== SMC_INIT
&& !smc
->connect_nonblock
) {
1771 smc_switch_to_fallback(smc
);
1772 smc
->fallback_rsn
= SMC_CLC_DECL_OPTUNSUPP
;
1778 if (sk
->sk_state
!= SMC_INIT
&&
1779 sk
->sk_state
!= SMC_LISTEN
&&
1780 sk
->sk_state
!= SMC_CLOSED
) {
1782 mod_delayed_work(system_wq
, &smc
->conn
.tx_work
,
1787 if (sk
->sk_state
!= SMC_INIT
&&
1788 sk
->sk_state
!= SMC_LISTEN
&&
1789 sk
->sk_state
!= SMC_CLOSED
) {
1791 mod_delayed_work(system_wq
, &smc
->conn
.tx_work
,
1795 case TCP_DEFER_ACCEPT
:
1796 smc
->sockopt_defer_accept
= val
;
1807 static int smc_getsockopt(struct socket
*sock
, int level
, int optname
,
1808 char __user
*optval
, int __user
*optlen
)
1810 struct smc_sock
*smc
;
1812 smc
= smc_sk(sock
->sk
);
1813 /* socket options apply to the CLC socket */
1814 if (unlikely(!smc
->clcsock
->ops
->getsockopt
))
1816 return smc
->clcsock
->ops
->getsockopt(smc
->clcsock
, level
, optname
,
1820 static int smc_ioctl(struct socket
*sock
, unsigned int cmd
,
1823 union smc_host_cursor cons
, urg
;
1824 struct smc_connection
*conn
;
1825 struct smc_sock
*smc
;
1828 smc
= smc_sk(sock
->sk
);
1830 lock_sock(&smc
->sk
);
1831 if (smc
->use_fallback
) {
1832 if (!smc
->clcsock
) {
1833 release_sock(&smc
->sk
);
1836 answ
= smc
->clcsock
->ops
->ioctl(smc
->clcsock
, cmd
, arg
);
1837 release_sock(&smc
->sk
);
1841 case SIOCINQ
: /* same as FIONREAD */
1842 if (smc
->sk
.sk_state
== SMC_LISTEN
) {
1843 release_sock(&smc
->sk
);
1846 if (smc
->sk
.sk_state
== SMC_INIT
||
1847 smc
->sk
.sk_state
== SMC_CLOSED
)
1850 answ
= atomic_read(&smc
->conn
.bytes_to_rcv
);
1853 /* output queue size (not send + not acked) */
1854 if (smc
->sk
.sk_state
== SMC_LISTEN
) {
1855 release_sock(&smc
->sk
);
1858 if (smc
->sk
.sk_state
== SMC_INIT
||
1859 smc
->sk
.sk_state
== SMC_CLOSED
)
1862 answ
= smc
->conn
.sndbuf_desc
->len
-
1863 atomic_read(&smc
->conn
.sndbuf_space
);
1866 /* output queue size (not send only) */
1867 if (smc
->sk
.sk_state
== SMC_LISTEN
) {
1868 release_sock(&smc
->sk
);
1871 if (smc
->sk
.sk_state
== SMC_INIT
||
1872 smc
->sk
.sk_state
== SMC_CLOSED
)
1875 answ
= smc_tx_prepared_sends(&smc
->conn
);
1878 if (smc
->sk
.sk_state
== SMC_LISTEN
) {
1879 release_sock(&smc
->sk
);
1882 if (smc
->sk
.sk_state
== SMC_INIT
||
1883 smc
->sk
.sk_state
== SMC_CLOSED
) {
1886 smc_curs_copy(&cons
, &conn
->local_tx_ctrl
.cons
, conn
);
1887 smc_curs_copy(&urg
, &conn
->urg_curs
, conn
);
1888 answ
= smc_curs_diff(conn
->rmb_desc
->len
,
1893 release_sock(&smc
->sk
);
1894 return -ENOIOCTLCMD
;
1896 release_sock(&smc
->sk
);
1898 return put_user(answ
, (int __user
*)arg
);
1901 static ssize_t
smc_sendpage(struct socket
*sock
, struct page
*page
,
1902 int offset
, size_t size
, int flags
)
1904 struct sock
*sk
= sock
->sk
;
1905 struct smc_sock
*smc
;
1910 if (sk
->sk_state
!= SMC_ACTIVE
) {
1915 if (smc
->use_fallback
)
1916 rc
= kernel_sendpage(smc
->clcsock
, page
, offset
,
1919 rc
= sock_no_sendpage(sock
, page
, offset
, size
, flags
);
1925 /* Map the affected portions of the rmbe into an spd, note the number of bytes
1926 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1927 * updates till whenever a respective page has been fully processed.
1928 * Note that subsequent recv() calls have to wait till all splice() processing
1931 static ssize_t
smc_splice_read(struct socket
*sock
, loff_t
*ppos
,
1932 struct pipe_inode_info
*pipe
, size_t len
,
1935 struct sock
*sk
= sock
->sk
;
1936 struct smc_sock
*smc
;
1941 if (sk
->sk_state
== SMC_CLOSED
&& (sk
->sk_shutdown
& RCV_SHUTDOWN
)) {
1942 /* socket was connected before, no more data to read */
1946 if (sk
->sk_state
== SMC_INIT
||
1947 sk
->sk_state
== SMC_LISTEN
||
1948 sk
->sk_state
== SMC_CLOSED
)
1951 if (sk
->sk_state
== SMC_PEERFINCLOSEWAIT
) {
1956 if (smc
->use_fallback
) {
1957 rc
= smc
->clcsock
->ops
->splice_read(smc
->clcsock
, ppos
,
1964 if (flags
& SPLICE_F_NONBLOCK
)
1965 flags
= MSG_DONTWAIT
;
1968 rc
= smc_rx_recvmsg(smc
, NULL
, pipe
, len
, flags
);
1976 /* must look like tcp */
1977 static const struct proto_ops smc_sock_ops
= {
1979 .owner
= THIS_MODULE
,
1980 .release
= smc_release
,
1982 .connect
= smc_connect
,
1983 .socketpair
= sock_no_socketpair
,
1984 .accept
= smc_accept
,
1985 .getname
= smc_getname
,
1988 .listen
= smc_listen
,
1989 .shutdown
= smc_shutdown
,
1990 .setsockopt
= smc_setsockopt
,
1991 .getsockopt
= smc_getsockopt
,
1992 .sendmsg
= smc_sendmsg
,
1993 .recvmsg
= smc_recvmsg
,
1994 .mmap
= sock_no_mmap
,
1995 .sendpage
= smc_sendpage
,
1996 .splice_read
= smc_splice_read
,
1999 static int smc_create(struct net
*net
, struct socket
*sock
, int protocol
,
2002 int family
= (protocol
== SMCPROTO_SMC6
) ? PF_INET6
: PF_INET
;
2003 struct smc_sock
*smc
;
2007 rc
= -ESOCKTNOSUPPORT
;
2008 if (sock
->type
!= SOCK_STREAM
)
2011 rc
= -EPROTONOSUPPORT
;
2012 if (protocol
!= SMCPROTO_SMC
&& protocol
!= SMCPROTO_SMC6
)
2016 sock
->ops
= &smc_sock_ops
;
2017 sk
= smc_sock_alloc(net
, sock
, protocol
);
2021 /* create internal TCP socket for CLC handshake and fallback */
2023 smc
->use_fallback
= false; /* assume rdma capability first */
2024 smc
->fallback_rsn
= 0;
2025 rc
= sock_create_kern(net
, family
, SOCK_STREAM
, IPPROTO_TCP
,
2028 sk_common_release(sk
);
2031 smc
->sk
.sk_sndbuf
= max(smc
->clcsock
->sk
->sk_sndbuf
, SMC_BUF_MIN_SIZE
);
2032 smc
->sk
.sk_rcvbuf
= max(smc
->clcsock
->sk
->sk_rcvbuf
, SMC_BUF_MIN_SIZE
);
2038 static const struct net_proto_family smc_sock_family_ops
= {
2040 .owner
= THIS_MODULE
,
2041 .create
= smc_create
,
2044 unsigned int smc_net_id
;
2046 static __net_init
int smc_net_init(struct net
*net
)
2048 return smc_pnet_net_init(net
);
2051 static void __net_exit
smc_net_exit(struct net
*net
)
2053 smc_pnet_net_exit(net
);
2056 static struct pernet_operations smc_net_ops
= {
2057 .init
= smc_net_init
,
2058 .exit
= smc_net_exit
,
2060 .size
= sizeof(struct smc_net
),
2063 static int __init
smc_init(void)
2067 rc
= register_pernet_subsys(&smc_net_ops
);
2071 rc
= smc_pnet_init();
2073 goto out_pernet_subsys
;
2075 rc
= smc_core_init();
2077 pr_err("%s: smc_core_init fails with %d\n", __func__
, rc
);
2081 rc
= smc_llc_init();
2083 pr_err("%s: smc_llc_init fails with %d\n", __func__
, rc
);
2087 rc
= smc_cdc_init();
2089 pr_err("%s: smc_cdc_init fails with %d\n", __func__
, rc
);
2093 rc
= proto_register(&smc_proto
, 1);
2095 pr_err("%s: proto_register(v4) fails with %d\n", __func__
, rc
);
2099 rc
= proto_register(&smc_proto6
, 1);
2101 pr_err("%s: proto_register(v6) fails with %d\n", __func__
, rc
);
2105 rc
= sock_register(&smc_sock_family_ops
);
2107 pr_err("%s: sock_register fails with %d\n", __func__
, rc
);
2110 INIT_HLIST_HEAD(&smc_v4_hashinfo
.ht
);
2111 INIT_HLIST_HEAD(&smc_v6_hashinfo
.ht
);
2113 rc
= smc_ib_register_client();
2115 pr_err("%s: ib_register fails with %d\n", __func__
, rc
);
2119 static_branch_enable(&tcp_have_smc
);
2123 sock_unregister(PF_SMC
);
2125 proto_unregister(&smc_proto6
);
2127 proto_unregister(&smc_proto
);
2133 unregister_pernet_subsys(&smc_net_ops
);
2138 static void __exit
smc_exit(void)
2140 static_branch_disable(&tcp_have_smc
);
2141 sock_unregister(PF_SMC
);
2143 smc_ib_unregister_client();
2144 proto_unregister(&smc_proto6
);
2145 proto_unregister(&smc_proto
);
2147 unregister_pernet_subsys(&smc_net_ops
);
2151 module_init(smc_init
);
2152 module_exit(smc_exit
);
2154 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2155 MODULE_DESCRIPTION("smc socket address family");
2156 MODULE_LICENSE("GPL");
2157 MODULE_ALIAS_NETPROTO(PF_SMC
);