2 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
5 * applies to SOCK_STREAM sockets only
6 * offers an alternative communication option for TCP-protocol sockets
7 * applicable with RoCE-cards only
9 * Initial restrictions:
10 * - support for alternate links postponed
11 * - partial support for non-blocking sockets only
12 * - support for urgent data postponed
14 * Copyright IBM Corp. 2016, 2018
16 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
17 * based on prototype from Frank Blaschka
20 #define KMSG_COMPONENT "smc"
21 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
23 #include <linux/module.h>
24 #include <linux/socket.h>
25 #include <linux/workqueue.h>
27 #include <linux/sched/signal.h>
42 #include "smc_close.h"
44 static DEFINE_MUTEX(smc_create_lgr_pending
); /* serialize link group
48 struct smc_lgr_list smc_lgr_list
= { /* established link groups */
49 .lock
= __SPIN_LOCK_UNLOCKED(smc_lgr_list
.lock
),
50 .list
= LIST_HEAD_INIT(smc_lgr_list
.list
),
53 static void smc_tcp_listen_work(struct work_struct
*);
55 static void smc_set_keepalive(struct sock
*sk
, int val
)
57 struct smc_sock
*smc
= smc_sk(sk
);
59 smc
->clcsock
->sk
->sk_prot
->keepalive(smc
->clcsock
->sk
, val
);
62 static struct smc_hashinfo smc_v4_hashinfo
= {
63 .lock
= __RW_LOCK_UNLOCKED(smc_v4_hashinfo
.lock
),
66 static struct smc_hashinfo smc_v6_hashinfo
= {
67 .lock
= __RW_LOCK_UNLOCKED(smc_v6_hashinfo
.lock
),
70 int smc_hash_sk(struct sock
*sk
)
72 struct smc_hashinfo
*h
= sk
->sk_prot
->h
.smc_hash
;
73 struct hlist_head
*head
;
77 write_lock_bh(&h
->lock
);
78 sk_add_node(sk
, head
);
79 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, 1);
80 write_unlock_bh(&h
->lock
);
84 EXPORT_SYMBOL_GPL(smc_hash_sk
);
86 void smc_unhash_sk(struct sock
*sk
)
88 struct smc_hashinfo
*h
= sk
->sk_prot
->h
.smc_hash
;
90 write_lock_bh(&h
->lock
);
91 if (sk_del_node_init(sk
))
92 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, -1);
93 write_unlock_bh(&h
->lock
);
95 EXPORT_SYMBOL_GPL(smc_unhash_sk
);
97 struct proto smc_proto
= {
100 .keepalive
= smc_set_keepalive
,
102 .unhash
= smc_unhash_sk
,
103 .obj_size
= sizeof(struct smc_sock
),
104 .h
.smc_hash
= &smc_v4_hashinfo
,
105 .slab_flags
= SLAB_TYPESAFE_BY_RCU
,
107 EXPORT_SYMBOL_GPL(smc_proto
);
109 struct proto smc_proto6
= {
111 .owner
= THIS_MODULE
,
112 .keepalive
= smc_set_keepalive
,
114 .unhash
= smc_unhash_sk
,
115 .obj_size
= sizeof(struct smc_sock
),
116 .h
.smc_hash
= &smc_v6_hashinfo
,
117 .slab_flags
= SLAB_TYPESAFE_BY_RCU
,
119 EXPORT_SYMBOL_GPL(smc_proto6
);
121 static int smc_release(struct socket
*sock
)
123 struct sock
*sk
= sock
->sk
;
124 struct smc_sock
*smc
;
131 if (sk
->sk_state
== SMC_LISTEN
)
132 /* smc_close_non_accepted() is called and acquires
133 * sock lock for child sockets again
135 lock_sock_nested(sk
, SINGLE_DEPTH_NESTING
);
139 if (!smc
->use_fallback
) {
140 rc
= smc_close_active(smc
);
141 sock_set_flag(sk
, SOCK_DEAD
);
142 sk
->sk_shutdown
|= SHUTDOWN_MASK
;
145 sock_release(smc
->clcsock
);
148 if (smc
->use_fallback
) {
149 sock_put(sk
); /* passive closing */
150 sk
->sk_state
= SMC_CLOSED
;
151 sk
->sk_state_change(sk
);
157 if (!smc
->use_fallback
&& sk
->sk_state
== SMC_CLOSED
)
158 smc_conn_free(&smc
->conn
);
161 sk
->sk_prot
->unhash(sk
);
162 sock_put(sk
); /* final sock_put */
167 static void smc_destruct(struct sock
*sk
)
169 if (sk
->sk_state
!= SMC_CLOSED
)
171 if (!sock_flag(sk
, SOCK_DEAD
))
174 sk_refcnt_debug_dec(sk
);
177 static struct sock
*smc_sock_alloc(struct net
*net
, struct socket
*sock
,
180 struct smc_sock
*smc
;
184 prot
= (protocol
== SMCPROTO_SMC6
) ? &smc_proto6
: &smc_proto
;
185 sk
= sk_alloc(net
, PF_SMC
, GFP_KERNEL
, prot
, 0);
189 sock_init_data(sock
, sk
); /* sets sk_refcnt to 1 */
190 sk
->sk_state
= SMC_INIT
;
191 sk
->sk_destruct
= smc_destruct
;
192 sk
->sk_protocol
= protocol
;
194 INIT_WORK(&smc
->tcp_listen_work
, smc_tcp_listen_work
);
195 INIT_LIST_HEAD(&smc
->accept_q
);
196 spin_lock_init(&smc
->accept_q_lock
);
197 sk
->sk_prot
->hash(sk
);
198 sk_refcnt_debug_inc(sk
);
203 static int smc_bind(struct socket
*sock
, struct sockaddr
*uaddr
,
206 struct sockaddr_in
*addr
= (struct sockaddr_in
*)uaddr
;
207 struct sock
*sk
= sock
->sk
;
208 struct smc_sock
*smc
;
213 /* replicate tests from inet_bind(), to be safe wrt. future changes */
215 if (addr_len
< sizeof(struct sockaddr_in
))
219 if (addr
->sin_family
!= AF_INET
&&
220 addr
->sin_family
!= AF_INET6
&&
221 addr
->sin_family
!= AF_UNSPEC
)
223 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
224 if (addr
->sin_family
== AF_UNSPEC
&&
225 addr
->sin_addr
.s_addr
!= htonl(INADDR_ANY
))
230 /* Check if socket is already active */
232 if (sk
->sk_state
!= SMC_INIT
)
235 smc
->clcsock
->sk
->sk_reuse
= sk
->sk_reuse
;
236 rc
= kernel_bind(smc
->clcsock
, uaddr
, addr_len
);
244 static void smc_copy_sock_settings(struct sock
*nsk
, struct sock
*osk
,
247 /* options we don't get control via setsockopt for */
248 nsk
->sk_type
= osk
->sk_type
;
249 nsk
->sk_sndbuf
= osk
->sk_sndbuf
;
250 nsk
->sk_rcvbuf
= osk
->sk_rcvbuf
;
251 nsk
->sk_sndtimeo
= osk
->sk_sndtimeo
;
252 nsk
->sk_rcvtimeo
= osk
->sk_rcvtimeo
;
253 nsk
->sk_mark
= osk
->sk_mark
;
254 nsk
->sk_priority
= osk
->sk_priority
;
255 nsk
->sk_rcvlowat
= osk
->sk_rcvlowat
;
256 nsk
->sk_bound_dev_if
= osk
->sk_bound_dev_if
;
257 nsk
->sk_err
= osk
->sk_err
;
259 nsk
->sk_flags
&= ~mask
;
260 nsk
->sk_flags
|= osk
->sk_flags
& mask
;
263 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
264 (1UL << SOCK_KEEPOPEN) | \
265 (1UL << SOCK_LINGER) | \
266 (1UL << SOCK_BROADCAST) | \
267 (1UL << SOCK_TIMESTAMP) | \
268 (1UL << SOCK_DBG) | \
269 (1UL << SOCK_RCVTSTAMP) | \
270 (1UL << SOCK_RCVTSTAMPNS) | \
271 (1UL << SOCK_LOCALROUTE) | \
272 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
273 (1UL << SOCK_RXQ_OVFL) | \
274 (1UL << SOCK_WIFI_STATUS) | \
275 (1UL << SOCK_NOFCS) | \
276 (1UL << SOCK_FILTER_LOCKED))
277 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
278 * clc socket (since smc is not called for these options from net/core)
280 static void smc_copy_sock_settings_to_clc(struct smc_sock
*smc
)
282 smc_copy_sock_settings(smc
->clcsock
->sk
, &smc
->sk
, SK_FLAGS_SMC_TO_CLC
);
285 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
286 (1UL << SOCK_KEEPOPEN) | \
287 (1UL << SOCK_LINGER) | \
289 /* copy only settings and flags relevant for smc from clc to smc socket */
290 static void smc_copy_sock_settings_to_smc(struct smc_sock
*smc
)
292 smc_copy_sock_settings(&smc
->sk
, smc
->clcsock
->sk
, SK_FLAGS_CLC_TO_SMC
);
295 static int smc_clnt_conf_first_link(struct smc_sock
*smc
)
297 struct smc_link_group
*lgr
= smc
->conn
.lgr
;
298 struct smc_link
*link
;
302 link
= &lgr
->lnk
[SMC_SINGLE_LINK
];
303 /* receive CONFIRM LINK request from server over RoCE fabric */
304 rest
= wait_for_completion_interruptible_timeout(
306 SMC_LLC_WAIT_FIRST_TIME
);
308 struct smc_clc_msg_decline dclc
;
310 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
315 if (link
->llc_confirm_rc
)
316 return SMC_CLC_DECL_RMBE_EC
;
318 rc
= smc_ib_modify_qp_rts(link
);
320 return SMC_CLC_DECL_INTERR
;
322 smc_wr_remember_qp_attr(link
);
324 rc
= smc_wr_reg_send(link
,
325 smc
->conn
.rmb_desc
->mr_rx
[SMC_SINGLE_LINK
]);
327 return SMC_CLC_DECL_INTERR
;
329 /* send CONFIRM LINK response over RoCE fabric */
330 rc
= smc_llc_send_confirm_link(link
,
331 link
->smcibdev
->mac
[link
->ibport
- 1],
332 &link
->smcibdev
->gid
[link
->ibport
- 1],
335 return SMC_CLC_DECL_TCL
;
337 /* receive ADD LINK request from server over RoCE fabric */
338 rest
= wait_for_completion_interruptible_timeout(&link
->llc_add
,
341 struct smc_clc_msg_decline dclc
;
343 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
348 /* send add link reject message, only one link supported for now */
349 rc
= smc_llc_send_add_link(link
,
350 link
->smcibdev
->mac
[link
->ibport
- 1],
351 &link
->smcibdev
->gid
[link
->ibport
- 1],
354 return SMC_CLC_DECL_TCL
;
356 link
->state
= SMC_LNK_ACTIVE
;
361 static void smc_conn_save_peer_info(struct smc_sock
*smc
,
362 struct smc_clc_msg_accept_confirm
*clc
)
364 smc
->conn
.peer_conn_idx
= clc
->conn_idx
;
365 smc
->conn
.local_tx_ctrl
.token
= ntohl(clc
->rmbe_alert_token
);
366 smc
->conn
.peer_rmbe_size
= smc_uncompress_bufsize(clc
->rmbe_size
);
367 atomic_set(&smc
->conn
.peer_rmbe_space
, smc
->conn
.peer_rmbe_size
);
370 static void smc_link_save_peer_info(struct smc_link
*link
,
371 struct smc_clc_msg_accept_confirm
*clc
)
373 link
->peer_qpn
= ntoh24(clc
->qpn
);
374 memcpy(link
->peer_gid
, clc
->lcl
.gid
, SMC_GID_SIZE
);
375 memcpy(link
->peer_mac
, clc
->lcl
.mac
, sizeof(link
->peer_mac
));
376 link
->peer_psn
= ntoh24(clc
->psn
);
377 link
->peer_mtu
= clc
->qp_mtu
;
380 /* setup for RDMA connection of client */
381 static int smc_connect_rdma(struct smc_sock
*smc
)
383 struct smc_clc_msg_accept_confirm aclc
;
384 int local_contact
= SMC_FIRST_CONTACT
;
385 struct smc_ib_device
*smcibdev
;
386 struct smc_link
*link
;
387 u8 srv_first_contact
;
392 sock_hold(&smc
->sk
); /* sock put in passive closing */
394 if (!tcp_sk(smc
->clcsock
->sk
)->syn_smc
) {
395 /* peer has not signalled SMC-capability */
396 smc
->use_fallback
= true;
400 /* IPSec connections opt out of SMC-R optimizations */
401 if (using_ipsec(smc
)) {
402 reason_code
= SMC_CLC_DECL_IPSEC
;
406 /* PNET table look up: search active ib_device and port
407 * within same PNETID that also contains the ethernet device
408 * used for the internal TCP socket
410 smc_pnet_find_roce_resource(smc
->clcsock
->sk
, &smcibdev
, &ibport
);
412 reason_code
= SMC_CLC_DECL_CNFERR
; /* configuration error */
416 /* do inband token exchange */
417 reason_code
= smc_clc_send_proposal(smc
, smcibdev
, ibport
);
418 if (reason_code
< 0) {
422 if (reason_code
> 0) /* configuration error */
424 /* receive SMC Accept CLC message */
425 reason_code
= smc_clc_wait_msg(smc
, &aclc
, sizeof(aclc
),
427 if (reason_code
< 0) {
434 srv_first_contact
= aclc
.hdr
.flag
;
435 mutex_lock(&smc_create_lgr_pending
);
436 local_contact
= smc_conn_create(smc
, smcibdev
, ibport
, &aclc
.lcl
,
438 if (local_contact
< 0) {
441 reason_code
= SMC_CLC_DECL_MEM
;/* insufficient memory*/
442 else if (rc
== -ENOLINK
)
443 reason_code
= SMC_CLC_DECL_SYNCERR
; /* synchr. error */
444 goto decline_rdma_unlock
;
446 link
= &smc
->conn
.lgr
->lnk
[SMC_SINGLE_LINK
];
448 smc_conn_save_peer_info(smc
, &aclc
);
450 /* create send buffer and rmb */
451 rc
= smc_buf_create(smc
);
453 reason_code
= SMC_CLC_DECL_MEM
;
454 goto decline_rdma_unlock
;
457 if (local_contact
== SMC_FIRST_CONTACT
)
458 smc_link_save_peer_info(link
, &aclc
);
460 rc
= smc_rmb_rtoken_handling(&smc
->conn
, &aclc
);
462 reason_code
= SMC_CLC_DECL_INTERR
;
463 goto decline_rdma_unlock
;
469 if (local_contact
== SMC_FIRST_CONTACT
) {
470 rc
= smc_ib_ready_link(link
);
472 reason_code
= SMC_CLC_DECL_INTERR
;
473 goto decline_rdma_unlock
;
476 struct smc_buf_desc
*buf_desc
= smc
->conn
.rmb_desc
;
478 if (!buf_desc
->reused
) {
479 /* register memory region for new rmb */
480 rc
= smc_wr_reg_send(link
,
481 buf_desc
->mr_rx
[SMC_SINGLE_LINK
]);
483 reason_code
= SMC_CLC_DECL_INTERR
;
484 goto decline_rdma_unlock
;
488 smc_rmb_sync_sg_for_device(&smc
->conn
);
490 rc
= smc_clc_send_confirm(smc
);
494 if (local_contact
== SMC_FIRST_CONTACT
) {
495 /* QP confirmation over RoCE fabric */
496 reason_code
= smc_clnt_conf_first_link(smc
);
497 if (reason_code
< 0) {
502 goto decline_rdma_unlock
;
505 mutex_unlock(&smc_create_lgr_pending
);
509 smc_copy_sock_settings_to_clc(smc
);
510 if (smc
->sk
.sk_state
== SMC_INIT
)
511 smc
->sk
.sk_state
= SMC_ACTIVE
;
513 return rc
? rc
: local_contact
;
516 if (local_contact
== SMC_FIRST_CONTACT
)
517 smc_lgr_forget(smc
->conn
.lgr
);
518 mutex_unlock(&smc_create_lgr_pending
);
519 smc_conn_free(&smc
->conn
);
521 /* RDMA setup failed, switch back to TCP */
522 smc
->use_fallback
= true;
523 if (reason_code
&& (reason_code
!= SMC_CLC_DECL_REPLY
)) {
524 rc
= smc_clc_send_decline(smc
, reason_code
);
531 if (local_contact
== SMC_FIRST_CONTACT
)
532 smc_lgr_forget(smc
->conn
.lgr
);
533 mutex_unlock(&smc_create_lgr_pending
);
534 smc_conn_free(&smc
->conn
);
536 if (smc
->sk
.sk_state
== SMC_INIT
)
537 sock_put(&smc
->sk
); /* passive closing */
541 static int smc_connect(struct socket
*sock
, struct sockaddr
*addr
,
544 struct sock
*sk
= sock
->sk
;
545 struct smc_sock
*smc
;
550 /* separate smc parameter checking to be safe */
551 if (alen
< sizeof(addr
->sa_family
))
553 if (addr
->sa_family
!= AF_INET
&& addr
->sa_family
!= AF_INET6
)
557 switch (sk
->sk_state
) {
568 smc_copy_sock_settings_to_clc(smc
);
569 tcp_sk(smc
->clcsock
->sk
)->syn_smc
= 1;
570 rc
= kernel_connect(smc
->clcsock
, addr
, alen
, flags
);
574 /* setup RDMA connection */
575 rc
= smc_connect_rdma(smc
);
579 rc
= 0; /* success cases including fallback */
587 static int smc_clcsock_accept(struct smc_sock
*lsmc
, struct smc_sock
**new_smc
)
589 struct socket
*new_clcsock
= NULL
;
590 struct sock
*lsk
= &lsmc
->sk
;
595 new_sk
= smc_sock_alloc(sock_net(lsk
), NULL
, lsk
->sk_protocol
);
598 lsk
->sk_err
= ENOMEM
;
603 *new_smc
= smc_sk(new_sk
);
605 rc
= kernel_accept(lsmc
->clcsock
, &new_clcsock
, 0);
609 if (rc
< 0 || lsk
->sk_state
== SMC_CLOSED
) {
611 sock_release(new_clcsock
);
612 new_sk
->sk_state
= SMC_CLOSED
;
613 sock_set_flag(new_sk
, SOCK_DEAD
);
614 new_sk
->sk_prot
->unhash(new_sk
);
615 sock_put(new_sk
); /* final */
620 (*new_smc
)->clcsock
= new_clcsock
;
625 /* add a just created sock to the accept queue of the listen sock as
626 * candidate for a following socket accept call from user space
628 static void smc_accept_enqueue(struct sock
*parent
, struct sock
*sk
)
630 struct smc_sock
*par
= smc_sk(parent
);
632 sock_hold(sk
); /* sock_put in smc_accept_unlink () */
633 spin_lock(&par
->accept_q_lock
);
634 list_add_tail(&smc_sk(sk
)->accept_q
, &par
->accept_q
);
635 spin_unlock(&par
->accept_q_lock
);
636 sk_acceptq_added(parent
);
639 /* remove a socket from the accept queue of its parental listening socket */
640 static void smc_accept_unlink(struct sock
*sk
)
642 struct smc_sock
*par
= smc_sk(sk
)->listen_smc
;
644 spin_lock(&par
->accept_q_lock
);
645 list_del_init(&smc_sk(sk
)->accept_q
);
646 spin_unlock(&par
->accept_q_lock
);
647 sk_acceptq_removed(&smc_sk(sk
)->listen_smc
->sk
);
648 sock_put(sk
); /* sock_hold in smc_accept_enqueue */
651 /* remove a sock from the accept queue to bind it to a new socket created
652 * for a socket accept call from user space
654 struct sock
*smc_accept_dequeue(struct sock
*parent
,
655 struct socket
*new_sock
)
657 struct smc_sock
*isk
, *n
;
660 list_for_each_entry_safe(isk
, n
, &smc_sk(parent
)->accept_q
, accept_q
) {
661 new_sk
= (struct sock
*)isk
;
663 smc_accept_unlink(new_sk
);
664 if (new_sk
->sk_state
== SMC_CLOSED
) {
666 sock_release(isk
->clcsock
);
669 new_sk
->sk_prot
->unhash(new_sk
);
670 sock_put(new_sk
); /* final */
674 sock_graft(new_sk
, new_sock
);
680 /* clean up for a created but never accepted sock */
681 void smc_close_non_accepted(struct sock
*sk
)
683 struct smc_sock
*smc
= smc_sk(sk
);
686 if (!sk
->sk_lingertime
)
687 /* wait for peer closing */
688 sk
->sk_lingertime
= SMC_MAX_STREAM_WAIT_TIMEOUT
;
689 if (!smc
->use_fallback
) {
690 smc_close_active(smc
);
691 sock_set_flag(sk
, SOCK_DEAD
);
692 sk
->sk_shutdown
|= SHUTDOWN_MASK
;
701 if (smc
->use_fallback
) {
702 sock_put(sk
); /* passive closing */
703 sk
->sk_state
= SMC_CLOSED
;
705 if (sk
->sk_state
== SMC_CLOSED
)
706 smc_conn_free(&smc
->conn
);
709 sk
->sk_prot
->unhash(sk
);
710 sock_put(sk
); /* final sock_put */
713 static int smc_serv_conf_first_link(struct smc_sock
*smc
)
715 struct smc_link_group
*lgr
= smc
->conn
.lgr
;
716 struct smc_link
*link
;
720 link
= &lgr
->lnk
[SMC_SINGLE_LINK
];
722 rc
= smc_wr_reg_send(link
,
723 smc
->conn
.rmb_desc
->mr_rx
[SMC_SINGLE_LINK
]);
725 return SMC_CLC_DECL_INTERR
;
727 /* send CONFIRM LINK request to client over the RoCE fabric */
728 rc
= smc_llc_send_confirm_link(link
,
729 link
->smcibdev
->mac
[link
->ibport
- 1],
730 &link
->smcibdev
->gid
[link
->ibport
- 1],
733 return SMC_CLC_DECL_TCL
;
735 /* receive CONFIRM LINK response from client over the RoCE fabric */
736 rest
= wait_for_completion_interruptible_timeout(
737 &link
->llc_confirm_resp
,
738 SMC_LLC_WAIT_FIRST_TIME
);
740 struct smc_clc_msg_decline dclc
;
742 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
747 if (link
->llc_confirm_resp_rc
)
748 return SMC_CLC_DECL_RMBE_EC
;
750 /* send ADD LINK request to client over the RoCE fabric */
751 rc
= smc_llc_send_add_link(link
,
752 link
->smcibdev
->mac
[link
->ibport
- 1],
753 &link
->smcibdev
->gid
[link
->ibport
- 1],
756 return SMC_CLC_DECL_TCL
;
758 /* receive ADD LINK response from client over the RoCE fabric */
759 rest
= wait_for_completion_interruptible_timeout(&link
->llc_add_resp
,
762 struct smc_clc_msg_decline dclc
;
764 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
769 link
->state
= SMC_LNK_ACTIVE
;
774 /* setup for RDMA connection of server */
775 static void smc_listen_work(struct work_struct
*work
)
777 struct smc_sock
*new_smc
= container_of(work
, struct smc_sock
,
779 struct smc_clc_msg_proposal_prefix
*pclc_prfx
;
780 struct socket
*newclcsock
= new_smc
->clcsock
;
781 struct smc_sock
*lsmc
= new_smc
->listen_smc
;
782 struct smc_clc_msg_accept_confirm cclc
;
783 int local_contact
= SMC_REUSE_CONTACT
;
784 struct sock
*newsmcsk
= &new_smc
->sk
;
785 struct smc_clc_msg_proposal
*pclc
;
786 struct smc_ib_device
*smcibdev
;
787 u8 buf
[SMC_CLC_MAX_LEN
];
788 struct smc_link
*link
;
793 /* check if peer is smc capable */
794 if (!tcp_sk(newclcsock
->sk
)->syn_smc
) {
795 new_smc
->use_fallback
= true;
799 /* do inband token exchange -
800 *wait for and receive SMC Proposal CLC message
802 reason_code
= smc_clc_wait_msg(new_smc
, &buf
, sizeof(buf
),
809 /* IPSec connections opt out of SMC-R optimizations */
810 if (using_ipsec(new_smc
)) {
811 reason_code
= SMC_CLC_DECL_IPSEC
;
815 /* PNET table look up: search active ib_device and port
816 * within same PNETID that also contains the ethernet device
817 * used for the internal TCP socket
819 smc_pnet_find_roce_resource(newclcsock
->sk
, &smcibdev
, &ibport
);
821 reason_code
= SMC_CLC_DECL_CNFERR
; /* configuration error */
825 pclc
= (struct smc_clc_msg_proposal
*)&buf
;
826 pclc_prfx
= smc_clc_proposal_get_prefix(pclc
);
828 rc
= smc_clc_prfx_match(newclcsock
, pclc_prfx
);
830 reason_code
= SMC_CLC_DECL_CNFERR
; /* configuration error */
834 /* allocate connection / link group */
835 mutex_lock(&smc_create_lgr_pending
);
836 local_contact
= smc_conn_create(new_smc
, smcibdev
, ibport
, &pclc
->lcl
,
838 if (local_contact
< 0) {
841 reason_code
= SMC_CLC_DECL_MEM
;/* insufficient memory*/
842 goto decline_rdma_unlock
;
844 link
= &new_smc
->conn
.lgr
->lnk
[SMC_SINGLE_LINK
];
846 /* create send buffer and rmb */
847 rc
= smc_buf_create(new_smc
);
849 reason_code
= SMC_CLC_DECL_MEM
;
850 goto decline_rdma_unlock
;
853 smc_close_init(new_smc
);
854 smc_rx_init(new_smc
);
856 if (local_contact
!= SMC_FIRST_CONTACT
) {
857 struct smc_buf_desc
*buf_desc
= new_smc
->conn
.rmb_desc
;
859 if (!buf_desc
->reused
) {
860 /* register memory region for new rmb */
861 rc
= smc_wr_reg_send(link
,
862 buf_desc
->mr_rx
[SMC_SINGLE_LINK
]);
864 reason_code
= SMC_CLC_DECL_INTERR
;
865 goto decline_rdma_unlock
;
869 smc_rmb_sync_sg_for_device(&new_smc
->conn
);
871 rc
= smc_clc_send_accept(new_smc
, local_contact
);
875 /* receive SMC Confirm CLC message */
876 reason_code
= smc_clc_wait_msg(new_smc
, &cclc
, sizeof(cclc
),
881 goto decline_rdma_unlock
;
882 smc_conn_save_peer_info(new_smc
, &cclc
);
883 if (local_contact
== SMC_FIRST_CONTACT
)
884 smc_link_save_peer_info(link
, &cclc
);
886 rc
= smc_rmb_rtoken_handling(&new_smc
->conn
, &cclc
);
888 reason_code
= SMC_CLC_DECL_INTERR
;
889 goto decline_rdma_unlock
;
892 if (local_contact
== SMC_FIRST_CONTACT
) {
893 rc
= smc_ib_ready_link(link
);
895 reason_code
= SMC_CLC_DECL_INTERR
;
896 goto decline_rdma_unlock
;
898 /* QP confirmation over RoCE fabric */
899 reason_code
= smc_serv_conf_first_link(new_smc
);
901 /* peer is not aware of a problem */
904 goto decline_rdma_unlock
;
907 smc_tx_init(new_smc
);
908 mutex_unlock(&smc_create_lgr_pending
);
911 sk_refcnt_debug_inc(newsmcsk
);
912 if (newsmcsk
->sk_state
== SMC_INIT
)
913 newsmcsk
->sk_state
= SMC_ACTIVE
;
915 lock_sock_nested(&lsmc
->sk
, SINGLE_DEPTH_NESTING
);
916 if (lsmc
->sk
.sk_state
== SMC_LISTEN
) {
917 smc_accept_enqueue(&lsmc
->sk
, newsmcsk
);
918 } else { /* no longer listening */
919 smc_close_non_accepted(newsmcsk
);
921 release_sock(&lsmc
->sk
);
924 lsmc
->sk
.sk_data_ready(&lsmc
->sk
);
925 sock_put(&lsmc
->sk
); /* sock_hold in smc_tcp_listen_work */
929 if (local_contact
== SMC_FIRST_CONTACT
)
930 smc_lgr_forget(new_smc
->conn
.lgr
);
931 mutex_unlock(&smc_create_lgr_pending
);
933 /* RDMA setup failed, switch back to TCP */
934 smc_conn_free(&new_smc
->conn
);
935 new_smc
->use_fallback
= true;
936 if (reason_code
&& (reason_code
!= SMC_CLC_DECL_REPLY
)) {
937 if (smc_clc_send_decline(new_smc
, reason_code
) < 0)
943 if (local_contact
== SMC_FIRST_CONTACT
)
944 smc_lgr_forget(new_smc
->conn
.lgr
);
945 mutex_unlock(&smc_create_lgr_pending
);
947 if (newsmcsk
->sk_state
== SMC_INIT
)
948 sock_put(&new_smc
->sk
); /* passive closing */
949 newsmcsk
->sk_state
= SMC_CLOSED
;
950 smc_conn_free(&new_smc
->conn
);
951 goto enqueue
; /* queue new sock with sk_err set */
954 static void smc_tcp_listen_work(struct work_struct
*work
)
956 struct smc_sock
*lsmc
= container_of(work
, struct smc_sock
,
958 struct sock
*lsk
= &lsmc
->sk
;
959 struct smc_sock
*new_smc
;
963 while (lsk
->sk_state
== SMC_LISTEN
) {
964 rc
= smc_clcsock_accept(lsmc
, &new_smc
);
970 new_smc
->listen_smc
= lsmc
;
971 new_smc
->use_fallback
= false; /* assume rdma capability first*/
972 sock_hold(lsk
); /* sock_put in smc_listen_work */
973 INIT_WORK(&new_smc
->smc_listen_work
, smc_listen_work
);
974 smc_copy_sock_settings_to_smc(new_smc
);
975 sock_hold(&new_smc
->sk
); /* sock_put in passive closing */
976 if (!schedule_work(&new_smc
->smc_listen_work
))
977 sock_put(&new_smc
->sk
);
982 sock_release(lsmc
->clcsock
);
983 lsmc
->clcsock
= NULL
;
986 sock_put(&lsmc
->sk
); /* sock_hold in smc_listen */
989 static int smc_listen(struct socket
*sock
, int backlog
)
991 struct sock
*sk
= sock
->sk
;
992 struct smc_sock
*smc
;
999 if ((sk
->sk_state
!= SMC_INIT
) && (sk
->sk_state
!= SMC_LISTEN
))
1003 if (sk
->sk_state
== SMC_LISTEN
) {
1004 sk
->sk_max_ack_backlog
= backlog
;
1007 /* some socket options are handled in core, so we could not apply
1008 * them to the clc socket -- copy smc socket options to clc socket
1010 smc_copy_sock_settings_to_clc(smc
);
1011 tcp_sk(smc
->clcsock
->sk
)->syn_smc
= 1;
1013 rc
= kernel_listen(smc
->clcsock
, backlog
);
1016 sk
->sk_max_ack_backlog
= backlog
;
1017 sk
->sk_ack_backlog
= 0;
1018 sk
->sk_state
= SMC_LISTEN
;
1019 INIT_WORK(&smc
->tcp_listen_work
, smc_tcp_listen_work
);
1020 sock_hold(sk
); /* sock_hold in tcp_listen_worker */
1021 if (!schedule_work(&smc
->tcp_listen_work
))
1029 static int smc_accept(struct socket
*sock
, struct socket
*new_sock
,
1030 int flags
, bool kern
)
1032 struct sock
*sk
= sock
->sk
, *nsk
;
1033 DECLARE_WAITQUEUE(wait
, current
);
1034 struct smc_sock
*lsmc
;
1039 sock_hold(sk
); /* sock_put below */
1042 if (lsmc
->sk
.sk_state
!= SMC_LISTEN
) {
1047 /* Wait for an incoming connection */
1048 timeo
= sock_rcvtimeo(sk
, flags
& O_NONBLOCK
);
1049 add_wait_queue_exclusive(sk_sleep(sk
), &wait
);
1050 while (!(nsk
= smc_accept_dequeue(sk
, new_sock
))) {
1051 set_current_state(TASK_INTERRUPTIBLE
);
1057 timeo
= schedule_timeout(timeo
);
1058 /* wakeup by sk_data_ready in smc_listen_work() */
1059 sched_annotate_sleep();
1061 if (signal_pending(current
)) {
1062 rc
= sock_intr_errno(timeo
);
1066 set_current_state(TASK_RUNNING
);
1067 remove_wait_queue(sk_sleep(sk
), &wait
);
1070 rc
= sock_error(nsk
);
1074 sock_put(sk
); /* sock_hold above */
1078 static int smc_getname(struct socket
*sock
, struct sockaddr
*addr
,
1081 struct smc_sock
*smc
;
1083 if (peer
&& (sock
->sk
->sk_state
!= SMC_ACTIVE
) &&
1084 (sock
->sk
->sk_state
!= SMC_APPCLOSEWAIT1
))
1087 smc
= smc_sk(sock
->sk
);
1089 return smc
->clcsock
->ops
->getname(smc
->clcsock
, addr
, peer
);
1092 static int smc_sendmsg(struct socket
*sock
, struct msghdr
*msg
, size_t len
)
1094 struct sock
*sk
= sock
->sk
;
1095 struct smc_sock
*smc
;
1100 if ((sk
->sk_state
!= SMC_ACTIVE
) &&
1101 (sk
->sk_state
!= SMC_APPCLOSEWAIT1
) &&
1102 (sk
->sk_state
!= SMC_INIT
))
1104 if (smc
->use_fallback
)
1105 rc
= smc
->clcsock
->ops
->sendmsg(smc
->clcsock
, msg
, len
);
1107 rc
= smc_tx_sendmsg(smc
, msg
, len
);
1113 static int smc_recvmsg(struct socket
*sock
, struct msghdr
*msg
, size_t len
,
1116 struct sock
*sk
= sock
->sk
;
1117 struct smc_sock
*smc
;
1122 if ((sk
->sk_state
== SMC_INIT
) ||
1123 (sk
->sk_state
== SMC_LISTEN
) ||
1124 (sk
->sk_state
== SMC_CLOSED
))
1127 if (sk
->sk_state
== SMC_PEERFINCLOSEWAIT
) {
1132 if (smc
->use_fallback
)
1133 rc
= smc
->clcsock
->ops
->recvmsg(smc
->clcsock
, msg
, len
, flags
);
1135 rc
= smc_rx_recvmsg(smc
, msg
, len
, flags
);
1142 static __poll_t
smc_accept_poll(struct sock
*parent
)
1144 struct smc_sock
*isk
= smc_sk(parent
);
1147 spin_lock(&isk
->accept_q_lock
);
1148 if (!list_empty(&isk
->accept_q
))
1149 mask
= EPOLLIN
| EPOLLRDNORM
;
1150 spin_unlock(&isk
->accept_q_lock
);
1155 static __poll_t
smc_poll(struct file
*file
, struct socket
*sock
,
1158 struct sock
*sk
= sock
->sk
;
1160 struct smc_sock
*smc
;
1166 smc
= smc_sk(sock
->sk
);
1169 if ((sk
->sk_state
== SMC_INIT
) || smc
->use_fallback
) {
1170 /* delegate to CLC child sock */
1172 mask
= smc
->clcsock
->ops
->poll(file
, smc
->clcsock
, wait
);
1173 /* if non-blocking connect finished ... */
1175 if ((sk
->sk_state
== SMC_INIT
) && (mask
& EPOLLOUT
)) {
1176 sk
->sk_err
= smc
->clcsock
->sk
->sk_err
;
1180 rc
= smc_connect_rdma(smc
);
1183 /* success cases including fallback */
1184 mask
|= EPOLLOUT
| EPOLLWRNORM
;
1188 if (sk
->sk_state
!= SMC_CLOSED
) {
1190 sock_poll_wait(file
, sk_sleep(sk
), wait
);
1195 if ((sk
->sk_shutdown
== SHUTDOWN_MASK
) ||
1196 (sk
->sk_state
== SMC_CLOSED
))
1198 if (sk
->sk_state
== SMC_LISTEN
) {
1199 /* woken up by sk_data_ready in smc_listen_work() */
1200 mask
= smc_accept_poll(sk
);
1202 if (atomic_read(&smc
->conn
.sndbuf_space
) ||
1203 sk
->sk_shutdown
& SEND_SHUTDOWN
) {
1204 mask
|= EPOLLOUT
| EPOLLWRNORM
;
1206 sk_set_bit(SOCKWQ_ASYNC_NOSPACE
, sk
);
1207 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
1209 if (atomic_read(&smc
->conn
.bytes_to_rcv
))
1210 mask
|= EPOLLIN
| EPOLLRDNORM
;
1211 if (sk
->sk_shutdown
& RCV_SHUTDOWN
)
1212 mask
|= EPOLLIN
| EPOLLRDNORM
| EPOLLRDHUP
;
1213 if (sk
->sk_state
== SMC_APPCLOSEWAIT1
)
1224 static int smc_shutdown(struct socket
*sock
, int how
)
1226 struct sock
*sk
= sock
->sk
;
1227 struct smc_sock
*smc
;
1233 if ((how
< SHUT_RD
) || (how
> SHUT_RDWR
))
1239 if ((sk
->sk_state
!= SMC_LISTEN
) &&
1240 (sk
->sk_state
!= SMC_ACTIVE
) &&
1241 (sk
->sk_state
!= SMC_PEERCLOSEWAIT1
) &&
1242 (sk
->sk_state
!= SMC_PEERCLOSEWAIT2
) &&
1243 (sk
->sk_state
!= SMC_APPCLOSEWAIT1
) &&
1244 (sk
->sk_state
!= SMC_APPCLOSEWAIT2
) &&
1245 (sk
->sk_state
!= SMC_APPFINCLOSEWAIT
))
1247 if (smc
->use_fallback
) {
1248 rc
= kernel_sock_shutdown(smc
->clcsock
, how
);
1249 sk
->sk_shutdown
= smc
->clcsock
->sk
->sk_shutdown
;
1250 if (sk
->sk_shutdown
== SHUTDOWN_MASK
)
1251 sk
->sk_state
= SMC_CLOSED
;
1255 case SHUT_RDWR
: /* shutdown in both directions */
1256 rc
= smc_close_active(smc
);
1259 rc
= smc_close_shutdown_write(smc
);
1262 if (sk
->sk_state
== SMC_LISTEN
)
1263 rc
= smc_close_active(smc
);
1266 /* nothing more to do because peer is not involved */
1269 rc1
= kernel_sock_shutdown(smc
->clcsock
, how
);
1270 /* map sock_shutdown_cmd constants to sk_shutdown value range */
1271 sk
->sk_shutdown
|= how
+ 1;
1275 return rc
? rc
: rc1
;
1278 static int smc_setsockopt(struct socket
*sock
, int level
, int optname
,
1279 char __user
*optval
, unsigned int optlen
)
1281 struct sock
*sk
= sock
->sk
;
1282 struct smc_sock
*smc
;
1286 /* generic setsockopts reaching us here always apply to the
1289 return smc
->clcsock
->ops
->setsockopt(smc
->clcsock
, level
, optname
,
1293 static int smc_getsockopt(struct socket
*sock
, int level
, int optname
,
1294 char __user
*optval
, int __user
*optlen
)
1296 struct smc_sock
*smc
;
1298 smc
= smc_sk(sock
->sk
);
1299 /* socket options apply to the CLC socket */
1300 return smc
->clcsock
->ops
->getsockopt(smc
->clcsock
, level
, optname
,
1304 static int smc_ioctl(struct socket
*sock
, unsigned int cmd
,
1307 struct smc_sock
*smc
;
1309 smc
= smc_sk(sock
->sk
);
1310 if (smc
->use_fallback
)
1311 return smc
->clcsock
->ops
->ioctl(smc
->clcsock
, cmd
, arg
);
1313 return sock_no_ioctl(sock
, cmd
, arg
);
1316 static ssize_t
smc_sendpage(struct socket
*sock
, struct page
*page
,
1317 int offset
, size_t size
, int flags
)
1319 struct sock
*sk
= sock
->sk
;
1320 struct smc_sock
*smc
;
1325 if (sk
->sk_state
!= SMC_ACTIVE
)
1327 if (smc
->use_fallback
)
1328 rc
= kernel_sendpage(smc
->clcsock
, page
, offset
,
1331 rc
= sock_no_sendpage(sock
, page
, offset
, size
, flags
);
1338 static ssize_t
smc_splice_read(struct socket
*sock
, loff_t
*ppos
,
1339 struct pipe_inode_info
*pipe
, size_t len
,
1342 struct sock
*sk
= sock
->sk
;
1343 struct smc_sock
*smc
;
1348 if ((sk
->sk_state
!= SMC_ACTIVE
) && (sk
->sk_state
!= SMC_CLOSED
))
1350 if (smc
->use_fallback
) {
1351 rc
= smc
->clcsock
->ops
->splice_read(smc
->clcsock
, ppos
,
1361 /* must look like tcp */
1362 static const struct proto_ops smc_sock_ops
= {
1364 .owner
= THIS_MODULE
,
1365 .release
= smc_release
,
1367 .connect
= smc_connect
,
1368 .socketpair
= sock_no_socketpair
,
1369 .accept
= smc_accept
,
1370 .getname
= smc_getname
,
1373 .listen
= smc_listen
,
1374 .shutdown
= smc_shutdown
,
1375 .setsockopt
= smc_setsockopt
,
1376 .getsockopt
= smc_getsockopt
,
1377 .sendmsg
= smc_sendmsg
,
1378 .recvmsg
= smc_recvmsg
,
1379 .mmap
= sock_no_mmap
,
1380 .sendpage
= smc_sendpage
,
1381 .splice_read
= smc_splice_read
,
1384 static int smc_create(struct net
*net
, struct socket
*sock
, int protocol
,
1387 int family
= (protocol
== SMCPROTO_SMC6
) ? PF_INET6
: PF_INET
;
1388 struct smc_sock
*smc
;
1392 rc
= -ESOCKTNOSUPPORT
;
1393 if (sock
->type
!= SOCK_STREAM
)
1396 rc
= -EPROTONOSUPPORT
;
1397 if (protocol
!= SMCPROTO_SMC
&& protocol
!= SMCPROTO_SMC6
)
1401 sock
->ops
= &smc_sock_ops
;
1402 sk
= smc_sock_alloc(net
, sock
, protocol
);
1406 /* create internal TCP socket for CLC handshake and fallback */
1408 smc
->use_fallback
= false; /* assume rdma capability first */
1409 rc
= sock_create_kern(net
, family
, SOCK_STREAM
, IPPROTO_TCP
,
1412 sk_common_release(sk
);
1415 smc
->sk
.sk_sndbuf
= max(smc
->clcsock
->sk
->sk_sndbuf
, SMC_BUF_MIN_SIZE
);
1416 smc
->sk
.sk_rcvbuf
= max(smc
->clcsock
->sk
->sk_rcvbuf
, SMC_BUF_MIN_SIZE
);
1422 static const struct net_proto_family smc_sock_family_ops
= {
1424 .owner
= THIS_MODULE
,
1425 .create
= smc_create
,
1428 static int __init
smc_init(void)
1432 rc
= smc_pnet_init();
1436 rc
= smc_llc_init();
1438 pr_err("%s: smc_llc_init fails with %d\n", __func__
, rc
);
1442 rc
= smc_cdc_init();
1444 pr_err("%s: smc_cdc_init fails with %d\n", __func__
, rc
);
1448 rc
= proto_register(&smc_proto
, 1);
1450 pr_err("%s: proto_register(v4) fails with %d\n", __func__
, rc
);
1454 rc
= proto_register(&smc_proto6
, 1);
1456 pr_err("%s: proto_register(v6) fails with %d\n", __func__
, rc
);
1460 rc
= sock_register(&smc_sock_family_ops
);
1462 pr_err("%s: sock_register fails with %d\n", __func__
, rc
);
1465 INIT_HLIST_HEAD(&smc_v4_hashinfo
.ht
);
1466 INIT_HLIST_HEAD(&smc_v6_hashinfo
.ht
);
1468 rc
= smc_ib_register_client();
1470 pr_err("%s: ib_register fails with %d\n", __func__
, rc
);
1474 static_branch_enable(&tcp_have_smc
);
1478 sock_unregister(PF_SMC
);
1480 proto_unregister(&smc_proto6
);
1482 proto_unregister(&smc_proto
);
1488 static void __exit
smc_exit(void)
1490 struct smc_link_group
*lgr
, *lg
;
1491 LIST_HEAD(lgr_freeing_list
);
1493 spin_lock_bh(&smc_lgr_list
.lock
);
1494 if (!list_empty(&smc_lgr_list
.list
))
1495 list_splice_init(&smc_lgr_list
.list
, &lgr_freeing_list
);
1496 spin_unlock_bh(&smc_lgr_list
.lock
);
1497 list_for_each_entry_safe(lgr
, lg
, &lgr_freeing_list
, list
) {
1498 list_del_init(&lgr
->list
);
1499 cancel_delayed_work_sync(&lgr
->free_work
);
1500 smc_lgr_free(lgr
); /* free link group */
1502 static_branch_disable(&tcp_have_smc
);
1503 smc_ib_unregister_client();
1504 sock_unregister(PF_SMC
);
1505 proto_unregister(&smc_proto6
);
1506 proto_unregister(&smc_proto
);
1510 module_init(smc_init
);
1511 module_exit(smc_exit
);
1513 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
1514 MODULE_DESCRIPTION("smc socket address family");
1515 MODULE_LICENSE("GPL");
1516 MODULE_ALIAS_NETPROTO(PF_SMC
);