1 // SPDX-License-Identifier: GPL-2.0-only
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
5 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
6 * applies to SOCK_STREAM sockets only
7 * offers an alternative communication option for TCP-protocol sockets
8 * applicable with RoCE-cards only
10 * Initial restrictions:
11 * - support for alternate links postponed
13 * Copyright IBM Corp. 2016, 2018
15 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
16 * based on prototype from Frank Blaschka
19 #define KMSG_COMPONENT "smc"
20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
22 #include <linux/module.h>
23 #include <linux/socket.h>
24 #include <linux/workqueue.h>
26 #include <linux/sched/signal.h>
27 #include <linux/if_vlan.h>
28 #include <linux/rcupdate_wait.h>
29 #include <linux/ctype.h>
34 #include <asm/ioctls.h>
36 #include <net/net_namespace.h>
37 #include <net/netns/generic.h>
38 #include "smc_netns.h"
48 #include "smc_netlink.h"
51 #include "smc_close.h"
52 #include "smc_stats.h"
54 static DEFINE_MUTEX(smc_server_lgr_pending
); /* serialize link group
57 static DEFINE_MUTEX(smc_client_lgr_pending
); /* serialize link group
61 struct workqueue_struct
*smc_hs_wq
; /* wq for handshake work */
62 struct workqueue_struct
*smc_close_wq
; /* wq for close work */
64 static void smc_tcp_listen_work(struct work_struct
*);
65 static void smc_connect_work(struct work_struct
*);
67 static void smc_set_keepalive(struct sock
*sk
, int val
)
69 struct smc_sock
*smc
= smc_sk(sk
);
71 smc
->clcsock
->sk
->sk_prot
->keepalive(smc
->clcsock
->sk
, val
);
74 static struct smc_hashinfo smc_v4_hashinfo
= {
75 .lock
= __RW_LOCK_UNLOCKED(smc_v4_hashinfo
.lock
),
78 static struct smc_hashinfo smc_v6_hashinfo
= {
79 .lock
= __RW_LOCK_UNLOCKED(smc_v6_hashinfo
.lock
),
82 int smc_hash_sk(struct sock
*sk
)
84 struct smc_hashinfo
*h
= sk
->sk_prot
->h
.smc_hash
;
85 struct hlist_head
*head
;
89 write_lock_bh(&h
->lock
);
90 sk_add_node(sk
, head
);
91 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, 1);
92 write_unlock_bh(&h
->lock
);
96 EXPORT_SYMBOL_GPL(smc_hash_sk
);
98 void smc_unhash_sk(struct sock
*sk
)
100 struct smc_hashinfo
*h
= sk
->sk_prot
->h
.smc_hash
;
102 write_lock_bh(&h
->lock
);
103 if (sk_del_node_init(sk
))
104 sock_prot_inuse_add(sock_net(sk
), sk
->sk_prot
, -1);
105 write_unlock_bh(&h
->lock
);
107 EXPORT_SYMBOL_GPL(smc_unhash_sk
);
109 struct proto smc_proto
= {
111 .owner
= THIS_MODULE
,
112 .keepalive
= smc_set_keepalive
,
114 .unhash
= smc_unhash_sk
,
115 .obj_size
= sizeof(struct smc_sock
),
116 .h
.smc_hash
= &smc_v4_hashinfo
,
117 .slab_flags
= SLAB_TYPESAFE_BY_RCU
,
119 EXPORT_SYMBOL_GPL(smc_proto
);
121 struct proto smc_proto6
= {
123 .owner
= THIS_MODULE
,
124 .keepalive
= smc_set_keepalive
,
126 .unhash
= smc_unhash_sk
,
127 .obj_size
= sizeof(struct smc_sock
),
128 .h
.smc_hash
= &smc_v6_hashinfo
,
129 .slab_flags
= SLAB_TYPESAFE_BY_RCU
,
131 EXPORT_SYMBOL_GPL(smc_proto6
);
133 static void smc_restore_fallback_changes(struct smc_sock
*smc
)
135 if (smc
->clcsock
->file
) { /* non-accepted sockets have no file yet */
136 smc
->clcsock
->file
->private_data
= smc
->sk
.sk_socket
;
137 smc
->clcsock
->file
= NULL
;
141 static int __smc_release(struct smc_sock
*smc
)
143 struct sock
*sk
= &smc
->sk
;
146 if (!smc
->use_fallback
) {
147 rc
= smc_close_active(smc
);
148 sock_set_flag(sk
, SOCK_DEAD
);
149 sk
->sk_shutdown
|= SHUTDOWN_MASK
;
151 if (sk
->sk_state
!= SMC_LISTEN
&& sk
->sk_state
!= SMC_INIT
)
152 sock_put(sk
); /* passive closing */
153 if (sk
->sk_state
== SMC_LISTEN
) {
154 /* wake up clcsock accept */
155 rc
= kernel_sock_shutdown(smc
->clcsock
, SHUT_RDWR
);
157 sk
->sk_state
= SMC_CLOSED
;
158 sk
->sk_state_change(sk
);
159 smc_restore_fallback_changes(smc
);
162 sk
->sk_prot
->unhash(sk
);
164 if (sk
->sk_state
== SMC_CLOSED
) {
167 smc_clcsock_release(smc
);
170 if (!smc
->use_fallback
)
171 smc_conn_free(&smc
->conn
);
177 static int smc_release(struct socket
*sock
)
179 struct sock
*sk
= sock
->sk
;
180 struct smc_sock
*smc
;
186 sock_hold(sk
); /* sock_put below */
189 /* cleanup for a dangling non-blocking connect */
190 if (smc
->connect_nonblock
&& sk
->sk_state
== SMC_INIT
)
191 tcp_abort(smc
->clcsock
->sk
, ECONNABORTED
);
192 flush_work(&smc
->connect_work
);
194 if (sk
->sk_state
== SMC_LISTEN
)
195 /* smc_close_non_accepted() is called and acquires
196 * sock lock for child sockets again
198 lock_sock_nested(sk
, SINGLE_DEPTH_NESTING
);
202 rc
= __smc_release(smc
);
209 sock_put(sk
); /* sock_hold above */
210 sock_put(sk
); /* final sock_put */
215 static void smc_destruct(struct sock
*sk
)
217 if (sk
->sk_state
!= SMC_CLOSED
)
219 if (!sock_flag(sk
, SOCK_DEAD
))
222 sk_refcnt_debug_dec(sk
);
225 static struct sock
*smc_sock_alloc(struct net
*net
, struct socket
*sock
,
228 struct smc_sock
*smc
;
232 prot
= (protocol
== SMCPROTO_SMC6
) ? &smc_proto6
: &smc_proto
;
233 sk
= sk_alloc(net
, PF_SMC
, GFP_KERNEL
, prot
, 0);
237 sock_init_data(sock
, sk
); /* sets sk_refcnt to 1 */
238 sk
->sk_state
= SMC_INIT
;
239 sk
->sk_destruct
= smc_destruct
;
240 sk
->sk_protocol
= protocol
;
242 INIT_WORK(&smc
->tcp_listen_work
, smc_tcp_listen_work
);
243 INIT_WORK(&smc
->connect_work
, smc_connect_work
);
244 INIT_DELAYED_WORK(&smc
->conn
.tx_work
, smc_tx_work
);
245 INIT_LIST_HEAD(&smc
->accept_q
);
246 spin_lock_init(&smc
->accept_q_lock
);
247 spin_lock_init(&smc
->conn
.send_lock
);
248 sk
->sk_prot
->hash(sk
);
249 sk_refcnt_debug_inc(sk
);
250 mutex_init(&smc
->clcsock_release_lock
);
255 static int smc_bind(struct socket
*sock
, struct sockaddr
*uaddr
,
258 struct sockaddr_in
*addr
= (struct sockaddr_in
*)uaddr
;
259 struct sock
*sk
= sock
->sk
;
260 struct smc_sock
*smc
;
265 /* replicate tests from inet_bind(), to be safe wrt. future changes */
267 if (addr_len
< sizeof(struct sockaddr_in
))
271 if (addr
->sin_family
!= AF_INET
&&
272 addr
->sin_family
!= AF_INET6
&&
273 addr
->sin_family
!= AF_UNSPEC
)
275 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
276 if (addr
->sin_family
== AF_UNSPEC
&&
277 addr
->sin_addr
.s_addr
!= htonl(INADDR_ANY
))
282 /* Check if socket is already active */
284 if (sk
->sk_state
!= SMC_INIT
|| smc
->connect_nonblock
)
287 smc
->clcsock
->sk
->sk_reuse
= sk
->sk_reuse
;
288 rc
= kernel_bind(smc
->clcsock
, uaddr
, addr_len
);
296 static void smc_copy_sock_settings(struct sock
*nsk
, struct sock
*osk
,
299 /* options we don't get control via setsockopt for */
300 nsk
->sk_type
= osk
->sk_type
;
301 nsk
->sk_sndbuf
= osk
->sk_sndbuf
;
302 nsk
->sk_rcvbuf
= osk
->sk_rcvbuf
;
303 nsk
->sk_sndtimeo
= osk
->sk_sndtimeo
;
304 nsk
->sk_rcvtimeo
= osk
->sk_rcvtimeo
;
305 nsk
->sk_mark
= osk
->sk_mark
;
306 nsk
->sk_priority
= osk
->sk_priority
;
307 nsk
->sk_rcvlowat
= osk
->sk_rcvlowat
;
308 nsk
->sk_bound_dev_if
= osk
->sk_bound_dev_if
;
309 nsk
->sk_err
= osk
->sk_err
;
311 nsk
->sk_flags
&= ~mask
;
312 nsk
->sk_flags
|= osk
->sk_flags
& mask
;
315 #define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
316 (1UL << SOCK_KEEPOPEN) | \
317 (1UL << SOCK_LINGER) | \
318 (1UL << SOCK_BROADCAST) | \
319 (1UL << SOCK_TIMESTAMP) | \
320 (1UL << SOCK_DBG) | \
321 (1UL << SOCK_RCVTSTAMP) | \
322 (1UL << SOCK_RCVTSTAMPNS) | \
323 (1UL << SOCK_LOCALROUTE) | \
324 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
325 (1UL << SOCK_RXQ_OVFL) | \
326 (1UL << SOCK_WIFI_STATUS) | \
327 (1UL << SOCK_NOFCS) | \
328 (1UL << SOCK_FILTER_LOCKED) | \
329 (1UL << SOCK_TSTAMP_NEW))
330 /* copy only relevant settings and flags of SOL_SOCKET level from smc to
331 * clc socket (since smc is not called for these options from net/core)
333 static void smc_copy_sock_settings_to_clc(struct smc_sock
*smc
)
335 smc_copy_sock_settings(smc
->clcsock
->sk
, &smc
->sk
, SK_FLAGS_SMC_TO_CLC
);
338 #define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
339 (1UL << SOCK_KEEPOPEN) | \
340 (1UL << SOCK_LINGER) | \
342 /* copy only settings and flags relevant for smc from clc to smc socket */
343 static void smc_copy_sock_settings_to_smc(struct smc_sock
*smc
)
345 smc_copy_sock_settings(&smc
->sk
, smc
->clcsock
->sk
, SK_FLAGS_CLC_TO_SMC
);
348 /* register the new rmb on all links */
349 static int smcr_lgr_reg_rmbs(struct smc_link
*link
,
350 struct smc_buf_desc
*rmb_desc
)
352 struct smc_link_group
*lgr
= link
->lgr
;
355 rc
= smc_llc_flow_initiate(lgr
, SMC_LLC_FLOW_RKEY
);
358 /* protect against parallel smc_llc_cli_rkey_exchange() and
359 * parallel smcr_link_reg_rmb()
361 mutex_lock(&lgr
->llc_conf_mutex
);
362 for (i
= 0; i
< SMC_LINKS_PER_LGR_MAX
; i
++) {
363 if (!smc_link_active(&lgr
->lnk
[i
]))
365 rc
= smcr_link_reg_rmb(&lgr
->lnk
[i
], rmb_desc
);
370 /* exchange confirm_rkey msg with peer */
371 rc
= smc_llc_do_confirm_rkey(link
, rmb_desc
);
376 rmb_desc
->is_conf_rkey
= true;
378 mutex_unlock(&lgr
->llc_conf_mutex
);
379 smc_llc_flow_stop(lgr
, &lgr
->llc_flow_lcl
);
383 static int smcr_clnt_conf_first_link(struct smc_sock
*smc
)
385 struct smc_link
*link
= smc
->conn
.lnk
;
386 struct smc_llc_qentry
*qentry
;
389 /* receive CONFIRM LINK request from server over RoCE fabric */
390 qentry
= smc_llc_wait(link
->lgr
, NULL
, SMC_LLC_WAIT_TIME
,
391 SMC_LLC_CONFIRM_LINK
);
393 struct smc_clc_msg_decline dclc
;
395 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
396 SMC_CLC_DECLINE
, CLC_WAIT_TIME_SHORT
);
397 return rc
== -EAGAIN
? SMC_CLC_DECL_TIMEOUT_CL
: rc
;
399 smc_llc_save_peer_uid(qentry
);
400 rc
= smc_llc_eval_conf_link(qentry
, SMC_LLC_REQ
);
401 smc_llc_flow_qentry_del(&link
->lgr
->llc_flow_lcl
);
403 return SMC_CLC_DECL_RMBE_EC
;
405 rc
= smc_ib_modify_qp_rts(link
);
407 return SMC_CLC_DECL_ERR_RDYLNK
;
409 smc_wr_remember_qp_attr(link
);
411 if (smcr_link_reg_rmb(link
, smc
->conn
.rmb_desc
))
412 return SMC_CLC_DECL_ERR_REGRMB
;
414 /* confirm_rkey is implicit on 1st contact */
415 smc
->conn
.rmb_desc
->is_conf_rkey
= true;
417 /* send CONFIRM LINK response over RoCE fabric */
418 rc
= smc_llc_send_confirm_link(link
, SMC_LLC_RESP
);
420 return SMC_CLC_DECL_TIMEOUT_CL
;
422 smc_llc_link_active(link
);
423 smcr_lgr_set_type(link
->lgr
, SMC_LGR_SINGLE
);
425 /* optional 2nd link, receive ADD LINK request from server */
426 qentry
= smc_llc_wait(link
->lgr
, NULL
, SMC_LLC_WAIT_TIME
,
429 struct smc_clc_msg_decline dclc
;
431 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
432 SMC_CLC_DECLINE
, CLC_WAIT_TIME_SHORT
);
434 rc
= 0; /* no DECLINE received, go with one link */
437 smc_llc_flow_qentry_clr(&link
->lgr
->llc_flow_lcl
);
438 smc_llc_cli_add_link(link
, qentry
);
442 static void smcr_conn_save_peer_info(struct smc_sock
*smc
,
443 struct smc_clc_msg_accept_confirm
*clc
)
445 int bufsize
= smc_uncompress_bufsize(clc
->r0
.rmbe_size
);
447 smc
->conn
.peer_rmbe_idx
= clc
->r0
.rmbe_idx
;
448 smc
->conn
.local_tx_ctrl
.token
= ntohl(clc
->r0
.rmbe_alert_token
);
449 smc
->conn
.peer_rmbe_size
= bufsize
;
450 atomic_set(&smc
->conn
.peer_rmbe_space
, smc
->conn
.peer_rmbe_size
);
451 smc
->conn
.tx_off
= bufsize
* (smc
->conn
.peer_rmbe_idx
- 1);
454 static bool smc_isascii(char *hostname
)
458 for (i
= 0; i
< SMC_MAX_HOSTNAME_LEN
; i
++)
459 if (!isascii(hostname
[i
]))
464 static void smcd_conn_save_peer_info(struct smc_sock
*smc
,
465 struct smc_clc_msg_accept_confirm
*clc
)
467 int bufsize
= smc_uncompress_bufsize(clc
->d0
.dmbe_size
);
469 smc
->conn
.peer_rmbe_idx
= clc
->d0
.dmbe_idx
;
470 smc
->conn
.peer_token
= clc
->d0
.token
;
471 /* msg header takes up space in the buffer */
472 smc
->conn
.peer_rmbe_size
= bufsize
- sizeof(struct smcd_cdc_msg
);
473 atomic_set(&smc
->conn
.peer_rmbe_space
, smc
->conn
.peer_rmbe_size
);
474 smc
->conn
.tx_off
= bufsize
* smc
->conn
.peer_rmbe_idx
;
475 if (clc
->hdr
.version
> SMC_V1
&&
476 (clc
->hdr
.typev2
& SMC_FIRST_CONTACT_MASK
)) {
477 struct smc_clc_msg_accept_confirm_v2
*clc_v2
=
478 (struct smc_clc_msg_accept_confirm_v2
*)clc
;
479 struct smc_clc_first_contact_ext
*fce
=
480 (struct smc_clc_first_contact_ext
*)
481 (((u8
*)clc_v2
) + sizeof(*clc_v2
));
483 memcpy(smc
->conn
.lgr
->negotiated_eid
, clc_v2
->eid
,
485 smc
->conn
.lgr
->peer_os
= fce
->os_type
;
486 smc
->conn
.lgr
->peer_smc_release
= fce
->release
;
487 if (smc_isascii(fce
->hostname
))
488 memcpy(smc
->conn
.lgr
->peer_hostname
, fce
->hostname
,
489 SMC_MAX_HOSTNAME_LEN
);
493 static void smc_conn_save_peer_info(struct smc_sock
*smc
,
494 struct smc_clc_msg_accept_confirm
*clc
)
496 if (smc
->conn
.lgr
->is_smcd
)
497 smcd_conn_save_peer_info(smc
, clc
);
499 smcr_conn_save_peer_info(smc
, clc
);
502 static void smc_link_save_peer_info(struct smc_link
*link
,
503 struct smc_clc_msg_accept_confirm
*clc
)
505 link
->peer_qpn
= ntoh24(clc
->r0
.qpn
);
506 memcpy(link
->peer_gid
, clc
->r0
.lcl
.gid
, SMC_GID_SIZE
);
507 memcpy(link
->peer_mac
, clc
->r0
.lcl
.mac
, sizeof(link
->peer_mac
));
508 link
->peer_psn
= ntoh24(clc
->r0
.psn
);
509 link
->peer_mtu
= clc
->r0
.qp_mtu
;
512 static void smc_stat_inc_fback_rsn_cnt(struct smc_sock
*smc
,
513 struct smc_stats_fback
*fback_arr
)
517 for (cnt
= 0; cnt
< SMC_MAX_FBACK_RSN_CNT
; cnt
++) {
518 if (fback_arr
[cnt
].fback_code
== smc
->fallback_rsn
) {
519 fback_arr
[cnt
].count
++;
522 if (!fback_arr
[cnt
].fback_code
) {
523 fback_arr
[cnt
].fback_code
= smc
->fallback_rsn
;
524 fback_arr
[cnt
].count
++;
530 static void smc_stat_fallback(struct smc_sock
*smc
)
532 struct net
*net
= sock_net(&smc
->sk
);
534 mutex_lock(&net
->smc
.mutex_fback_rsn
);
535 if (smc
->listen_smc
) {
536 smc_stat_inc_fback_rsn_cnt(smc
, net
->smc
.fback_rsn
->srv
);
537 net
->smc
.fback_rsn
->srv_fback_cnt
++;
539 smc_stat_inc_fback_rsn_cnt(smc
, net
->smc
.fback_rsn
->clnt
);
540 net
->smc
.fback_rsn
->clnt_fback_cnt
++;
542 mutex_unlock(&net
->smc
.mutex_fback_rsn
);
545 static void smc_switch_to_fallback(struct smc_sock
*smc
, int reason_code
)
547 smc
->use_fallback
= true;
548 smc
->fallback_rsn
= reason_code
;
549 smc_stat_fallback(smc
);
550 if (smc
->sk
.sk_socket
&& smc
->sk
.sk_socket
->file
) {
551 smc
->clcsock
->file
= smc
->sk
.sk_socket
->file
;
552 smc
->clcsock
->file
->private_data
= smc
->clcsock
;
553 smc
->clcsock
->wq
.fasync_list
=
554 smc
->sk
.sk_socket
->wq
.fasync_list
;
558 /* fall back during connect */
559 static int smc_connect_fallback(struct smc_sock
*smc
, int reason_code
)
561 smc_switch_to_fallback(smc
, reason_code
);
562 smc_copy_sock_settings_to_clc(smc
);
563 smc
->connect_nonblock
= 0;
564 if (smc
->sk
.sk_state
== SMC_INIT
)
565 smc
->sk
.sk_state
= SMC_ACTIVE
;
569 /* decline and fall back during connect */
570 static int smc_connect_decline_fallback(struct smc_sock
*smc
, int reason_code
,
573 struct net
*net
= sock_net(&smc
->sk
);
576 if (reason_code
< 0) { /* error, fallback is not possible */
577 this_cpu_inc(net
->smc
.smc_stats
->clnt_hshake_err_cnt
);
578 if (smc
->sk
.sk_state
== SMC_INIT
)
579 sock_put(&smc
->sk
); /* passive closing */
582 if (reason_code
!= SMC_CLC_DECL_PEERDECL
) {
583 rc
= smc_clc_send_decline(smc
, reason_code
, version
);
585 this_cpu_inc(net
->smc
.smc_stats
->clnt_hshake_err_cnt
);
586 if (smc
->sk
.sk_state
== SMC_INIT
)
587 sock_put(&smc
->sk
); /* passive closing */
591 return smc_connect_fallback(smc
, reason_code
);
594 static void smc_conn_abort(struct smc_sock
*smc
, int local_first
)
597 smc_lgr_cleanup_early(&smc
->conn
);
599 smc_conn_free(&smc
->conn
);
602 /* check if there is a rdma device available for this connection. */
603 /* called for connect and listen */
604 static int smc_find_rdma_device(struct smc_sock
*smc
, struct smc_init_info
*ini
)
606 /* PNET table look up: search active ib_device and port
607 * within same PNETID that also contains the ethernet device
608 * used for the internal TCP socket
610 smc_pnet_find_roce_resource(smc
->clcsock
->sk
, ini
);
612 return SMC_CLC_DECL_NOSMCRDEV
;
616 /* check if there is an ISM device available for this connection. */
617 /* called for connect and listen */
618 static int smc_find_ism_device(struct smc_sock
*smc
, struct smc_init_info
*ini
)
620 /* Find ISM device with same PNETID as connecting interface */
621 smc_pnet_find_ism_resource(smc
->clcsock
->sk
, ini
);
622 if (!ini
->ism_dev
[0])
623 return SMC_CLC_DECL_NOSMCDDEV
;
625 ini
->ism_chid
[0] = smc_ism_get_chid(ini
->ism_dev
[0]);
629 /* is chid unique for the ism devices that are already determined? */
630 static bool smc_find_ism_v2_is_unique_chid(u16 chid
, struct smc_init_info
*ini
,
633 int i
= (!ini
->ism_dev
[0]) ? 1 : 0;
636 if (ini
->ism_chid
[i
] == chid
)
641 /* determine possible V2 ISM devices (either without PNETID or with PNETID plus
642 * PNETID matching net_device)
644 static int smc_find_ism_v2_device_clnt(struct smc_sock
*smc
,
645 struct smc_init_info
*ini
)
647 int rc
= SMC_CLC_DECL_NOSMCDDEV
;
648 struct smcd_dev
*smcd
;
652 if (smcd_indicated(ini
->smc_type_v1
))
653 rc
= 0; /* already initialized for V1 */
654 mutex_lock(&smcd_dev_list
.mutex
);
655 list_for_each_entry(smcd
, &smcd_dev_list
.list
, list
) {
656 if (smcd
->going_away
|| smcd
== ini
->ism_dev
[0])
658 chid
= smc_ism_get_chid(smcd
);
659 if (!smc_find_ism_v2_is_unique_chid(chid
, ini
, i
))
661 if (!smc_pnet_is_pnetid_set(smcd
->pnetid
) ||
662 smc_pnet_is_ndev_pnetid(sock_net(&smc
->sk
), smcd
->pnetid
)) {
663 ini
->ism_dev
[i
] = smcd
;
664 ini
->ism_chid
[i
] = chid
;
668 if (i
> SMC_MAX_ISM_DEVS
)
672 mutex_unlock(&smcd_dev_list
.mutex
);
673 ini
->ism_offered_cnt
= i
- 1;
674 if (!ini
->ism_dev
[0] && !ini
->ism_dev
[1])
675 ini
->smcd_version
= 0;
680 /* Check for VLAN ID and register it on ISM device just for CLC handshake */
681 static int smc_connect_ism_vlan_setup(struct smc_sock
*smc
,
682 struct smc_init_info
*ini
)
684 if (ini
->vlan_id
&& smc_ism_get_vlan(ini
->ism_dev
[0], ini
->vlan_id
))
685 return SMC_CLC_DECL_ISMVLANERR
;
689 static int smc_find_proposal_devices(struct smc_sock
*smc
,
690 struct smc_init_info
*ini
)
694 /* check if there is an ism device available */
695 if (ini
->smcd_version
& SMC_V1
) {
696 if (smc_find_ism_device(smc
, ini
) ||
697 smc_connect_ism_vlan_setup(smc
, ini
)) {
698 if (ini
->smc_type_v1
== SMC_TYPE_B
)
699 ini
->smc_type_v1
= SMC_TYPE_R
;
701 ini
->smc_type_v1
= SMC_TYPE_N
;
702 } /* else ISM V1 is supported for this connection */
703 if (smc_find_rdma_device(smc
, ini
)) {
704 if (ini
->smc_type_v1
== SMC_TYPE_B
)
705 ini
->smc_type_v1
= SMC_TYPE_D
;
707 ini
->smc_type_v1
= SMC_TYPE_N
;
708 } /* else RDMA is supported for this connection */
710 if (smc_ism_is_v2_capable() && smc_find_ism_v2_device_clnt(smc
, ini
))
711 ini
->smc_type_v2
= SMC_TYPE_N
;
713 /* if neither ISM nor RDMA are supported, fallback */
714 if (!smcr_indicated(ini
->smc_type_v1
) &&
715 ini
->smc_type_v1
== SMC_TYPE_N
&& ini
->smc_type_v2
== SMC_TYPE_N
)
716 rc
= SMC_CLC_DECL_NOSMCDEV
;
721 /* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
722 * used, the VLAN ID will be registered again during the connection setup.
724 static int smc_connect_ism_vlan_cleanup(struct smc_sock
*smc
,
725 struct smc_init_info
*ini
)
727 if (!smcd_indicated(ini
->smc_type_v1
))
729 if (ini
->vlan_id
&& smc_ism_put_vlan(ini
->ism_dev
[0], ini
->vlan_id
))
730 return SMC_CLC_DECL_CNFERR
;
734 #define SMC_CLC_MAX_ACCEPT_LEN \
735 (sizeof(struct smc_clc_msg_accept_confirm_v2) + \
736 sizeof(struct smc_clc_first_contact_ext) + \
737 sizeof(struct smc_clc_msg_trail))
739 /* CLC handshake during connect */
740 static int smc_connect_clc(struct smc_sock
*smc
,
741 struct smc_clc_msg_accept_confirm_v2
*aclc2
,
742 struct smc_init_info
*ini
)
746 /* do inband token exchange */
747 rc
= smc_clc_send_proposal(smc
, ini
);
750 /* receive SMC Accept CLC message */
751 return smc_clc_wait_msg(smc
, aclc2
, SMC_CLC_MAX_ACCEPT_LEN
,
752 SMC_CLC_ACCEPT
, CLC_WAIT_TIME
);
755 /* setup for RDMA connection of client */
756 static int smc_connect_rdma(struct smc_sock
*smc
,
757 struct smc_clc_msg_accept_confirm
*aclc
,
758 struct smc_init_info
*ini
)
760 int i
, reason_code
= 0;
761 struct smc_link
*link
;
763 ini
->is_smcd
= false;
764 ini
->ib_lcl
= &aclc
->r0
.lcl
;
765 ini
->ib_clcqpn
= ntoh24(aclc
->r0
.qpn
);
766 ini
->first_contact_peer
= aclc
->hdr
.typev2
& SMC_FIRST_CONTACT_MASK
;
768 mutex_lock(&smc_client_lgr_pending
);
769 reason_code
= smc_conn_create(smc
, ini
);
771 mutex_unlock(&smc_client_lgr_pending
);
775 smc_conn_save_peer_info(smc
, aclc
);
777 if (ini
->first_contact_local
) {
778 link
= smc
->conn
.lnk
;
780 /* set link that was assigned by server */
782 for (i
= 0; i
< SMC_LINKS_PER_LGR_MAX
; i
++) {
783 struct smc_link
*l
= &smc
->conn
.lgr
->lnk
[i
];
785 if (l
->peer_qpn
== ntoh24(aclc
->r0
.qpn
) &&
786 !memcmp(l
->peer_gid
, &aclc
->r0
.lcl
.gid
,
788 !memcmp(l
->peer_mac
, &aclc
->r0
.lcl
.mac
,
789 sizeof(l
->peer_mac
))) {
795 reason_code
= SMC_CLC_DECL_NOSRVLINK
;
798 smc_switch_link_and_count(&smc
->conn
, link
);
801 /* create send buffer and rmb */
802 if (smc_buf_create(smc
, false)) {
803 reason_code
= SMC_CLC_DECL_MEM
;
807 if (ini
->first_contact_local
)
808 smc_link_save_peer_info(link
, aclc
);
810 if (smc_rmb_rtoken_handling(&smc
->conn
, link
, aclc
)) {
811 reason_code
= SMC_CLC_DECL_ERR_RTOK
;
818 if (ini
->first_contact_local
) {
819 if (smc_ib_ready_link(link
)) {
820 reason_code
= SMC_CLC_DECL_ERR_RDYLNK
;
824 if (smcr_lgr_reg_rmbs(link
, smc
->conn
.rmb_desc
)) {
825 reason_code
= SMC_CLC_DECL_ERR_REGRMB
;
829 smc_rmb_sync_sg_for_device(&smc
->conn
);
831 reason_code
= smc_clc_send_confirm(smc
, ini
->first_contact_local
,
838 if (ini
->first_contact_local
) {
839 /* QP confirmation over RoCE fabric */
840 smc_llc_flow_initiate(link
->lgr
, SMC_LLC_FLOW_ADD_LINK
);
841 reason_code
= smcr_clnt_conf_first_link(smc
);
842 smc_llc_flow_stop(link
->lgr
, &link
->lgr
->llc_flow_lcl
);
846 mutex_unlock(&smc_client_lgr_pending
);
848 smc_copy_sock_settings_to_clc(smc
);
849 smc
->connect_nonblock
= 0;
850 if (smc
->sk
.sk_state
== SMC_INIT
)
851 smc
->sk
.sk_state
= SMC_ACTIVE
;
855 smc_conn_abort(smc
, ini
->first_contact_local
);
856 mutex_unlock(&smc_client_lgr_pending
);
857 smc
->connect_nonblock
= 0;
862 /* The server has chosen one of the proposed ISM devices for the communication.
863 * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
866 smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2
*aclc
,
867 struct smc_init_info
*ini
)
871 for (i
= 0; i
< ini
->ism_offered_cnt
+ 1; i
++) {
872 if (ini
->ism_chid
[i
] == ntohs(aclc
->chid
)) {
873 ini
->ism_selected
= i
;
881 /* setup for ISM connection of client */
882 static int smc_connect_ism(struct smc_sock
*smc
,
883 struct smc_clc_msg_accept_confirm
*aclc
,
884 struct smc_init_info
*ini
)
889 ini
->first_contact_peer
= aclc
->hdr
.typev2
& SMC_FIRST_CONTACT_MASK
;
891 if (aclc
->hdr
.version
== SMC_V2
) {
892 struct smc_clc_msg_accept_confirm_v2
*aclc_v2
=
893 (struct smc_clc_msg_accept_confirm_v2
*)aclc
;
895 rc
= smc_v2_determine_accepted_chid(aclc_v2
, ini
);
899 ini
->ism_peer_gid
[ini
->ism_selected
] = aclc
->d0
.gid
;
901 /* there is only one lgr role for SMC-D; use server lock */
902 mutex_lock(&smc_server_lgr_pending
);
903 rc
= smc_conn_create(smc
, ini
);
905 mutex_unlock(&smc_server_lgr_pending
);
909 /* Create send and receive buffers */
910 rc
= smc_buf_create(smc
, true);
912 rc
= (rc
== -ENOSPC
) ? SMC_CLC_DECL_MAX_DMB
: SMC_CLC_DECL_MEM
;
916 smc_conn_save_peer_info(smc
, aclc
);
921 rc
= smc_clc_send_confirm(smc
, ini
->first_contact_local
,
925 mutex_unlock(&smc_server_lgr_pending
);
927 smc_copy_sock_settings_to_clc(smc
);
928 smc
->connect_nonblock
= 0;
929 if (smc
->sk
.sk_state
== SMC_INIT
)
930 smc
->sk
.sk_state
= SMC_ACTIVE
;
934 smc_conn_abort(smc
, ini
->first_contact_local
);
935 mutex_unlock(&smc_server_lgr_pending
);
936 smc
->connect_nonblock
= 0;
941 /* check if received accept type and version matches a proposed one */
942 static int smc_connect_check_aclc(struct smc_init_info
*ini
,
943 struct smc_clc_msg_accept_confirm
*aclc
)
945 if ((aclc
->hdr
.typev1
== SMC_TYPE_R
&&
946 !smcr_indicated(ini
->smc_type_v1
)) ||
947 (aclc
->hdr
.typev1
== SMC_TYPE_D
&&
948 ((!smcd_indicated(ini
->smc_type_v1
) &&
949 !smcd_indicated(ini
->smc_type_v2
)) ||
950 (aclc
->hdr
.version
== SMC_V1
&&
951 !smcd_indicated(ini
->smc_type_v1
)) ||
952 (aclc
->hdr
.version
== SMC_V2
&&
953 !smcd_indicated(ini
->smc_type_v2
)))))
954 return SMC_CLC_DECL_MODEUNSUPP
;
959 /* perform steps before actually connecting */
960 static int __smc_connect(struct smc_sock
*smc
)
962 u8 version
= smc_ism_is_v2_capable() ? SMC_V2
: SMC_V1
;
963 struct smc_clc_msg_accept_confirm_v2
*aclc2
;
964 struct smc_clc_msg_accept_confirm
*aclc
;
965 struct smc_init_info
*ini
= NULL
;
969 if (smc
->use_fallback
)
970 return smc_connect_fallback(smc
, smc
->fallback_rsn
);
972 /* if peer has not signalled SMC-capability, fall back */
973 if (!tcp_sk(smc
->clcsock
->sk
)->syn_smc
)
974 return smc_connect_fallback(smc
, SMC_CLC_DECL_PEERNOSMC
);
976 /* IPSec connections opt out of SMC optimizations */
977 if (using_ipsec(smc
))
978 return smc_connect_decline_fallback(smc
, SMC_CLC_DECL_IPSEC
,
981 ini
= kzalloc(sizeof(*ini
), GFP_KERNEL
);
983 return smc_connect_decline_fallback(smc
, SMC_CLC_DECL_MEM
,
986 ini
->smcd_version
= SMC_V1
;
987 ini
->smcd_version
|= smc_ism_is_v2_capable() ? SMC_V2
: 0;
988 ini
->smc_type_v1
= SMC_TYPE_B
;
989 ini
->smc_type_v2
= smc_ism_is_v2_capable() ? SMC_TYPE_D
: SMC_TYPE_N
;
991 /* get vlan id from IP device */
992 if (smc_vlan_by_tcpsk(smc
->clcsock
, ini
)) {
993 ini
->smcd_version
&= ~SMC_V1
;
994 ini
->smc_type_v1
= SMC_TYPE_N
;
995 if (!ini
->smcd_version
) {
996 rc
= SMC_CLC_DECL_GETVLANERR
;
1001 rc
= smc_find_proposal_devices(smc
, ini
);
1005 buf
= kzalloc(SMC_CLC_MAX_ACCEPT_LEN
, GFP_KERNEL
);
1007 rc
= SMC_CLC_DECL_MEM
;
1010 aclc2
= (struct smc_clc_msg_accept_confirm_v2
*)buf
;
1011 aclc
= (struct smc_clc_msg_accept_confirm
*)aclc2
;
1013 /* perform CLC handshake */
1014 rc
= smc_connect_clc(smc
, aclc2
, ini
);
1018 /* check if smc modes and versions of CLC proposal and accept match */
1019 rc
= smc_connect_check_aclc(ini
, aclc
);
1020 version
= aclc
->hdr
.version
== SMC_V1
? SMC_V1
: SMC_V2
;
1021 ini
->smcd_version
= version
;
1025 /* depending on previous steps, connect using rdma or ism */
1026 if (aclc
->hdr
.typev1
== SMC_TYPE_R
)
1027 rc
= smc_connect_rdma(smc
, aclc
, ini
);
1028 else if (aclc
->hdr
.typev1
== SMC_TYPE_D
)
1029 rc
= smc_connect_ism(smc
, aclc
, ini
);
1033 SMC_STAT_CLNT_SUCC_INC(sock_net(smc
->clcsock
->sk
), aclc
);
1034 smc_connect_ism_vlan_cleanup(smc
, ini
);
1040 smc_connect_ism_vlan_cleanup(smc
, ini
);
1044 return smc_connect_decline_fallback(smc
, rc
, version
);
1047 static void smc_connect_work(struct work_struct
*work
)
1049 struct smc_sock
*smc
= container_of(work
, struct smc_sock
,
1051 long timeo
= smc
->sk
.sk_sndtimeo
;
1055 timeo
= MAX_SCHEDULE_TIMEOUT
;
1056 lock_sock(smc
->clcsock
->sk
);
1057 if (smc
->clcsock
->sk
->sk_err
) {
1058 smc
->sk
.sk_err
= smc
->clcsock
->sk
->sk_err
;
1059 } else if ((1 << smc
->clcsock
->sk
->sk_state
) &
1060 (TCPF_SYN_SENT
| TCPF_SYN_RECV
)) {
1061 rc
= sk_stream_wait_connect(smc
->clcsock
->sk
, &timeo
);
1062 if ((rc
== -EPIPE
) &&
1063 ((1 << smc
->clcsock
->sk
->sk_state
) &
1064 (TCPF_ESTABLISHED
| TCPF_CLOSE_WAIT
)))
1067 release_sock(smc
->clcsock
->sk
);
1068 lock_sock(&smc
->sk
);
1069 if (rc
!= 0 || smc
->sk
.sk_err
) {
1070 smc
->sk
.sk_state
= SMC_CLOSED
;
1071 if (rc
== -EPIPE
|| rc
== -EAGAIN
)
1072 smc
->sk
.sk_err
= EPIPE
;
1073 else if (signal_pending(current
))
1074 smc
->sk
.sk_err
= -sock_intr_errno(timeo
);
1075 sock_put(&smc
->sk
); /* passive closing */
1079 rc
= __smc_connect(smc
);
1081 smc
->sk
.sk_err
= -rc
;
1084 if (!sock_flag(&smc
->sk
, SOCK_DEAD
)) {
1085 if (smc
->sk
.sk_err
) {
1086 smc
->sk
.sk_state_change(&smc
->sk
);
1087 } else { /* allow polling before and after fallback decision */
1088 smc
->clcsock
->sk
->sk_write_space(smc
->clcsock
->sk
);
1089 smc
->sk
.sk_write_space(&smc
->sk
);
1092 release_sock(&smc
->sk
);
1095 static int smc_connect(struct socket
*sock
, struct sockaddr
*addr
,
1096 int alen
, int flags
)
1098 struct sock
*sk
= sock
->sk
;
1099 struct smc_sock
*smc
;
1104 /* separate smc parameter checking to be safe */
1105 if (alen
< sizeof(addr
->sa_family
))
1107 if (addr
->sa_family
!= AF_INET
&& addr
->sa_family
!= AF_INET6
)
1111 switch (sk
->sk_state
) {
1121 smc_copy_sock_settings_to_clc(smc
);
1122 tcp_sk(smc
->clcsock
->sk
)->syn_smc
= 1;
1123 if (smc
->connect_nonblock
) {
1127 rc
= kernel_connect(smc
->clcsock
, addr
, alen
, flags
);
1128 if (rc
&& rc
!= -EINPROGRESS
)
1131 sock_hold(&smc
->sk
); /* sock put in passive closing */
1132 if (smc
->use_fallback
)
1134 if (flags
& O_NONBLOCK
) {
1135 if (queue_work(smc_hs_wq
, &smc
->connect_work
))
1136 smc
->connect_nonblock
= 1;
1139 rc
= __smc_connect(smc
);
1143 rc
= 0; /* success cases including fallback */
1152 static int smc_clcsock_accept(struct smc_sock
*lsmc
, struct smc_sock
**new_smc
)
1154 struct socket
*new_clcsock
= NULL
;
1155 struct sock
*lsk
= &lsmc
->sk
;
1156 struct sock
*new_sk
;
1160 new_sk
= smc_sock_alloc(sock_net(lsk
), NULL
, lsk
->sk_protocol
);
1163 lsk
->sk_err
= ENOMEM
;
1168 *new_smc
= smc_sk(new_sk
);
1170 mutex_lock(&lsmc
->clcsock_release_lock
);
1172 rc
= kernel_accept(lsmc
->clcsock
, &new_clcsock
, SOCK_NONBLOCK
);
1173 mutex_unlock(&lsmc
->clcsock_release_lock
);
1175 if (rc
< 0 && rc
!= -EAGAIN
)
1177 if (rc
< 0 || lsk
->sk_state
== SMC_CLOSED
) {
1178 new_sk
->sk_prot
->unhash(new_sk
);
1180 sock_release(new_clcsock
);
1181 new_sk
->sk_state
= SMC_CLOSED
;
1182 sock_set_flag(new_sk
, SOCK_DEAD
);
1183 sock_put(new_sk
); /* final */
1188 /* new clcsock has inherited the smc listen-specific sk_data_ready
1189 * function; switch it back to the original sk_data_ready function
1191 new_clcsock
->sk
->sk_data_ready
= lsmc
->clcsk_data_ready
;
1192 (*new_smc
)->clcsock
= new_clcsock
;
1197 /* add a just created sock to the accept queue of the listen sock as
1198 * candidate for a following socket accept call from user space
1200 static void smc_accept_enqueue(struct sock
*parent
, struct sock
*sk
)
1202 struct smc_sock
*par
= smc_sk(parent
);
1204 sock_hold(sk
); /* sock_put in smc_accept_unlink () */
1205 spin_lock(&par
->accept_q_lock
);
1206 list_add_tail(&smc_sk(sk
)->accept_q
, &par
->accept_q
);
1207 spin_unlock(&par
->accept_q_lock
);
1208 sk_acceptq_added(parent
);
1211 /* remove a socket from the accept queue of its parental listening socket */
1212 static void smc_accept_unlink(struct sock
*sk
)
1214 struct smc_sock
*par
= smc_sk(sk
)->listen_smc
;
1216 spin_lock(&par
->accept_q_lock
);
1217 list_del_init(&smc_sk(sk
)->accept_q
);
1218 spin_unlock(&par
->accept_q_lock
);
1219 sk_acceptq_removed(&smc_sk(sk
)->listen_smc
->sk
);
1220 sock_put(sk
); /* sock_hold in smc_accept_enqueue */
1223 /* remove a sock from the accept queue to bind it to a new socket created
1224 * for a socket accept call from user space
1226 struct sock
*smc_accept_dequeue(struct sock
*parent
,
1227 struct socket
*new_sock
)
1229 struct smc_sock
*isk
, *n
;
1230 struct sock
*new_sk
;
1232 list_for_each_entry_safe(isk
, n
, &smc_sk(parent
)->accept_q
, accept_q
) {
1233 new_sk
= (struct sock
*)isk
;
1235 smc_accept_unlink(new_sk
);
1236 if (new_sk
->sk_state
== SMC_CLOSED
) {
1237 new_sk
->sk_prot
->unhash(new_sk
);
1239 sock_release(isk
->clcsock
);
1240 isk
->clcsock
= NULL
;
1242 sock_put(new_sk
); /* final */
1246 sock_graft(new_sk
, new_sock
);
1247 if (isk
->use_fallback
) {
1248 smc_sk(new_sk
)->clcsock
->file
= new_sock
->file
;
1249 isk
->clcsock
->file
->private_data
= isk
->clcsock
;
1257 /* clean up for a created but never accepted sock */
1258 void smc_close_non_accepted(struct sock
*sk
)
1260 struct smc_sock
*smc
= smc_sk(sk
);
1262 sock_hold(sk
); /* sock_put below */
1264 if (!sk
->sk_lingertime
)
1265 /* wait for peer closing */
1266 sk
->sk_lingertime
= SMC_MAX_STREAM_WAIT_TIMEOUT
;
1269 sock_put(sk
); /* sock_hold above */
1270 sock_put(sk
); /* final sock_put */
1273 static int smcr_serv_conf_first_link(struct smc_sock
*smc
)
1275 struct smc_link
*link
= smc
->conn
.lnk
;
1276 struct smc_llc_qentry
*qentry
;
1279 if (smcr_link_reg_rmb(link
, smc
->conn
.rmb_desc
))
1280 return SMC_CLC_DECL_ERR_REGRMB
;
1282 /* send CONFIRM LINK request to client over the RoCE fabric */
1283 rc
= smc_llc_send_confirm_link(link
, SMC_LLC_REQ
);
1285 return SMC_CLC_DECL_TIMEOUT_CL
;
1287 /* receive CONFIRM LINK response from client over the RoCE fabric */
1288 qentry
= smc_llc_wait(link
->lgr
, link
, SMC_LLC_WAIT_TIME
,
1289 SMC_LLC_CONFIRM_LINK
);
1291 struct smc_clc_msg_decline dclc
;
1293 rc
= smc_clc_wait_msg(smc
, &dclc
, sizeof(dclc
),
1294 SMC_CLC_DECLINE
, CLC_WAIT_TIME_SHORT
);
1295 return rc
== -EAGAIN
? SMC_CLC_DECL_TIMEOUT_CL
: rc
;
1297 smc_llc_save_peer_uid(qentry
);
1298 rc
= smc_llc_eval_conf_link(qentry
, SMC_LLC_RESP
);
1299 smc_llc_flow_qentry_del(&link
->lgr
->llc_flow_lcl
);
1301 return SMC_CLC_DECL_RMBE_EC
;
1303 /* confirm_rkey is implicit on 1st contact */
1304 smc
->conn
.rmb_desc
->is_conf_rkey
= true;
1306 smc_llc_link_active(link
);
1307 smcr_lgr_set_type(link
->lgr
, SMC_LGR_SINGLE
);
1309 /* initial contact - try to establish second link */
1310 smc_llc_srv_add_link(link
);
1314 /* listen worker: finish */
1315 static void smc_listen_out(struct smc_sock
*new_smc
)
1317 struct smc_sock
*lsmc
= new_smc
->listen_smc
;
1318 struct sock
*newsmcsk
= &new_smc
->sk
;
1320 if (lsmc
->sk
.sk_state
== SMC_LISTEN
) {
1321 lock_sock_nested(&lsmc
->sk
, SINGLE_DEPTH_NESTING
);
1322 smc_accept_enqueue(&lsmc
->sk
, newsmcsk
);
1323 release_sock(&lsmc
->sk
);
1324 } else { /* no longer listening */
1325 smc_close_non_accepted(newsmcsk
);
1328 /* Wake up accept */
1329 lsmc
->sk
.sk_data_ready(&lsmc
->sk
);
1330 sock_put(&lsmc
->sk
); /* sock_hold in smc_tcp_listen_work */
1333 /* listen worker: finish in state connected */
1334 static void smc_listen_out_connected(struct smc_sock
*new_smc
)
1336 struct sock
*newsmcsk
= &new_smc
->sk
;
1338 sk_refcnt_debug_inc(newsmcsk
);
1339 if (newsmcsk
->sk_state
== SMC_INIT
)
1340 newsmcsk
->sk_state
= SMC_ACTIVE
;
1342 smc_listen_out(new_smc
);
1345 /* listen worker: finish in error state */
1346 static void smc_listen_out_err(struct smc_sock
*new_smc
)
1348 struct sock
*newsmcsk
= &new_smc
->sk
;
1349 struct net
*net
= sock_net(newsmcsk
);
1351 this_cpu_inc(net
->smc
.smc_stats
->srv_hshake_err_cnt
);
1352 if (newsmcsk
->sk_state
== SMC_INIT
)
1353 sock_put(&new_smc
->sk
); /* passive closing */
1354 newsmcsk
->sk_state
= SMC_CLOSED
;
1356 smc_listen_out(new_smc
);
1359 /* listen worker: decline and fall back if possible */
1360 static void smc_listen_decline(struct smc_sock
*new_smc
, int reason_code
,
1361 int local_first
, u8 version
)
1363 /* RDMA setup failed, switch back to TCP */
1364 smc_conn_abort(new_smc
, local_first
);
1365 if (reason_code
< 0) { /* error, no fallback possible */
1366 smc_listen_out_err(new_smc
);
1369 smc_switch_to_fallback(new_smc
, reason_code
);
1370 if (reason_code
&& reason_code
!= SMC_CLC_DECL_PEERDECL
) {
1371 if (smc_clc_send_decline(new_smc
, reason_code
, version
) < 0) {
1372 smc_listen_out_err(new_smc
);
1376 smc_listen_out_connected(new_smc
);
1379 /* listen worker: version checking */
1380 static int smc_listen_v2_check(struct smc_sock
*new_smc
,
1381 struct smc_clc_msg_proposal
*pclc
,
1382 struct smc_init_info
*ini
)
1384 struct smc_clc_smcd_v2_extension
*pclc_smcd_v2_ext
;
1385 struct smc_clc_v2_extension
*pclc_v2_ext
;
1386 int rc
= SMC_CLC_DECL_PEERNOSMC
;
1388 ini
->smc_type_v1
= pclc
->hdr
.typev1
;
1389 ini
->smc_type_v2
= pclc
->hdr
.typev2
;
1390 ini
->smcd_version
= ini
->smc_type_v1
!= SMC_TYPE_N
? SMC_V1
: 0;
1391 if (pclc
->hdr
.version
> SMC_V1
)
1392 ini
->smcd_version
|=
1393 ini
->smc_type_v2
!= SMC_TYPE_N
? SMC_V2
: 0;
1394 if (!(ini
->smcd_version
& SMC_V2
)) {
1395 rc
= SMC_CLC_DECL_PEERNOSMC
;
1398 if (!smc_ism_is_v2_capable()) {
1399 ini
->smcd_version
&= ~SMC_V2
;
1400 rc
= SMC_CLC_DECL_NOISM2SUPP
;
1403 pclc_v2_ext
= smc_get_clc_v2_ext(pclc
);
1405 ini
->smcd_version
&= ~SMC_V2
;
1406 rc
= SMC_CLC_DECL_NOV2EXT
;
1409 pclc_smcd_v2_ext
= smc_get_clc_smcd_v2_ext(pclc_v2_ext
);
1410 if (!pclc_smcd_v2_ext
) {
1411 ini
->smcd_version
&= ~SMC_V2
;
1412 rc
= SMC_CLC_DECL_NOV2DEXT
;
1416 if (!ini
->smcd_version
)
1422 /* listen worker: check prefixes */
1423 static int smc_listen_prfx_check(struct smc_sock
*new_smc
,
1424 struct smc_clc_msg_proposal
*pclc
)
1426 struct smc_clc_msg_proposal_prefix
*pclc_prfx
;
1427 struct socket
*newclcsock
= new_smc
->clcsock
;
1429 if (pclc
->hdr
.typev1
== SMC_TYPE_N
)
1431 pclc_prfx
= smc_clc_proposal_get_prefix(pclc
);
1432 if (smc_clc_prfx_match(newclcsock
, pclc_prfx
))
1433 return SMC_CLC_DECL_DIFFPREFIX
;
1438 /* listen worker: initialize connection and buffers */
1439 static int smc_listen_rdma_init(struct smc_sock
*new_smc
,
1440 struct smc_init_info
*ini
)
1444 /* allocate connection / link group */
1445 rc
= smc_conn_create(new_smc
, ini
);
1449 /* create send buffer and rmb */
1450 if (smc_buf_create(new_smc
, false))
1451 return SMC_CLC_DECL_MEM
;
1456 /* listen worker: initialize connection and buffers for SMC-D */
1457 static int smc_listen_ism_init(struct smc_sock
*new_smc
,
1458 struct smc_init_info
*ini
)
1462 rc
= smc_conn_create(new_smc
, ini
);
1466 /* Create send and receive buffers */
1467 rc
= smc_buf_create(new_smc
, true);
1469 smc_conn_abort(new_smc
, ini
->first_contact_local
);
1470 return (rc
== -ENOSPC
) ? SMC_CLC_DECL_MAX_DMB
:
1477 static bool smc_is_already_selected(struct smcd_dev
*smcd
,
1478 struct smc_init_info
*ini
,
1483 for (i
= 0; i
< matches
; i
++)
1484 if (smcd
== ini
->ism_dev
[i
])
1490 /* check for ISM devices matching proposed ISM devices */
1491 static void smc_check_ism_v2_match(struct smc_init_info
*ini
,
1492 u16 proposed_chid
, u64 proposed_gid
,
1493 unsigned int *matches
)
1495 struct smcd_dev
*smcd
;
1497 list_for_each_entry(smcd
, &smcd_dev_list
.list
, list
) {
1498 if (smcd
->going_away
)
1500 if (smc_is_already_selected(smcd
, ini
, *matches
))
1502 if (smc_ism_get_chid(smcd
) == proposed_chid
&&
1503 !smc_ism_cantalk(proposed_gid
, ISM_RESERVED_VLANID
, smcd
)) {
1504 ini
->ism_peer_gid
[*matches
] = proposed_gid
;
1505 ini
->ism_dev
[*matches
] = smcd
;
1512 static void smc_find_ism_store_rc(u32 rc
, struct smc_init_info
*ini
)
1518 static void smc_find_ism_v2_device_serv(struct smc_sock
*new_smc
,
1519 struct smc_clc_msg_proposal
*pclc
,
1520 struct smc_init_info
*ini
)
1522 struct smc_clc_smcd_v2_extension
*smcd_v2_ext
;
1523 struct smc_clc_v2_extension
*smc_v2_ext
;
1524 struct smc_clc_msg_smcd
*pclc_smcd
;
1525 unsigned int matches
= 0;
1530 if (!(ini
->smcd_version
& SMC_V2
) || !smcd_indicated(ini
->smc_type_v2
))
1533 pclc_smcd
= smc_get_clc_msg_smcd(pclc
);
1534 smc_v2_ext
= smc_get_clc_v2_ext(pclc
);
1535 smcd_v2_ext
= smc_get_clc_smcd_v2_ext(smc_v2_ext
);
1537 !smc_v2_ext
->hdr
.flag
.seid
) { /* no system EID support for SMCD */
1538 smc_find_ism_store_rc(SMC_CLC_DECL_NOSEID
, ini
);
1542 mutex_lock(&smcd_dev_list
.mutex
);
1543 if (pclc_smcd
->ism
.chid
)
1544 /* check for ISM device matching proposed native ISM device */
1545 smc_check_ism_v2_match(ini
, ntohs(pclc_smcd
->ism
.chid
),
1546 ntohll(pclc_smcd
->ism
.gid
), &matches
);
1547 for (i
= 1; i
<= smc_v2_ext
->hdr
.ism_gid_cnt
; i
++) {
1548 /* check for ISM devices matching proposed non-native ISM
1551 smc_check_ism_v2_match(ini
,
1552 ntohs(smcd_v2_ext
->gidchid
[i
- 1].chid
),
1553 ntohll(smcd_v2_ext
->gidchid
[i
- 1].gid
),
1556 mutex_unlock(&smcd_dev_list
.mutex
);
1558 if (ini
->ism_dev
[0]) {
1559 smc_ism_get_system_eid(ini
->ism_dev
[0], &eid
);
1560 if (memcmp(eid
, smcd_v2_ext
->system_eid
, SMC_MAX_EID_LEN
))
1566 /* separate - outside the smcd_dev_list.lock */
1567 smcd_version
= ini
->smcd_version
;
1568 for (i
= 0; i
< matches
; i
++) {
1569 ini
->smcd_version
= SMC_V2
;
1570 ini
->is_smcd
= true;
1571 ini
->ism_selected
= i
;
1572 rc
= smc_listen_ism_init(new_smc
, ini
);
1574 smc_find_ism_store_rc(rc
, ini
);
1575 /* try next active ISM device */
1578 return; /* matching and usable V2 ISM device found */
1580 /* no V2 ISM device could be initialized */
1581 ini
->smcd_version
= smcd_version
; /* restore original value */
1584 ini
->smcd_version
&= ~SMC_V2
;
1585 ini
->ism_dev
[0] = NULL
;
1586 ini
->is_smcd
= false;
1589 static void smc_find_ism_v1_device_serv(struct smc_sock
*new_smc
,
1590 struct smc_clc_msg_proposal
*pclc
,
1591 struct smc_init_info
*ini
)
1593 struct smc_clc_msg_smcd
*pclc_smcd
= smc_get_clc_msg_smcd(pclc
);
1596 /* check if ISM V1 is available */
1597 if (!(ini
->smcd_version
& SMC_V1
) || !smcd_indicated(ini
->smc_type_v1
))
1599 ini
->is_smcd
= true; /* prepare ISM check */
1600 ini
->ism_peer_gid
[0] = ntohll(pclc_smcd
->ism
.gid
);
1601 rc
= smc_find_ism_device(new_smc
, ini
);
1604 ini
->ism_selected
= 0;
1605 rc
= smc_listen_ism_init(new_smc
, ini
);
1607 return; /* V1 ISM device found */
1610 smc_find_ism_store_rc(rc
, ini
);
1611 ini
->ism_dev
[0] = NULL
;
1612 ini
->is_smcd
= false;
1615 /* listen worker: register buffers */
1616 static int smc_listen_rdma_reg(struct smc_sock
*new_smc
, bool local_first
)
1618 struct smc_connection
*conn
= &new_smc
->conn
;
1621 if (smcr_lgr_reg_rmbs(conn
->lnk
, conn
->rmb_desc
))
1622 return SMC_CLC_DECL_ERR_REGRMB
;
1624 smc_rmb_sync_sg_for_device(&new_smc
->conn
);
1629 static int smc_find_rdma_v1_device_serv(struct smc_sock
*new_smc
,
1630 struct smc_clc_msg_proposal
*pclc
,
1631 struct smc_init_info
*ini
)
1635 if (!smcr_indicated(ini
->smc_type_v1
))
1636 return SMC_CLC_DECL_NOSMCDEV
;
1638 /* prepare RDMA check */
1639 ini
->ib_lcl
= &pclc
->lcl
;
1640 rc
= smc_find_rdma_device(new_smc
, ini
);
1642 /* no RDMA device found */
1643 if (ini
->smc_type_v1
== SMC_TYPE_B
)
1644 /* neither ISM nor RDMA device found */
1645 rc
= SMC_CLC_DECL_NOSMCDEV
;
1648 rc
= smc_listen_rdma_init(new_smc
, ini
);
1651 return smc_listen_rdma_reg(new_smc
, ini
->first_contact_local
);
1654 /* determine the local device matching to proposal */
1655 static int smc_listen_find_device(struct smc_sock
*new_smc
,
1656 struct smc_clc_msg_proposal
*pclc
,
1657 struct smc_init_info
*ini
)
1661 /* check for ISM device matching V2 proposed device */
1662 smc_find_ism_v2_device_serv(new_smc
, pclc
, ini
);
1663 if (ini
->ism_dev
[0])
1666 if (!(ini
->smcd_version
& SMC_V1
))
1667 return ini
->rc
?: SMC_CLC_DECL_NOSMCD2DEV
;
1669 /* check for matching IP prefix and subnet length */
1670 rc
= smc_listen_prfx_check(new_smc
, pclc
);
1672 return ini
->rc
?: rc
;
1674 /* get vlan id from IP device */
1675 if (smc_vlan_by_tcpsk(new_smc
->clcsock
, ini
))
1676 return ini
->rc
?: SMC_CLC_DECL_GETVLANERR
;
1678 /* check for ISM device matching V1 proposed device */
1679 smc_find_ism_v1_device_serv(new_smc
, pclc
, ini
);
1680 if (ini
->ism_dev
[0])
1683 if (pclc
->hdr
.typev1
== SMC_TYPE_D
)
1684 /* skip RDMA and decline */
1685 return ini
->rc
?: SMC_CLC_DECL_NOSMCDDEV
;
1687 /* check if RDMA is available */
1688 rc
= smc_find_rdma_v1_device_serv(new_smc
, pclc
, ini
);
1689 smc_find_ism_store_rc(rc
, ini
);
1691 return (!rc
) ? 0 : ini
->rc
;
1694 /* listen worker: finish RDMA setup */
1695 static int smc_listen_rdma_finish(struct smc_sock
*new_smc
,
1696 struct smc_clc_msg_accept_confirm
*cclc
,
1699 struct smc_link
*link
= new_smc
->conn
.lnk
;
1700 int reason_code
= 0;
1703 smc_link_save_peer_info(link
, cclc
);
1705 if (smc_rmb_rtoken_handling(&new_smc
->conn
, link
, cclc
))
1706 return SMC_CLC_DECL_ERR_RTOK
;
1709 if (smc_ib_ready_link(link
))
1710 return SMC_CLC_DECL_ERR_RDYLNK
;
1711 /* QP confirmation over RoCE fabric */
1712 smc_llc_flow_initiate(link
->lgr
, SMC_LLC_FLOW_ADD_LINK
);
1713 reason_code
= smcr_serv_conf_first_link(new_smc
);
1714 smc_llc_flow_stop(link
->lgr
, &link
->lgr
->llc_flow_lcl
);
1719 /* setup for connection of server */
1720 static void smc_listen_work(struct work_struct
*work
)
1722 struct smc_sock
*new_smc
= container_of(work
, struct smc_sock
,
1724 u8 version
= smc_ism_is_v2_capable() ? SMC_V2
: SMC_V1
;
1725 struct socket
*newclcsock
= new_smc
->clcsock
;
1726 struct smc_clc_msg_accept_confirm
*cclc
;
1727 struct smc_clc_msg_proposal_area
*buf
;
1728 struct smc_clc_msg_proposal
*pclc
;
1729 struct smc_init_info
*ini
= NULL
;
1732 if (new_smc
->listen_smc
->sk
.sk_state
!= SMC_LISTEN
)
1733 return smc_listen_out_err(new_smc
);
1735 if (new_smc
->use_fallback
) {
1736 smc_listen_out_connected(new_smc
);
1740 /* check if peer is smc capable */
1741 if (!tcp_sk(newclcsock
->sk
)->syn_smc
) {
1742 smc_switch_to_fallback(new_smc
, SMC_CLC_DECL_PEERNOSMC
);
1743 smc_listen_out_connected(new_smc
);
1747 /* do inband token exchange -
1748 * wait for and receive SMC Proposal CLC message
1750 buf
= kzalloc(sizeof(*buf
), GFP_KERNEL
);
1752 rc
= SMC_CLC_DECL_MEM
;
1755 pclc
= (struct smc_clc_msg_proposal
*)buf
;
1756 rc
= smc_clc_wait_msg(new_smc
, pclc
, sizeof(*buf
),
1757 SMC_CLC_PROPOSAL
, CLC_WAIT_TIME
);
1760 version
= pclc
->hdr
.version
== SMC_V1
? SMC_V1
: version
;
1762 /* IPSec connections opt out of SMC optimizations */
1763 if (using_ipsec(new_smc
)) {
1764 rc
= SMC_CLC_DECL_IPSEC
;
1768 ini
= kzalloc(sizeof(*ini
), GFP_KERNEL
);
1770 rc
= SMC_CLC_DECL_MEM
;
1774 /* initial version checking */
1775 rc
= smc_listen_v2_check(new_smc
, pclc
, ini
);
1779 mutex_lock(&smc_server_lgr_pending
);
1780 smc_close_init(new_smc
);
1781 smc_rx_init(new_smc
);
1782 smc_tx_init(new_smc
);
1784 /* determine ISM or RoCE device used for connection */
1785 rc
= smc_listen_find_device(new_smc
, pclc
, ini
);
1789 /* send SMC Accept CLC message */
1790 rc
= smc_clc_send_accept(new_smc
, ini
->first_contact_local
,
1791 ini
->smcd_version
== SMC_V2
? SMC_V2
: SMC_V1
);
1795 /* SMC-D does not need this lock any more */
1797 mutex_unlock(&smc_server_lgr_pending
);
1799 /* receive SMC Confirm CLC message */
1800 memset(buf
, 0, sizeof(*buf
));
1801 cclc
= (struct smc_clc_msg_accept_confirm
*)buf
;
1802 rc
= smc_clc_wait_msg(new_smc
, cclc
, sizeof(*buf
),
1803 SMC_CLC_CONFIRM
, CLC_WAIT_TIME
);
1811 if (!ini
->is_smcd
) {
1812 rc
= smc_listen_rdma_finish(new_smc
, cclc
,
1813 ini
->first_contact_local
);
1816 mutex_unlock(&smc_server_lgr_pending
);
1818 smc_conn_save_peer_info(new_smc
, cclc
);
1819 smc_listen_out_connected(new_smc
);
1820 SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock
->sk
), ini
);
1824 mutex_unlock(&smc_server_lgr_pending
);
1826 smc_listen_decline(new_smc
, rc
, ini
? ini
->first_contact_local
: 0,
1833 static void smc_tcp_listen_work(struct work_struct
*work
)
1835 struct smc_sock
*lsmc
= container_of(work
, struct smc_sock
,
1837 struct sock
*lsk
= &lsmc
->sk
;
1838 struct smc_sock
*new_smc
;
1842 while (lsk
->sk_state
== SMC_LISTEN
) {
1843 rc
= smc_clcsock_accept(lsmc
, &new_smc
);
1844 if (rc
) /* clcsock accept queue empty or error */
1849 new_smc
->listen_smc
= lsmc
;
1850 new_smc
->use_fallback
= lsmc
->use_fallback
;
1851 new_smc
->fallback_rsn
= lsmc
->fallback_rsn
;
1852 sock_hold(lsk
); /* sock_put in smc_listen_work */
1853 INIT_WORK(&new_smc
->smc_listen_work
, smc_listen_work
);
1854 smc_copy_sock_settings_to_smc(new_smc
);
1855 new_smc
->sk
.sk_sndbuf
= lsmc
->sk
.sk_sndbuf
;
1856 new_smc
->sk
.sk_rcvbuf
= lsmc
->sk
.sk_rcvbuf
;
1857 sock_hold(&new_smc
->sk
); /* sock_put in passive closing */
1858 if (!queue_work(smc_hs_wq
, &new_smc
->smc_listen_work
))
1859 sock_put(&new_smc
->sk
);
1864 sock_put(&lsmc
->sk
); /* sock_hold in smc_clcsock_data_ready() */
1867 static void smc_clcsock_data_ready(struct sock
*listen_clcsock
)
1869 struct smc_sock
*lsmc
;
1871 lsmc
= (struct smc_sock
*)
1872 ((uintptr_t)listen_clcsock
->sk_user_data
& ~SK_USER_DATA_NOCOPY
);
1875 lsmc
->clcsk_data_ready(listen_clcsock
);
1876 if (lsmc
->sk
.sk_state
== SMC_LISTEN
) {
1877 sock_hold(&lsmc
->sk
); /* sock_put in smc_tcp_listen_work() */
1878 if (!queue_work(smc_hs_wq
, &lsmc
->tcp_listen_work
))
1879 sock_put(&lsmc
->sk
);
1883 static int smc_listen(struct socket
*sock
, int backlog
)
1885 struct sock
*sk
= sock
->sk
;
1886 struct smc_sock
*smc
;
1893 if ((sk
->sk_state
!= SMC_INIT
&& sk
->sk_state
!= SMC_LISTEN
) ||
1894 smc
->connect_nonblock
)
1898 if (sk
->sk_state
== SMC_LISTEN
) {
1899 sk
->sk_max_ack_backlog
= backlog
;
1902 /* some socket options are handled in core, so we could not apply
1903 * them to the clc socket -- copy smc socket options to clc socket
1905 smc_copy_sock_settings_to_clc(smc
);
1906 if (!smc
->use_fallback
)
1907 tcp_sk(smc
->clcsock
->sk
)->syn_smc
= 1;
1909 /* save original sk_data_ready function and establish
1910 * smc-specific sk_data_ready function
1912 smc
->clcsk_data_ready
= smc
->clcsock
->sk
->sk_data_ready
;
1913 smc
->clcsock
->sk
->sk_data_ready
= smc_clcsock_data_ready
;
1914 smc
->clcsock
->sk
->sk_user_data
=
1915 (void *)((uintptr_t)smc
| SK_USER_DATA_NOCOPY
);
1916 rc
= kernel_listen(smc
->clcsock
, backlog
);
1919 sk
->sk_max_ack_backlog
= backlog
;
1920 sk
->sk_ack_backlog
= 0;
1921 sk
->sk_state
= SMC_LISTEN
;
1928 static int smc_accept(struct socket
*sock
, struct socket
*new_sock
,
1929 int flags
, bool kern
)
1931 struct sock
*sk
= sock
->sk
, *nsk
;
1932 DECLARE_WAITQUEUE(wait
, current
);
1933 struct smc_sock
*lsmc
;
1938 sock_hold(sk
); /* sock_put below */
1941 if (lsmc
->sk
.sk_state
!= SMC_LISTEN
) {
1947 /* Wait for an incoming connection */
1948 timeo
= sock_rcvtimeo(sk
, flags
& O_NONBLOCK
);
1949 add_wait_queue_exclusive(sk_sleep(sk
), &wait
);
1950 while (!(nsk
= smc_accept_dequeue(sk
, new_sock
))) {
1951 set_current_state(TASK_INTERRUPTIBLE
);
1957 timeo
= schedule_timeout(timeo
);
1958 /* wakeup by sk_data_ready in smc_listen_work() */
1959 sched_annotate_sleep();
1961 if (signal_pending(current
)) {
1962 rc
= sock_intr_errno(timeo
);
1966 set_current_state(TASK_RUNNING
);
1967 remove_wait_queue(sk_sleep(sk
), &wait
);
1970 rc
= sock_error(nsk
);
1975 if (lsmc
->sockopt_defer_accept
&& !(flags
& O_NONBLOCK
)) {
1976 /* wait till data arrives on the socket */
1977 timeo
= msecs_to_jiffies(lsmc
->sockopt_defer_accept
*
1979 if (smc_sk(nsk
)->use_fallback
) {
1980 struct sock
*clcsk
= smc_sk(nsk
)->clcsock
->sk
;
1983 if (skb_queue_empty(&clcsk
->sk_receive_queue
))
1984 sk_wait_data(clcsk
, &timeo
, NULL
);
1985 release_sock(clcsk
);
1986 } else if (!atomic_read(&smc_sk(nsk
)->conn
.bytes_to_rcv
)) {
1988 smc_rx_wait(smc_sk(nsk
), &timeo
, smc_rx_data_available
);
1994 sock_put(sk
); /* sock_hold above */
1998 static int smc_getname(struct socket
*sock
, struct sockaddr
*addr
,
2001 struct smc_sock
*smc
;
2003 if (peer
&& (sock
->sk
->sk_state
!= SMC_ACTIVE
) &&
2004 (sock
->sk
->sk_state
!= SMC_APPCLOSEWAIT1
))
2007 smc
= smc_sk(sock
->sk
);
2009 return smc
->clcsock
->ops
->getname(smc
->clcsock
, addr
, peer
);
2012 static int smc_sendmsg(struct socket
*sock
, struct msghdr
*msg
, size_t len
)
2014 struct sock
*sk
= sock
->sk
;
2015 struct smc_sock
*smc
;
2020 if ((sk
->sk_state
!= SMC_ACTIVE
) &&
2021 (sk
->sk_state
!= SMC_APPCLOSEWAIT1
) &&
2022 (sk
->sk_state
!= SMC_INIT
))
2025 if (msg
->msg_flags
& MSG_FASTOPEN
) {
2026 if (sk
->sk_state
== SMC_INIT
&& !smc
->connect_nonblock
) {
2027 smc_switch_to_fallback(smc
, SMC_CLC_DECL_OPTUNSUPP
);
2034 if (smc
->use_fallback
) {
2035 rc
= smc
->clcsock
->ops
->sendmsg(smc
->clcsock
, msg
, len
);
2037 rc
= smc_tx_sendmsg(smc
, msg
, len
);
2038 SMC_STAT_TX_PAYLOAD(smc
, len
, rc
);
2045 static int smc_recvmsg(struct socket
*sock
, struct msghdr
*msg
, size_t len
,
2048 struct sock
*sk
= sock
->sk
;
2049 struct smc_sock
*smc
;
2054 if (sk
->sk_state
== SMC_CLOSED
&& (sk
->sk_shutdown
& RCV_SHUTDOWN
)) {
2055 /* socket was connected before, no more data to read */
2059 if ((sk
->sk_state
== SMC_INIT
) ||
2060 (sk
->sk_state
== SMC_LISTEN
) ||
2061 (sk
->sk_state
== SMC_CLOSED
))
2064 if (sk
->sk_state
== SMC_PEERFINCLOSEWAIT
) {
2069 if (smc
->use_fallback
) {
2070 rc
= smc
->clcsock
->ops
->recvmsg(smc
->clcsock
, msg
, len
, flags
);
2072 msg
->msg_namelen
= 0;
2073 rc
= smc_rx_recvmsg(smc
, msg
, NULL
, len
, flags
);
2074 SMC_STAT_RX_PAYLOAD(smc
, rc
, rc
);
2082 static __poll_t
smc_accept_poll(struct sock
*parent
)
2084 struct smc_sock
*isk
= smc_sk(parent
);
2087 spin_lock(&isk
->accept_q_lock
);
2088 if (!list_empty(&isk
->accept_q
))
2089 mask
= EPOLLIN
| EPOLLRDNORM
;
2090 spin_unlock(&isk
->accept_q_lock
);
2095 static __poll_t
smc_poll(struct file
*file
, struct socket
*sock
,
2098 struct sock
*sk
= sock
->sk
;
2099 struct smc_sock
*smc
;
2105 smc
= smc_sk(sock
->sk
);
2106 if (smc
->use_fallback
) {
2107 /* delegate to CLC child sock */
2108 mask
= smc
->clcsock
->ops
->poll(file
, smc
->clcsock
, wait
);
2109 sk
->sk_err
= smc
->clcsock
->sk
->sk_err
;
2111 if (sk
->sk_state
!= SMC_CLOSED
)
2112 sock_poll_wait(file
, sock
, wait
);
2115 if ((sk
->sk_shutdown
== SHUTDOWN_MASK
) ||
2116 (sk
->sk_state
== SMC_CLOSED
))
2118 if (sk
->sk_state
== SMC_LISTEN
) {
2119 /* woken up by sk_data_ready in smc_listen_work() */
2120 mask
|= smc_accept_poll(sk
);
2121 } else if (smc
->use_fallback
) { /* as result of connect_work()*/
2122 mask
|= smc
->clcsock
->ops
->poll(file
, smc
->clcsock
,
2124 sk
->sk_err
= smc
->clcsock
->sk
->sk_err
;
2126 if ((sk
->sk_state
!= SMC_INIT
&&
2127 atomic_read(&smc
->conn
.sndbuf_space
)) ||
2128 sk
->sk_shutdown
& SEND_SHUTDOWN
) {
2129 mask
|= EPOLLOUT
| EPOLLWRNORM
;
2131 sk_set_bit(SOCKWQ_ASYNC_NOSPACE
, sk
);
2132 set_bit(SOCK_NOSPACE
, &sk
->sk_socket
->flags
);
2134 if (atomic_read(&smc
->conn
.bytes_to_rcv
))
2135 mask
|= EPOLLIN
| EPOLLRDNORM
;
2136 if (sk
->sk_shutdown
& RCV_SHUTDOWN
)
2137 mask
|= EPOLLIN
| EPOLLRDNORM
| EPOLLRDHUP
;
2138 if (sk
->sk_state
== SMC_APPCLOSEWAIT1
)
2140 if (smc
->conn
.urg_state
== SMC_URG_VALID
)
2148 static int smc_shutdown(struct socket
*sock
, int how
)
2150 struct sock
*sk
= sock
->sk
;
2151 struct smc_sock
*smc
;
2157 if ((how
< SHUT_RD
) || (how
> SHUT_RDWR
))
2163 if ((sk
->sk_state
!= SMC_ACTIVE
) &&
2164 (sk
->sk_state
!= SMC_PEERCLOSEWAIT1
) &&
2165 (sk
->sk_state
!= SMC_PEERCLOSEWAIT2
) &&
2166 (sk
->sk_state
!= SMC_APPCLOSEWAIT1
) &&
2167 (sk
->sk_state
!= SMC_APPCLOSEWAIT2
) &&
2168 (sk
->sk_state
!= SMC_APPFINCLOSEWAIT
))
2170 if (smc
->use_fallback
) {
2171 rc
= kernel_sock_shutdown(smc
->clcsock
, how
);
2172 sk
->sk_shutdown
= smc
->clcsock
->sk
->sk_shutdown
;
2173 if (sk
->sk_shutdown
== SHUTDOWN_MASK
)
2174 sk
->sk_state
= SMC_CLOSED
;
2178 case SHUT_RDWR
: /* shutdown in both directions */
2179 rc
= smc_close_active(smc
);
2182 rc
= smc_close_shutdown_write(smc
);
2186 /* nothing more to do because peer is not involved */
2190 rc1
= kernel_sock_shutdown(smc
->clcsock
, how
);
2191 /* map sock_shutdown_cmd constants to sk_shutdown value range */
2192 sk
->sk_shutdown
|= how
+ 1;
2196 return rc
? rc
: rc1
;
2199 static int smc_setsockopt(struct socket
*sock
, int level
, int optname
,
2200 sockptr_t optval
, unsigned int optlen
)
2202 struct sock
*sk
= sock
->sk
;
2203 struct smc_sock
*smc
;
2206 if (level
== SOL_TCP
&& optname
== TCP_ULP
)
2211 /* generic setsockopts reaching us here always apply to the
2214 if (unlikely(!smc
->clcsock
->ops
->setsockopt
))
2217 rc
= smc
->clcsock
->ops
->setsockopt(smc
->clcsock
, level
, optname
,
2219 if (smc
->clcsock
->sk
->sk_err
) {
2220 sk
->sk_err
= smc
->clcsock
->sk
->sk_err
;
2221 sk_error_report(sk
);
2224 if (optlen
< sizeof(int))
2226 if (copy_from_sockptr(&val
, optval
, sizeof(int)))
2230 if (rc
|| smc
->use_fallback
)
2234 case TCP_FASTOPEN_CONNECT
:
2235 case TCP_FASTOPEN_KEY
:
2236 case TCP_FASTOPEN_NO_COOKIE
:
2237 /* option not supported by SMC */
2238 if (sk
->sk_state
== SMC_INIT
&& !smc
->connect_nonblock
) {
2239 smc_switch_to_fallback(smc
, SMC_CLC_DECL_OPTUNSUPP
);
2245 if (sk
->sk_state
!= SMC_INIT
&&
2246 sk
->sk_state
!= SMC_LISTEN
&&
2247 sk
->sk_state
!= SMC_CLOSED
) {
2249 SMC_STAT_INC(smc
, ndly_cnt
);
2250 mod_delayed_work(smc
->conn
.lgr
->tx_wq
,
2251 &smc
->conn
.tx_work
, 0);
2256 if (sk
->sk_state
!= SMC_INIT
&&
2257 sk
->sk_state
!= SMC_LISTEN
&&
2258 sk
->sk_state
!= SMC_CLOSED
) {
2260 SMC_STAT_INC(smc
, cork_cnt
);
2261 mod_delayed_work(smc
->conn
.lgr
->tx_wq
,
2262 &smc
->conn
.tx_work
, 0);
2266 case TCP_DEFER_ACCEPT
:
2267 smc
->sockopt_defer_accept
= val
;
2278 static int smc_getsockopt(struct socket
*sock
, int level
, int optname
,
2279 char __user
*optval
, int __user
*optlen
)
2281 struct smc_sock
*smc
;
2283 smc
= smc_sk(sock
->sk
);
2284 /* socket options apply to the CLC socket */
2285 if (unlikely(!smc
->clcsock
->ops
->getsockopt
))
2287 return smc
->clcsock
->ops
->getsockopt(smc
->clcsock
, level
, optname
,
2291 static int smc_ioctl(struct socket
*sock
, unsigned int cmd
,
2294 union smc_host_cursor cons
, urg
;
2295 struct smc_connection
*conn
;
2296 struct smc_sock
*smc
;
2299 smc
= smc_sk(sock
->sk
);
2301 lock_sock(&smc
->sk
);
2302 if (smc
->use_fallback
) {
2303 if (!smc
->clcsock
) {
2304 release_sock(&smc
->sk
);
2307 answ
= smc
->clcsock
->ops
->ioctl(smc
->clcsock
, cmd
, arg
);
2308 release_sock(&smc
->sk
);
2312 case SIOCINQ
: /* same as FIONREAD */
2313 if (smc
->sk
.sk_state
== SMC_LISTEN
) {
2314 release_sock(&smc
->sk
);
2317 if (smc
->sk
.sk_state
== SMC_INIT
||
2318 smc
->sk
.sk_state
== SMC_CLOSED
)
2321 answ
= atomic_read(&smc
->conn
.bytes_to_rcv
);
2324 /* output queue size (not send + not acked) */
2325 if (smc
->sk
.sk_state
== SMC_LISTEN
) {
2326 release_sock(&smc
->sk
);
2329 if (smc
->sk
.sk_state
== SMC_INIT
||
2330 smc
->sk
.sk_state
== SMC_CLOSED
)
2333 answ
= smc
->conn
.sndbuf_desc
->len
-
2334 atomic_read(&smc
->conn
.sndbuf_space
);
2337 /* output queue size (not send only) */
2338 if (smc
->sk
.sk_state
== SMC_LISTEN
) {
2339 release_sock(&smc
->sk
);
2342 if (smc
->sk
.sk_state
== SMC_INIT
||
2343 smc
->sk
.sk_state
== SMC_CLOSED
)
2346 answ
= smc_tx_prepared_sends(&smc
->conn
);
2349 if (smc
->sk
.sk_state
== SMC_LISTEN
) {
2350 release_sock(&smc
->sk
);
2353 if (smc
->sk
.sk_state
== SMC_INIT
||
2354 smc
->sk
.sk_state
== SMC_CLOSED
) {
2357 smc_curs_copy(&cons
, &conn
->local_tx_ctrl
.cons
, conn
);
2358 smc_curs_copy(&urg
, &conn
->urg_curs
, conn
);
2359 answ
= smc_curs_diff(conn
->rmb_desc
->len
,
2364 release_sock(&smc
->sk
);
2365 return -ENOIOCTLCMD
;
2367 release_sock(&smc
->sk
);
2369 return put_user(answ
, (int __user
*)arg
);
2372 static ssize_t
smc_sendpage(struct socket
*sock
, struct page
*page
,
2373 int offset
, size_t size
, int flags
)
2375 struct sock
*sk
= sock
->sk
;
2376 struct smc_sock
*smc
;
2381 if (sk
->sk_state
!= SMC_ACTIVE
) {
2386 if (smc
->use_fallback
) {
2387 rc
= kernel_sendpage(smc
->clcsock
, page
, offset
,
2390 SMC_STAT_INC(smc
, sendpage_cnt
);
2391 rc
= sock_no_sendpage(sock
, page
, offset
, size
, flags
);
2398 /* Map the affected portions of the rmbe into an spd, note the number of bytes
2399 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
2400 * updates till whenever a respective page has been fully processed.
2401 * Note that subsequent recv() calls have to wait till all splice() processing
2404 static ssize_t
smc_splice_read(struct socket
*sock
, loff_t
*ppos
,
2405 struct pipe_inode_info
*pipe
, size_t len
,
2408 struct sock
*sk
= sock
->sk
;
2409 struct smc_sock
*smc
;
2414 if (sk
->sk_state
== SMC_CLOSED
&& (sk
->sk_shutdown
& RCV_SHUTDOWN
)) {
2415 /* socket was connected before, no more data to read */
2419 if (sk
->sk_state
== SMC_INIT
||
2420 sk
->sk_state
== SMC_LISTEN
||
2421 sk
->sk_state
== SMC_CLOSED
)
2424 if (sk
->sk_state
== SMC_PEERFINCLOSEWAIT
) {
2429 if (smc
->use_fallback
) {
2430 rc
= smc
->clcsock
->ops
->splice_read(smc
->clcsock
, ppos
,
2437 if (flags
& SPLICE_F_NONBLOCK
)
2438 flags
= MSG_DONTWAIT
;
2441 SMC_STAT_INC(smc
, splice_cnt
);
2442 rc
= smc_rx_recvmsg(smc
, NULL
, pipe
, len
, flags
);
2450 /* must look like tcp */
2451 static const struct proto_ops smc_sock_ops
= {
2453 .owner
= THIS_MODULE
,
2454 .release
= smc_release
,
2456 .connect
= smc_connect
,
2457 .socketpair
= sock_no_socketpair
,
2458 .accept
= smc_accept
,
2459 .getname
= smc_getname
,
2462 .listen
= smc_listen
,
2463 .shutdown
= smc_shutdown
,
2464 .setsockopt
= smc_setsockopt
,
2465 .getsockopt
= smc_getsockopt
,
2466 .sendmsg
= smc_sendmsg
,
2467 .recvmsg
= smc_recvmsg
,
2468 .mmap
= sock_no_mmap
,
2469 .sendpage
= smc_sendpage
,
2470 .splice_read
= smc_splice_read
,
2473 static int smc_create(struct net
*net
, struct socket
*sock
, int protocol
,
2476 int family
= (protocol
== SMCPROTO_SMC6
) ? PF_INET6
: PF_INET
;
2477 struct smc_sock
*smc
;
2481 rc
= -ESOCKTNOSUPPORT
;
2482 if (sock
->type
!= SOCK_STREAM
)
2485 rc
= -EPROTONOSUPPORT
;
2486 if (protocol
!= SMCPROTO_SMC
&& protocol
!= SMCPROTO_SMC6
)
2490 sock
->ops
= &smc_sock_ops
;
2491 sk
= smc_sock_alloc(net
, sock
, protocol
);
2495 /* create internal TCP socket for CLC handshake and fallback */
2497 smc
->use_fallback
= false; /* assume rdma capability first */
2498 smc
->fallback_rsn
= 0;
2499 rc
= sock_create_kern(net
, family
, SOCK_STREAM
, IPPROTO_TCP
,
2502 sk_common_release(sk
);
2505 smc
->sk
.sk_sndbuf
= max(smc
->clcsock
->sk
->sk_sndbuf
, SMC_BUF_MIN_SIZE
);
2506 smc
->sk
.sk_rcvbuf
= max(smc
->clcsock
->sk
->sk_rcvbuf
, SMC_BUF_MIN_SIZE
);
2512 static const struct net_proto_family smc_sock_family_ops
= {
2514 .owner
= THIS_MODULE
,
2515 .create
= smc_create
,
2518 unsigned int smc_net_id
;
2520 static __net_init
int smc_net_init(struct net
*net
)
2522 return smc_pnet_net_init(net
);
2525 static void __net_exit
smc_net_exit(struct net
*net
)
2527 smc_pnet_net_exit(net
);
2530 static __net_init
int smc_net_stat_init(struct net
*net
)
2532 return smc_stats_init(net
);
2535 static void __net_exit
smc_net_stat_exit(struct net
*net
)
2537 smc_stats_exit(net
);
2540 static struct pernet_operations smc_net_ops
= {
2541 .init
= smc_net_init
,
2542 .exit
= smc_net_exit
,
2544 .size
= sizeof(struct smc_net
),
2547 static struct pernet_operations smc_net_stat_ops
= {
2548 .init
= smc_net_stat_init
,
2549 .exit
= smc_net_stat_exit
,
2552 static int __init
smc_init(void)
2556 rc
= register_pernet_subsys(&smc_net_ops
);
2560 rc
= register_pernet_subsys(&smc_net_stat_ops
);
2569 goto out_pernet_subsys
;
2571 rc
= smc_pnet_init();
2576 smc_hs_wq
= alloc_workqueue("smc_hs_wq", 0, 0);
2580 smc_close_wq
= alloc_workqueue("smc_close_wq", 0, 0);
2582 goto out_alloc_hs_wq
;
2584 rc
= smc_core_init();
2586 pr_err("%s: smc_core_init fails with %d\n", __func__
, rc
);
2590 rc
= smc_llc_init();
2592 pr_err("%s: smc_llc_init fails with %d\n", __func__
, rc
);
2596 rc
= smc_cdc_init();
2598 pr_err("%s: smc_cdc_init fails with %d\n", __func__
, rc
);
2602 rc
= proto_register(&smc_proto
, 1);
2604 pr_err("%s: proto_register(v4) fails with %d\n", __func__
, rc
);
2608 rc
= proto_register(&smc_proto6
, 1);
2610 pr_err("%s: proto_register(v6) fails with %d\n", __func__
, rc
);
2614 rc
= sock_register(&smc_sock_family_ops
);
2616 pr_err("%s: sock_register fails with %d\n", __func__
, rc
);
2619 INIT_HLIST_HEAD(&smc_v4_hashinfo
.ht
);
2620 INIT_HLIST_HEAD(&smc_v6_hashinfo
.ht
);
2622 rc
= smc_ib_register_client();
2624 pr_err("%s: ib_register fails with %d\n", __func__
, rc
);
2628 static_branch_enable(&tcp_have_smc
);
2632 sock_unregister(PF_SMC
);
2634 proto_unregister(&smc_proto6
);
2636 proto_unregister(&smc_proto
);
2640 destroy_workqueue(smc_close_wq
);
2642 destroy_workqueue(smc_hs_wq
);
2648 unregister_pernet_subsys(&smc_net_ops
);
2653 static void __exit
smc_exit(void)
2655 static_branch_disable(&tcp_have_smc
);
2656 sock_unregister(PF_SMC
);
2658 smc_ib_unregister_client();
2659 destroy_workqueue(smc_close_wq
);
2660 destroy_workqueue(smc_hs_wq
);
2661 proto_unregister(&smc_proto6
);
2662 proto_unregister(&smc_proto
);
2665 unregister_pernet_subsys(&smc_net_stat_ops
);
2666 unregister_pernet_subsys(&smc_net_ops
);
2670 module_init(smc_init
);
2671 module_exit(smc_exit
);
2673 MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2674 MODULE_DESCRIPTION("smc socket address family");
2675 MODULE_LICENSE("GPL");
2676 MODULE_ALIAS_NETPROTO(PF_SMC
);