]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - net/smc/af_smc.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net: Fix double 0x prefix print in SKB dump
[mirror_ubuntu-jammy-kernel.git] / net / smc / af_smc.c
CommitLineData
09c434b8 1// SPDX-License-Identifier: GPL-2.0-only
ac713874
UB		 2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
6 * applies to SOCK_STREAM sockets only
7 * offers an alternative communication option for TCP-protocol sockets
8 * applicable with RoCE-cards only
9 *
a046d57d 10 * Initial restrictions:
a046d57d 11 * - support for alternate links postponed
a046d57d 12 *
aaa4d33f 13 * Copyright IBM Corp. 2016, 2018
ac713874
UB		 14 *
15 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
16 * based on prototype from Frank Blaschka
17 */
18
19#define KMSG_COMPONENT "smc"
20#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21
22#include <linux/module.h>
23#include <linux/socket.h>
a046d57d 24#include <linux/workqueue.h>
5f08318f 25#include <linux/in.h>
c3edc401 26#include <linux/sched/signal.h>
41349844 27#include <linux/if_vlan.h>
4ead9c96 28#include <linux/rcupdate_wait.h>
b81a5eb7 29#include <linux/ctype.h>
c3edc401 30
ac713874 31#include <net/sock.h>
a046d57d 32#include <net/tcp.h>
f16a7dd5 33#include <net/smc.h>
9b67e26f 34#include <asm/ioctls.h>
ac713874 35
64e28b52
HW		 36#include <net/net_namespace.h>
37#include <net/netns/generic.h>
38#include "smc_netns.h"
39
ac713874 40#include "smc.h"
a046d57d 41#include "smc_clc.h"
9bf9abea 42#include "smc_llc.h"
5f08318f 43#include "smc_cdc.h"
0cfdd8f9 44#include "smc_core.h"
a4cf0443 45#include "smc_ib.h"
41349844 46#include "smc_ism.h"
6812baab 47#include "smc_pnet.h"
e8372d9d 48#include "smc_netlink.h"
e6727f39 49#include "smc_tx.h"
952310cc 50#include "smc_rx.h"
b38d7324 51#include "smc_close.h"
e0e4b8fa 52#include "smc_stats.h"
ac713874 53
72a36a8a
HW		 54static DEFINE_MUTEX(smc_server_lgr_pending); /* serialize link group
55 * creation on server
56 */
57static DEFINE_MUTEX(smc_client_lgr_pending); /* serialize link group
58 * creation on client
0cfdd8f9
UB		 59 */
60
22ef473d
KG		 61struct workqueue_struct *smc_hs_wq; /* wq for handshake work */
62struct workqueue_struct *smc_close_wq; /* wq for close work */
63
a046d57d 64static void smc_tcp_listen_work(struct work_struct *);
24ac3a08 65static void smc_connect_work(struct work_struct *);
a046d57d 66
ac713874
UB		 67static void smc_set_keepalive(struct sock *sk, int val)
68{
69 struct smc_sock *smc = smc_sk(sk);
70
71 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
72}
73
f16a7dd5
UB		 74static struct smc_hashinfo smc_v4_hashinfo = {
75 .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
76};
77
aaa4d33f
KG		 78static struct smc_hashinfo smc_v6_hashinfo = {
79 .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
80};
81
f16a7dd5
UB		 82int smc_hash_sk(struct sock *sk)
83{
84 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
85 struct hlist_head *head;
86
87 head = &h->ht;
88
89 write_lock_bh(&h->lock);
90 sk_add_node(sk, head);
91 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
92 write_unlock_bh(&h->lock);
93
94 return 0;
95}
96EXPORT_SYMBOL_GPL(smc_hash_sk);
97
98void smc_unhash_sk(struct sock *sk)
99{
100 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
101
102 write_lock_bh(&h->lock);
103 if (sk_del_node_init(sk))
104 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
105 write_unlock_bh(&h->lock);
106}
107EXPORT_SYMBOL_GPL(smc_unhash_sk);
108
109struct proto smc_proto = {
ac713874
UB		 110 .name = "SMC",
111 .owner = THIS_MODULE,
112 .keepalive = smc_set_keepalive,
f16a7dd5
UB		 113 .hash = smc_hash_sk,
114 .unhash = smc_unhash_sk,
ac713874 115 .obj_size = sizeof(struct smc_sock),
f16a7dd5 116 .h.smc_hash = &smc_v4_hashinfo,
5f0d5a3a 117 .slab_flags = SLAB_TYPESAFE_BY_RCU,
ac713874 118};
f16a7dd5 119EXPORT_SYMBOL_GPL(smc_proto);
ac713874 120
aaa4d33f
KG		 121struct proto smc_proto6 = {
122 .name = "SMC6",
123 .owner = THIS_MODULE,
124 .keepalive = smc_set_keepalive,
125 .hash = smc_hash_sk,
126 .unhash = smc_unhash_sk,
127 .obj_size = sizeof(struct smc_sock),
128 .h.smc_hash = &smc_v6_hashinfo,
129 .slab_flags = SLAB_TYPESAFE_BY_RCU,
130};
131EXPORT_SYMBOL_GPL(smc_proto6);
132
f536dffc
UB		 133static void smc_restore_fallback_changes(struct smc_sock *smc)
134{
1ad24058
KG		 135 if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
136 smc->clcsock->file->private_data = smc->sk.sk_socket;
137 smc->clcsock->file = NULL;
138 }
f536dffc
UB		 139}
140
39f41f36 141static int __smc_release(struct smc_sock *smc)
ac713874 142{
39f41f36 143 struct sock *sk = &smc->sk;
b38d7324 144 int rc = 0;
ac713874 145
51f1de79 146 if (!smc->use_fallback) {
b38d7324
UB		 147 rc = smc_close_active(smc);
148 sock_set_flag(sk, SOCK_DEAD);
149 sk->sk_shutdown |= SHUTDOWN_MASK;
b03faa1f 150 } else {
0cad438e
DL		 151 if (sk->sk_state != SMC_CLOSED) {
152 if (sk->sk_state != SMC_LISTEN &&
153 sk->sk_state != SMC_INIT)
154 sock_put(sk); /* passive closing */
155 if (sk->sk_state == SMC_LISTEN) {
156 /* wake up clcsock accept */
157 rc = kernel_sock_shutdown(smc->clcsock,
158 SHUT_RDWR);
159 }
160 sk->sk_state = SMC_CLOSED;
161 sk->sk_state_change(sk);
78abe3d0 162 }
f536dffc 163 smc_restore_fallback_changes(smc);
51f1de79 164 }
ac713874 165
b03faa1f
UB		 166 sk->sk_prot->unhash(sk);
167
168 if (sk->sk_state == SMC_CLOSED) {
169 if (smc->clcsock) {
fd57770d
KG		 170 release_sock(sk);
171 smc_clcsock_release(smc);
172 lock_sock(sk);
b03faa1f
UB		 173 }
174 if (!smc->use_fallback)
175 smc_conn_free(&smc->conn);
176 }
177
39f41f36
UB		 178 return rc;
179}
180
181static int smc_release(struct socket *sock)
182{
183 struct sock *sk = sock->sk;
184 struct smc_sock *smc;
185 int rc = 0;
186
187 if (!sk)
188 goto out;
189
81cf4f47 190 sock_hold(sk); /* sock_put below */
39f41f36
UB		 191 smc = smc_sk(sk);
192
193 /* cleanup for a dangling non-blocking connect */
194 if (smc->connect_nonblock && sk->sk_state == SMC_INIT)
195 tcp_abort(smc->clcsock->sk, ECONNABORTED);
196 flush_work(&smc->connect_work);
197
198 if (sk->sk_state == SMC_LISTEN)
199 /* smc_close_non_accepted() is called and acquires
200 * sock lock for child sockets again
201 */
202 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
203 else
204 lock_sock(sk);
205
206 rc = __smc_release(smc);
207
ac713874
UB		 208 /* detach socket */
209 sock_orphan(sk);
210 sock->sk = NULL;
211 release_sock(sk);
212
81cf4f47 213 sock_put(sk); /* sock_hold above */
51f1de79 214 sock_put(sk); /* final sock_put */
ac713874 215out:
b38d7324 216 return rc;
ac713874
UB		 217}
218
219static void smc_destruct(struct sock *sk)
220{
221 if (sk->sk_state != SMC_CLOSED)
222 return;
223 if (!sock_flag(sk, SOCK_DEAD))
224 return;
225
226 sk_refcnt_debug_dec(sk);
227}
228
aaa4d33f
KG		 229static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
230 int protocol)
ac713874
UB		 231{
232 struct smc_sock *smc;
aaa4d33f 233 struct proto *prot;
ac713874
UB		 234 struct sock *sk;
235
aaa4d33f
KG		 236 prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
237 sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
ac713874
UB		 238 if (!sk)
239 return NULL;
240
241 sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
242 sk->sk_state = SMC_INIT;
243 sk->sk_destruct = smc_destruct;
aaa4d33f 244 sk->sk_protocol = protocol;
ac713874 245 smc = smc_sk(sk);
a046d57d 246 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
24ac3a08 247 INIT_WORK(&smc->connect_work, smc_connect_work);
be7f3e59 248 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
a046d57d
UB		 249 INIT_LIST_HEAD(&smc->accept_q);
250 spin_lock_init(&smc->accept_q_lock);
be7f3e59 251 spin_lock_init(&smc->conn.send_lock);
f16a7dd5 252 sk->sk_prot->hash(sk);
a046d57d 253 sk_refcnt_debug_inc(sk);
78abe3d0 254 mutex_init(&smc->clcsock_release_lock);
ac713874
UB		 255
256 return sk;
257}
258
259static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
260 int addr_len)
261{
262 struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
263 struct sock *sk = sock->sk;
264 struct smc_sock *smc;
265 int rc;
266
267 smc = smc_sk(sk);
268
269 /* replicate tests from inet_bind(), to be safe wrt. future changes */
270 rc = -EINVAL;
271 if (addr_len < sizeof(struct sockaddr_in))
272 goto out;
273
274 rc = -EAFNOSUPPORT;
aaa4d33f
KG		 275 if (addr->sin_family != AF_INET &&
276 addr->sin_family != AF_INET6 &&
277 addr->sin_family != AF_UNSPEC)
278 goto out;
ac713874 279 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
aaa4d33f
KG		 280 if (addr->sin_family == AF_UNSPEC &&
281 addr->sin_addr.s_addr != htonl(INADDR_ANY))
ac713874
UB		 282 goto out;
283
284 lock_sock(sk);
285
286 /* Check if socket is already active */
287 rc = -EINVAL;
cd206360 288 if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
ac713874
UB		 289 goto out_rel;
290
291 smc->clcsock->sk->sk_reuse = sk->sk_reuse;
292 rc = kernel_bind(smc->clcsock, uaddr, addr_len);
293
294out_rel:
295 release_sock(sk);
296out:
297 return rc;
298}
299
300static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
301 unsigned long mask)
302{
303 /* options we don't get control via setsockopt for */
304 nsk->sk_type = osk->sk_type;
305 nsk->sk_sndbuf = osk->sk_sndbuf;
306 nsk->sk_rcvbuf = osk->sk_rcvbuf;
307 nsk->sk_sndtimeo = osk->sk_sndtimeo;
308 nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
309 nsk->sk_mark = osk->sk_mark;
310 nsk->sk_priority = osk->sk_priority;
311 nsk->sk_rcvlowat = osk->sk_rcvlowat;
312 nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
313 nsk->sk_err = osk->sk_err;
314
315 nsk->sk_flags &= ~mask;
316 nsk->sk_flags |= osk->sk_flags & mask;
317}
318
319#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
320 (1UL << SOCK_KEEPOPEN) | \
321 (1UL << SOCK_LINGER) | \
322 (1UL << SOCK_BROADCAST) | \
323 (1UL << SOCK_TIMESTAMP) | \
324 (1UL << SOCK_DBG) | \
325 (1UL << SOCK_RCVTSTAMP) | \
326 (1UL << SOCK_RCVTSTAMPNS) | \
327 (1UL << SOCK_LOCALROUTE) | \
328 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
329 (1UL << SOCK_RXQ_OVFL) | \
330 (1UL << SOCK_WIFI_STATUS) | \
331 (1UL << SOCK_NOFCS) | \
9718475e
DD		 332 (1UL << SOCK_FILTER_LOCKED) | \
333 (1UL << SOCK_TSTAMP_NEW))
ac713874
UB		 334/* copy only relevant settings and flags of SOL_SOCKET level from smc to
335 * clc socket (since smc is not called for these options from net/core)
336 */
337static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
338{
339 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
340}
341
342#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
343 (1UL << SOCK_KEEPOPEN) | \
344 (1UL << SOCK_LINGER) | \
345 (1UL << SOCK_DBG))
346/* copy only settings and flags relevant for smc from clc to smc socket */
347static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
348{
349 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
350}
351
b9247544 352/* register the new rmb on all links */
7562a13d 353static int smcr_lgr_reg_rmbs(struct smc_link *link,
b9247544
KG		 354 struct smc_buf_desc *rmb_desc)
355{
7562a13d
KG		 356 struct smc_link_group *lgr = link->lgr;
357 int i, rc = 0;
b9247544 358
d5500667
KG		 359 rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
360 if (rc)
361 return rc;
362 /* protect against parallel smc_llc_cli_rkey_exchange() and
363 * parallel smcr_link_reg_rmb()
364 */
365 mutex_lock(&lgr->llc_conf_mutex);
b9247544 366 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
741a49a4 367 if (!smc_link_active(&lgr->lnk[i]))
b9247544 368 continue;
7562a13d 369 rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc);
b9247544 370 if (rc)
7562a13d 371 goto out;
44aa81ce 372 }
7562a13d
KG		 373
374 /* exchange confirm_rkey msg with peer */
375 rc = smc_llc_do_confirm_rkey(link, rmb_desc);
376 if (rc) {
377 rc = -EFAULT;
378 goto out;
379 }
380 rmb_desc->is_conf_rkey = true;
381out:
d5500667
KG		 382 mutex_unlock(&lgr->llc_conf_mutex);
383 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
7562a13d 384 return rc;
e63a5f8c
KG		 385}
386
b9247544 387static int smcr_clnt_conf_first_link(struct smc_sock *smc)
9bf9abea 388{
387707fd 389 struct smc_link *link = smc->conn.lnk;
0fb0b02b 390 struct smc_llc_qentry *qentry;
9bf9abea
UB		 391 int rc;
392
9bf9abea 393 /* receive CONFIRM LINK request from server over RoCE fabric */
0fb0b02b
KG		 394 qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
395 SMC_LLC_CONFIRM_LINK);
396 if (!qentry) {
9bf9abea
UB		 397 struct smc_clc_msg_decline dclc;
398
399 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
2b59f58e 400 SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
9ed28556 401 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
9bf9abea 402 }
649758ff 403 smc_llc_save_peer_uid(qentry);
0fb0b02b
KG		 404 rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
405 smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
406 if (rc)
75d320d6
KG		 407 return SMC_CLC_DECL_RMBE_EC;
408
9bf9abea
UB		 409 rc = smc_ib_modify_qp_rts(link);
410 if (rc)
603cc149 411 return SMC_CLC_DECL_ERR_RDYLNK;
9bf9abea
UB		 412
413 smc_wr_remember_qp_attr(link);
652a1e41 414
7562a13d 415 if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
603cc149 416 return SMC_CLC_DECL_ERR_REGRMB;
652a1e41 417
0fb0b02b
KG		 418 /* confirm_rkey is implicit on 1st contact */
419 smc->conn.rmb_desc->is_conf_rkey = true;
420
9bf9abea 421 /* send CONFIRM LINK response over RoCE fabric */
947541f3 422 rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
9bf9abea 423 if (rc < 0)
603cc149 424 return SMC_CLC_DECL_TIMEOUT_CL;
9bf9abea 425
0fb0b02b 426 smc_llc_link_active(link);
0a99be43 427 smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
0fb0b02b
KG		 428
429 /* optional 2nd link, receive ADD LINK request from server */
430 qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
431 SMC_LLC_ADD_LINK);
432 if (!qentry) {
52bedf37
KG		 433 struct smc_clc_msg_decline dclc;
434
435 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
2b59f58e 436 SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
0fb0b02b
KG		 437 if (rc == -EAGAIN)
438 rc = 0; /* no DECLINE received, go with one link */
439 return rc;
52bedf37 440 }
0fb0b02b 441 smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
b1570a87 442 smc_llc_cli_add_link(link, qentry);
75d320d6 443 return 0;
9bf9abea
UB		 444}
445
41349844
HW		 446static void smcr_conn_save_peer_info(struct smc_sock *smc,
447 struct smc_clc_msg_accept_confirm *clc)
0cfdd8f9 448{
3d9725a6 449 int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
95d8d263 450
3d9725a6
UB		 451 smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
452 smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
95d8d263 453 smc->conn.peer_rmbe_size = bufsize;
cd6851f3 454 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
95d8d263 455 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
0cfdd8f9
UB		 456}
457
b81a5eb7
UB		 458static bool smc_isascii(char *hostname)
459{
460 int i;
461
462 for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
463 if (!isascii(hostname[i]))
464 return false;
465 return true;
466}
467
41349844
HW		 468static void smcd_conn_save_peer_info(struct smc_sock *smc,
469 struct smc_clc_msg_accept_confirm *clc)
470{
3d9725a6 471 int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
41349844 472
3d9725a6
UB		 473 smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
474 smc->conn.peer_token = clc->d0.token;
41349844
HW		 475 /* msg header takes up space in the buffer */
476 smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
477 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
478 smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
b81a5eb7
UB		 479 if (clc->hdr.version > SMC_V1 &&
480 (clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK)) {
481 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
482 (struct smc_clc_msg_accept_confirm_v2 *)clc;
483 struct smc_clc_first_contact_ext *fce =
484 (struct smc_clc_first_contact_ext *)
485 (((u8 *)clc_v2) + sizeof(*clc_v2));
486
487 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->eid,
488 SMC_MAX_EID_LEN);
489 smc->conn.lgr->peer_os = fce->os_type;
490 smc->conn.lgr->peer_smc_release = fce->release;
491 if (smc_isascii(fce->hostname))
492 memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
493 SMC_MAX_HOSTNAME_LEN);
494 }
41349844
HW		 495}
496
497static void smc_conn_save_peer_info(struct smc_sock *smc,
498 struct smc_clc_msg_accept_confirm *clc)
499{
500 if (smc->conn.lgr->is_smcd)
501 smcd_conn_save_peer_info(smc, clc);
502 else
503 smcr_conn_save_peer_info(smc, clc);
504}
505
0cfdd8f9
UB		 506static void smc_link_save_peer_info(struct smc_link *link,
507 struct smc_clc_msg_accept_confirm *clc)
508{
3d9725a6
UB		 509 link->peer_qpn = ntoh24(clc->r0.qpn);
510 memcpy(link->peer_gid, clc->r0.lcl.gid, SMC_GID_SIZE);
511 memcpy(link->peer_mac, clc->r0.lcl.mac, sizeof(link->peer_mac));
512 link->peer_psn = ntoh24(clc->r0.psn);
513 link->peer_mtu = clc->r0.qp_mtu;
0cfdd8f9
UB		 514}
515
e0e4b8fa
GG		 516static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc,
517 struct smc_stats_fback *fback_arr)
518{
519 int cnt;
520
521 for (cnt = 0; cnt < SMC_MAX_FBACK_RSN_CNT; cnt++) {
522 if (fback_arr[cnt].fback_code == smc->fallback_rsn) {
523 fback_arr[cnt].count++;
524 break;
525 }
526 if (!fback_arr[cnt].fback_code) {
527 fback_arr[cnt].fback_code = smc->fallback_rsn;
528 fback_arr[cnt].count++;
529 break;
530 }
531 }
532}
533
534static void smc_stat_fallback(struct smc_sock *smc)
535{
194730a9
GG		 536 struct net *net = sock_net(&smc->sk);
537
538 mutex_lock(&net->smc.mutex_fback_rsn);
e0e4b8fa 539 if (smc->listen_smc) {
194730a9
GG		 540 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv);
541 net->smc.fback_rsn->srv_fback_cnt++;
e0e4b8fa 542 } else {
194730a9
GG		 543 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt);
544 net->smc.fback_rsn->clnt_fback_cnt++;
e0e4b8fa 545 }
194730a9 546 mutex_unlock(&net->smc.mutex_fback_rsn);
e0e4b8fa
GG		 547}
548
549static void smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
07603b23 550{
bd6b2124
WG		 551 wait_queue_head_t *smc_wait = sk_sleep(&smc->sk);
552 wait_queue_head_t *clc_wait = sk_sleep(smc->clcsock->sk);
553 unsigned long flags;
554
07603b23 555 smc->use_fallback = true;
e0e4b8fa
GG		 556 smc->fallback_rsn = reason_code;
557 smc_stat_fallback(smc);
07603b23
UB		 558 if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
559 smc->clcsock->file = smc->sk.sk_socket->file;
560 smc->clcsock->file->private_data = smc->clcsock;
67f562e3
UB		 561 smc->clcsock->wq.fasync_list =
562 smc->sk.sk_socket->wq.fasync_list;
bd6b2124
WG		 563
564 /* There may be some entries remaining in
565 * smc socket->wq, which should be removed
566 * to clcsocket->wq during the fallback.
567 */
568 spin_lock_irqsave(&smc_wait->lock, flags);
6c3fd56d 569 spin_lock_nested(&clc_wait->lock, SINGLE_DEPTH_NESTING);
bd6b2124
WG		 570 list_splice_init(&smc_wait->head, &clc_wait->head);
571 spin_unlock(&clc_wait->lock);
572 spin_unlock_irqrestore(&smc_wait->lock, flags);
07603b23
UB		 573 }
574}
575
3b2dec26 576/* fall back during connect */
603cc149 577static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
a046d57d 578{
e0e4b8fa 579 smc_switch_to_fallback(smc, reason_code);
3b2dec26 580 smc_copy_sock_settings_to_clc(smc);
50717a37 581 smc->connect_nonblock = 0;
3b2dec26
HW		 582 if (smc->sk.sk_state == SMC_INIT)
583 smc->sk.sk_state = SMC_ACTIVE;
584 return 0;
585}
51f1de79 586
3b2dec26 587/* decline and fall back during connect */
e8d726c8
UB		 588static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
589 u8 version)
3b2dec26 590{
194730a9 591 struct net *net = sock_net(&smc->sk);
3b2dec26 592 int rc;
ee9dfbef 593
e1bbdd57 594 if (reason_code < 0) { /* error, fallback is not possible */
194730a9 595 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
e1bbdd57
UB		 596 if (smc->sk.sk_state == SMC_INIT)
597 sock_put(&smc->sk); /* passive closing */
3b2dec26 598 return reason_code;
e1bbdd57 599 }
603cc149 600 if (reason_code != SMC_CLC_DECL_PEERDECL) {
e8d726c8 601 rc = smc_clc_send_decline(smc, reason_code, version);
e1bbdd57 602 if (rc < 0) {
194730a9 603 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
e1bbdd57
UB		 604 if (smc->sk.sk_state == SMC_INIT)
605 sock_put(&smc->sk); /* passive closing */
3b2dec26 606 return rc;
e1bbdd57 607 }
c5c1cc9c 608 }
603cc149 609 return smc_connect_fallback(smc, reason_code);
3b2dec26 610}
c5c1cc9c 611
8cf3f3e4 612static void smc_conn_abort(struct smc_sock *smc, int local_first)
3b2dec26 613{
5ac54d87 614 if (local_first)
51e3dfa8
UB		 615 smc_lgr_cleanup_early(&smc->conn);
616 else
617 smc_conn_free(&smc->conn);
3b2dec26
HW		 618}
619
620/* check if there is a rdma device available for this connection. */
621/* called for connect and listen */
228bae05 622static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
3b2dec26 623{
a046d57d
UB		 624 /* PNET table look up: search active ib_device and port
625 * within same PNETID that also contains the ethernet device
626 * used for the internal TCP socket
627 */
bc36d2fc 628 smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
9aa68d29
KG		 629 if (!ini->ib_dev)
630 return SMC_CLC_DECL_NOSMCRDEV;
bc36d2fc 631 return 0;
3b2dec26
HW		 632}
633
41349844
HW		 634/* check if there is an ISM device available for this connection. */
635/* called for connect and listen */
228bae05 636static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
41349844
HW		 637{
638 /* Find ISM device with same PNETID as connecting interface */
bc36d2fc 639 smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
3fc64937 640 if (!ini->ism_dev[0])
9aa68d29 641 return SMC_CLC_DECL_NOSMCDDEV;
8caaccf5
UB		 642 else
643 ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
41349844
HW		 644 return 0;
645}
646
839d696f
KG		 647/* is chid unique for the ism devices that are already determined? */
648static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
649 int cnt)
650{
651 int i = (!ini->ism_dev[0]) ? 1 : 0;
652
653 for (; i < cnt; i++)
654 if (ini->ism_chid[i] == chid)
655 return false;
656 return true;
657}
658
d70bf4f7
UB		 659/* determine possible V2 ISM devices (either without PNETID or with PNETID plus
660 * PNETID matching net_device)
661 */
662static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
663 struct smc_init_info *ini)
664{
665 int rc = SMC_CLC_DECL_NOSMCDDEV;
666 struct smcd_dev *smcd;
667 int i = 1;
839d696f 668 u16 chid;
d70bf4f7
UB		 669
670 if (smcd_indicated(ini->smc_type_v1))
671 rc = 0; /* already initialized for V1 */
672 mutex_lock(&smcd_dev_list.mutex);
673 list_for_each_entry(smcd, &smcd_dev_list.list, list) {
674 if (smcd->going_away || smcd == ini->ism_dev[0])
675 continue;
839d696f
KG		 676 chid = smc_ism_get_chid(smcd);
677 if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
678 continue;
d70bf4f7
UB		 679 if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
680 smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
681 ini->ism_dev[i] = smcd;
839d696f 682 ini->ism_chid[i] = chid;
d70bf4f7
UB		 683 ini->is_smcd = true;
684 rc = 0;
685 i++;
686 if (i > SMC_MAX_ISM_DEVS)
687 break;
688 }
689 }
690 mutex_unlock(&smcd_dev_list.mutex);
691 ini->ism_offered_cnt = i - 1;
692 if (!ini->ism_dev[0] && !ini->ism_dev[1])
693 ini->smcd_version = 0;
694
695 return rc;
696}
697
41349844
HW		 698/* Check for VLAN ID and register it on ISM device just for CLC handshake */
699static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
bc36d2fc 700 struct smc_init_info *ini)
41349844 701{
3fc64937 702 if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
7a62725a 703 return SMC_CLC_DECL_ISMVLANERR;
41349844
HW		 704 return 0;
705}
706
d70bf4f7
UB		 707static int smc_find_proposal_devices(struct smc_sock *smc,
708 struct smc_init_info *ini)
709{
710 int rc = 0;
711
712 /* check if there is an ism device available */
713 if (ini->smcd_version & SMC_V1) {
714 if (smc_find_ism_device(smc, ini) ||
715 smc_connect_ism_vlan_setup(smc, ini)) {
716 if (ini->smc_type_v1 == SMC_TYPE_B)
717 ini->smc_type_v1 = SMC_TYPE_R;
718 else
719 ini->smc_type_v1 = SMC_TYPE_N;
720 } /* else ISM V1 is supported for this connection */
721 if (smc_find_rdma_device(smc, ini)) {
722 if (ini->smc_type_v1 == SMC_TYPE_B)
723 ini->smc_type_v1 = SMC_TYPE_D;
724 else
725 ini->smc_type_v1 = SMC_TYPE_N;
726 } /* else RDMA is supported for this connection */
727 }
49407ae2 728 if (smc_ism_is_v2_capable() && smc_find_ism_v2_device_clnt(smc, ini))
d70bf4f7
UB		 729 ini->smc_type_v2 = SMC_TYPE_N;
730
731 /* if neither ISM nor RDMA are supported, fallback */
732 if (!smcr_indicated(ini->smc_type_v1) &&
733 ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
734 rc = SMC_CLC_DECL_NOSMCDEV;
735
736 return rc;
737}
738
41349844
HW		 739/* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
740 * used, the VLAN ID will be registered again during the connection setup.
741 */
d70bf4f7 742static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
bc36d2fc 743 struct smc_init_info *ini)
41349844 744{
d70bf4f7 745 if (!smcd_indicated(ini->smc_type_v1))
41349844 746 return 0;
3fc64937 747 if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
41349844
HW		 748 return SMC_CLC_DECL_CNFERR;
749 return 0;
750}
751
a7c9c5f4
UB		 752#define SMC_CLC_MAX_ACCEPT_LEN \
753 (sizeof(struct smc_clc_msg_accept_confirm_v2) + \
b81a5eb7 754 sizeof(struct smc_clc_first_contact_ext) + \
a7c9c5f4
UB		 755 sizeof(struct smc_clc_msg_trail))
756
3b2dec26 757/* CLC handshake during connect */
d70bf4f7 758static int smc_connect_clc(struct smc_sock *smc,
a7c9c5f4 759 struct smc_clc_msg_accept_confirm_v2 *aclc2,
bc36d2fc 760 struct smc_init_info *ini)
3b2dec26
HW		 761{
762 int rc = 0;
a046d57d
UB		 763
764 /* do inband token exchange */
d70bf4f7 765 rc = smc_clc_send_proposal(smc, ini);
3b2dec26
HW		 766 if (rc)
767 return rc;
a046d57d 768 /* receive SMC Accept CLC message */
a7c9c5f4
UB		 769 return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN,
770 SMC_CLC_ACCEPT, CLC_WAIT_TIME);
3b2dec26
HW		 771}
772
773/* setup for RDMA connection of client */
774static int smc_connect_rdma(struct smc_sock *smc,
775 struct smc_clc_msg_accept_confirm *aclc,
bc36d2fc 776 struct smc_init_info *ini)
3b2dec26 777{
0fb0b02b 778 int i, reason_code = 0;
3b2dec26 779 struct smc_link *link;
a046d57d 780
bc36d2fc 781 ini->is_smcd = false;
3d9725a6
UB		 782 ini->ib_lcl = &aclc->r0.lcl;
783 ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
f1eb02f9 784 ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
bc36d2fc 785
72a36a8a 786 mutex_lock(&smc_client_lgr_pending);
7a62725a
KG		 787 reason_code = smc_conn_create(smc, ini);
788 if (reason_code) {
72a36a8a
HW		 789 mutex_unlock(&smc_client_lgr_pending);
790 return reason_code;
0cfdd8f9 791 }
a046d57d 792
3b2dec26 793 smc_conn_save_peer_info(smc, aclc);
cd6851f3 794
5ac54d87 795 if (ini->first_contact_local) {
0fb0b02b
KG		 796 link = smc->conn.lnk;
797 } else {
798 /* set link that was assigned by server */
799 link = NULL;
800 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
801 struct smc_link *l = &smc->conn.lgr->lnk[i];
802
3d9725a6
UB		 803 if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
804 !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
805 SMC_GID_SIZE) &&
806 !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
807 sizeof(l->peer_mac))) {
0fb0b02b
KG		 808 link = l;
809 break;
810 }
811 }
c60a2cef
KG		 812 if (!link) {
813 reason_code = SMC_CLC_DECL_NOSRVLINK;
814 goto connect_abort;
815 }
64513d26 816 smc_switch_link_and_count(&smc->conn, link);
0fb0b02b
KG		 817 }
818
3e034725 819 /* create send buffer and rmb */
c60a2cef
KG		 820 if (smc_buf_create(smc, false)) {
821 reason_code = SMC_CLC_DECL_MEM;
822 goto connect_abort;
823 }
cd6851f3 824
5ac54d87 825 if (ini->first_contact_local)
3b2dec26 826 smc_link_save_peer_info(link, aclc);
bd4ad577 827
c60a2cef
KG		 828 if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
829 reason_code = SMC_CLC_DECL_ERR_RTOK;
830 goto connect_abort;
831 }
bd4ad577 832
46c28dbd
UB		 833 smc_close_init(smc);
834 smc_rx_init(smc);
835
5ac54d87 836 if (ini->first_contact_local) {
c60a2cef
KG		 837 if (smc_ib_ready_link(link)) {
838 reason_code = SMC_CLC_DECL_ERR_RDYLNK;
839 goto connect_abort;
840 }
652a1e41 841 } else {
c60a2cef
KG		 842 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
843 reason_code = SMC_CLC_DECL_ERR_REGRMB;
844 goto connect_abort;
845 }
bd4ad577 846 }
10428dd8 847 smc_rmb_sync_sg_for_device(&smc->conn);
a046d57d 848
a7c9c5f4
UB		 849 reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
850 SMC_V1);
3b2dec26 851 if (reason_code)
c60a2cef 852 goto connect_abort;
3b2dec26
HW		 853
854 smc_tx_init(smc);
a046d57d 855
5ac54d87 856 if (ini->first_contact_local) {
9bf9abea 857 /* QP confirmation over RoCE fabric */
0fb0b02b 858 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
b9247544 859 reason_code = smcr_clnt_conf_first_link(smc);
0fb0b02b 860 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
3b2dec26 861 if (reason_code)
c60a2cef 862 goto connect_abort;
9bf9abea 863 }
72a36a8a 864 mutex_unlock(&smc_client_lgr_pending);
e6727f39 865
a046d57d 866 smc_copy_sock_settings_to_clc(smc);
50717a37 867 smc->connect_nonblock = 0;
b38d7324
UB		 868 if (smc->sk.sk_state == SMC_INIT)
869 smc->sk.sk_state = SMC_ACTIVE;
a046d57d 870
3b2dec26 871 return 0;
c60a2cef 872connect_abort:
8cf3f3e4 873 smc_conn_abort(smc, ini->first_contact_local);
c60a2cef
KG		 874 mutex_unlock(&smc_client_lgr_pending);
875 smc->connect_nonblock = 0;
876
877 return reason_code;
3b2dec26 878}
a046d57d 879
a7c9c5f4
UB		 880/* The server has chosen one of the proposed ISM devices for the communication.
881 * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
882 */
883static int
884smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
885 struct smc_init_info *ini)
886{
887 int i;
888
889 for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
890 if (ini->ism_chid[i] == ntohs(aclc->chid)) {
891 ini->ism_selected = i;
892 return 0;
893 }
894 }
895
896 return -EPROTO;
897}
898
41349844
HW		 899/* setup for ISM connection of client */
900static int smc_connect_ism(struct smc_sock *smc,
901 struct smc_clc_msg_accept_confirm *aclc,
bc36d2fc 902 struct smc_init_info *ini)
41349844 903{
41349844
HW		 904 int rc = 0;
905
bc36d2fc 906 ini->is_smcd = true;
f1eb02f9 907 ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
bc36d2fc 908
a7c9c5f4
UB		 909 if (aclc->hdr.version == SMC_V2) {
910 struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
911 (struct smc_clc_msg_accept_confirm_v2 *)aclc;
912
913 rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
914 if (rc)
915 return rc;
916 }
917 ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
918
72a36a8a
HW		 919 /* there is only one lgr role for SMC-D; use server lock */
920 mutex_lock(&smc_server_lgr_pending);
7a62725a
KG		 921 rc = smc_conn_create(smc, ini);
922 if (rc) {
72a36a8a 923 mutex_unlock(&smc_server_lgr_pending);
7a62725a 924 return rc;
72a36a8a 925 }
41349844
HW		 926
927 /* Create send and receive buffers */
72b7f6c4 928 rc = smc_buf_create(smc, true);
c60a2cef
KG		 929 if (rc) {
930 rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
931 goto connect_abort;
932 }
41349844
HW		 933
934 smc_conn_save_peer_info(smc, aclc);
935 smc_close_init(smc);
936 smc_rx_init(smc);
937 smc_tx_init(smc);
938
a7c9c5f4
UB		 939 rc = smc_clc_send_confirm(smc, ini->first_contact_local,
940 aclc->hdr.version);
41349844 941 if (rc)
c60a2cef 942 goto connect_abort;
72a36a8a 943 mutex_unlock(&smc_server_lgr_pending);
41349844
HW		 944
945 smc_copy_sock_settings_to_clc(smc);
50717a37 946 smc->connect_nonblock = 0;
41349844
HW		 947 if (smc->sk.sk_state == SMC_INIT)
948 smc->sk.sk_state = SMC_ACTIVE;
949
950 return 0;
c60a2cef 951connect_abort:
8cf3f3e4 952 smc_conn_abort(smc, ini->first_contact_local);
c60a2cef
KG		 953 mutex_unlock(&smc_server_lgr_pending);
954 smc->connect_nonblock = 0;
955
956 return rc;
41349844
HW		 957}
958
d70bf4f7
UB		 959/* check if received accept type and version matches a proposed one */
960static int smc_connect_check_aclc(struct smc_init_info *ini,
961 struct smc_clc_msg_accept_confirm *aclc)
962{
963 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
964 !smcr_indicated(ini->smc_type_v1)) ||
965 (aclc->hdr.typev1 == SMC_TYPE_D &&
a7c9c5f4
UB		 966 ((!smcd_indicated(ini->smc_type_v1) &&
967 !smcd_indicated(ini->smc_type_v2)) ||
968 (aclc->hdr.version == SMC_V1 &&
969 !smcd_indicated(ini->smc_type_v1)) ||
970 (aclc->hdr.version == SMC_V2 &&
971 !smcd_indicated(ini->smc_type_v2)))))
d70bf4f7
UB		 972 return SMC_CLC_DECL_MODEUNSUPP;
973
974 return 0;
975}
976
3b2dec26
HW		 977/* perform steps before actually connecting */
978static int __smc_connect(struct smc_sock *smc)
979{
49407ae2 980 u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
a7c9c5f4
UB		 981 struct smc_clc_msg_accept_confirm_v2 *aclc2;
982 struct smc_clc_msg_accept_confirm *aclc;
3fc64937 983 struct smc_init_info *ini = NULL;
a7c9c5f4 984 u8 *buf = NULL;
3b2dec26 985 int rc = 0;
a046d57d 986
3b2dec26 987 if (smc->use_fallback)
603cc149 988 return smc_connect_fallback(smc, smc->fallback_rsn);
3b2dec26
HW		 989
990 /* if peer has not signalled SMC-capability, fall back */
991 if (!tcp_sk(smc->clcsock->sk)->syn_smc)
603cc149 992 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
3b2dec26 993
e8d726c8 994 /* IPSec connections opt out of SMC optimizations */
3b2dec26 995 if (using_ipsec(smc))
e8d726c8
UB		 996 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
997 version);
3b2dec26 998
3fc64937
UB		 999 ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1000 if (!ini)
e8d726c8
UB		 1001 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
1002 version);
3fc64937 1003
d70bf4f7 1004 ini->smcd_version = SMC_V1;
49407ae2 1005 ini->smcd_version |= smc_ism_is_v2_capable() ? SMC_V2 : 0;
d70bf4f7 1006 ini->smc_type_v1 = SMC_TYPE_B;
49407ae2 1007 ini->smc_type_v2 = smc_ism_is_v2_capable() ? SMC_TYPE_D : SMC_TYPE_N;
d70bf4f7 1008
fba7e8ef 1009 /* get vlan id from IP device */
3fc64937 1010 if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
d70bf4f7
UB		 1011 ini->smcd_version &= ~SMC_V1;
1012 ini->smc_type_v1 = SMC_TYPE_N;
1013 if (!ini->smcd_version) {
1014 rc = SMC_CLC_DECL_GETVLANERR;
1015 goto fallback;
1016 }
41349844
HW		 1017 }
1018
d70bf4f7
UB		 1019 rc = smc_find_proposal_devices(smc, ini);
1020 if (rc)
1021 goto fallback;
3b2dec26 1022
a7c9c5f4
UB		 1023 buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
1024 if (!buf) {
1025 rc = SMC_CLC_DECL_MEM;
1026 goto fallback;
1027 }
1028 aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
1029 aclc = (struct smc_clc_msg_accept_confirm *)aclc2;
1030
3b2dec26 1031 /* perform CLC handshake */
a7c9c5f4 1032 rc = smc_connect_clc(smc, aclc2, ini);
d70bf4f7
UB		 1033 if (rc)
1034 goto vlan_cleanup;
1035
1036 /* check if smc modes and versions of CLC proposal and accept match */
a7c9c5f4 1037 rc = smc_connect_check_aclc(ini, aclc);
0530bd6e
KG		 1038 version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
1039 ini->smcd_version = version;
d70bf4f7
UB		 1040 if (rc)
1041 goto vlan_cleanup;
3b2dec26 1042
41349844 1043 /* depending on previous steps, connect using rdma or ism */
a7c9c5f4
UB		 1044 if (aclc->hdr.typev1 == SMC_TYPE_R)
1045 rc = smc_connect_rdma(smc, aclc, ini);
1046 else if (aclc->hdr.typev1 == SMC_TYPE_D)
1047 rc = smc_connect_ism(smc, aclc, ini);
d70bf4f7
UB		 1048 if (rc)
1049 goto vlan_cleanup;
3b2dec26 1050
194730a9 1051 SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
d70bf4f7 1052 smc_connect_ism_vlan_cleanup(smc, ini);
a7c9c5f4 1053 kfree(buf);
3fc64937 1054 kfree(ini);
3b2dec26 1055 return 0;
d70bf4f7
UB		 1056
1057vlan_cleanup:
1058 smc_connect_ism_vlan_cleanup(smc, ini);
a7c9c5f4 1059 kfree(buf);
d70bf4f7
UB		 1060fallback:
1061 kfree(ini);
e8d726c8 1062 return smc_connect_decline_fallback(smc, rc, version);
a046d57d
UB		 1063}
1064
24ac3a08
UB		 1065static void smc_connect_work(struct work_struct *work)
1066{
1067 struct smc_sock *smc = container_of(work, struct smc_sock,
1068 connect_work);
50717a37
UB		 1069 long timeo = smc->sk.sk_sndtimeo;
1070 int rc = 0;
24ac3a08 1071
50717a37
UB		 1072 if (!timeo)
1073 timeo = MAX_SCHEDULE_TIMEOUT;
1074 lock_sock(smc->clcsock->sk);
24ac3a08
UB		 1075 if (smc->clcsock->sk->sk_err) {
1076 smc->sk.sk_err = smc->clcsock->sk->sk_err;
50717a37 1077 } else if ((1 << smc->clcsock->sk->sk_state) &
f3a3a0fe 1078 (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
50717a37
UB		 1079 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
1080 if ((rc == -EPIPE) &&
1081 ((1 << smc->clcsock->sk->sk_state) &
1082 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
1083 rc = 0;
24ac3a08 1084 }
50717a37
UB		 1085 release_sock(smc->clcsock->sk);
1086 lock_sock(&smc->sk);
1087 if (rc != 0 || smc->sk.sk_err) {
1088 smc->sk.sk_state = SMC_CLOSED;
1089 if (rc == -EPIPE || rc == -EAGAIN)
1090 smc->sk.sk_err = EPIPE;
1091 else if (signal_pending(current))
1092 smc->sk.sk_err = -sock_intr_errno(timeo);
6d6dd528 1093 sock_put(&smc->sk); /* passive closing */
24ac3a08
UB		 1094 goto out;
1095 }
1096
1097 rc = __smc_connect(smc);
1098 if (rc < 0)
1099 smc->sk.sk_err = -rc;
1100
1101out:
07603b23
UB		 1102 if (!sock_flag(&smc->sk, SOCK_DEAD)) {
1103 if (smc->sk.sk_err) {
1104 smc->sk.sk_state_change(&smc->sk);
1105 } else { /* allow polling before and after fallback decision */
1106 smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1107 smc->sk.sk_write_space(&smc->sk);
1108 }
1109 }
24ac3a08
UB		 1110 release_sock(&smc->sk);
1111}
1112
ac713874
UB		 1113static int smc_connect(struct socket *sock, struct sockaddr *addr,
1114 int alen, int flags)
1115{
1116 struct sock *sk = sock->sk;
1117 struct smc_sock *smc;
1118 int rc = -EINVAL;
1119
1120 smc = smc_sk(sk);
1121
1122 /* separate smc parameter checking to be safe */
1123 if (alen < sizeof(addr->sa_family))
1124 goto out_err;
aaa4d33f 1125 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
ac713874
UB		 1126 goto out_err;
1127
1128 lock_sock(sk);
1129 switch (sk->sk_state) {
1130 default:
1131 goto out;
1132 case SMC_ACTIVE:
1133 rc = -EISCONN;
1134 goto out;
1135 case SMC_INIT:
ac713874
UB		 1136 break;
1137 }
1138
1139 smc_copy_sock_settings_to_clc(smc);
c5c1cc9c 1140 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
50717a37
UB		 1141 if (smc->connect_nonblock) {
1142 rc = -EALREADY;
1143 goto out;
1144 }
1145 rc = kernel_connect(smc->clcsock, addr, alen, flags);
1146 if (rc && rc != -EINPROGRESS)
1147 goto out;
301428ea
UB		 1148
1149 sock_hold(&smc->sk); /* sock put in passive closing */
86434744
UB		 1150 if (smc->use_fallback)
1151 goto out;
24ac3a08 1152 if (flags & O_NONBLOCK) {
22ef473d 1153 if (queue_work(smc_hs_wq, &smc->connect_work))
50717a37 1154 smc->connect_nonblock = 1;
24ac3a08
UB		 1155 rc = -EINPROGRESS;
1156 } else {
24ac3a08
UB		 1157 rc = __smc_connect(smc);
1158 if (rc < 0)
1159 goto out;
1160 else
1161 rc = 0; /* success cases including fallback */
1162 }
ac713874
UB		 1163
1164out:
1165 release_sock(sk);
1166out_err:
1167 return rc;
1168}
1169
1170static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
1171{
3163c507
UB		 1172 struct socket *new_clcsock = NULL;
1173 struct sock *lsk = &lsmc->sk;
ac713874 1174 struct sock *new_sk;
78abe3d0 1175 int rc = -EINVAL;
ac713874 1176
3163c507 1177 release_sock(lsk);
aaa4d33f 1178 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
ac713874
UB		 1179 if (!new_sk) {
1180 rc = -ENOMEM;
3163c507 1181 lsk->sk_err = ENOMEM;
ac713874 1182 *new_smc = NULL;
3163c507 1183 lock_sock(lsk);
ac713874
UB		 1184 goto out;
1185 }
1186 *new_smc = smc_sk(new_sk);
1187
78abe3d0
MJ		 1188 mutex_lock(&lsmc->clcsock_release_lock);
1189 if (lsmc->clcsock)
a60a2b1e 1190 rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
78abe3d0 1191 mutex_unlock(&lsmc->clcsock_release_lock);
3163c507 1192 lock_sock(lsk);
a60a2b1e 1193 if (rc < 0 && rc != -EAGAIN)
3163c507 1194 lsk->sk_err = -rc;
35a6b178 1195 if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
f61bca58 1196 new_sk->sk_prot->unhash(new_sk);
a046d57d
UB		 1197 if (new_clcsock)
1198 sock_release(new_clcsock);
1199 new_sk->sk_state = SMC_CLOSED;
1200 sock_set_flag(new_sk, SOCK_DEAD);
51f1de79 1201 sock_put(new_sk); /* final */
ac713874
UB		 1202 *new_smc = NULL;
1203 goto out;
1204 }
1205
a60a2b1e
UB		 1206 /* new clcsock has inherited the smc listen-specific sk_data_ready
1207 * function; switch it back to the original sk_data_ready function
1208 */
1209 new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
ac713874
UB		 1210 (*new_smc)->clcsock = new_clcsock;
1211out:
1212 return rc;
1213}
1214
a046d57d
UB		 1215/* add a just created sock to the accept queue of the listen sock as
1216 * candidate for a following socket accept call from user space
1217 */
1218static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
1219{
1220 struct smc_sock *par = smc_sk(parent);
1221
51f1de79 1222 sock_hold(sk); /* sock_put in smc_accept_unlink () */
a046d57d
UB		 1223 spin_lock(&par->accept_q_lock);
1224 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
1225 spin_unlock(&par->accept_q_lock);
1226 sk_acceptq_added(parent);
1227}
1228
1229/* remove a socket from the accept queue of its parental listening socket */
1230static void smc_accept_unlink(struct sock *sk)
1231{
1232 struct smc_sock *par = smc_sk(sk)->listen_smc;
1233
1234 spin_lock(&par->accept_q_lock);
1235 list_del_init(&smc_sk(sk)->accept_q);
1236 spin_unlock(&par->accept_q_lock);
1237 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
51f1de79 1238 sock_put(sk); /* sock_hold in smc_accept_enqueue */
a046d57d
UB		 1239}
1240
1241/* remove a sock from the accept queue to bind it to a new socket created
1242 * for a socket accept call from user space
1243 */
b38d7324
UB		 1244struct sock *smc_accept_dequeue(struct sock *parent,
1245 struct socket *new_sock)
a046d57d
UB		 1246{
1247 struct smc_sock *isk, *n;
1248 struct sock *new_sk;
1249
1250 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1251 new_sk = (struct sock *)isk;
1252
1253 smc_accept_unlink(new_sk);
1254 if (new_sk->sk_state == SMC_CLOSED) {
f61bca58 1255 new_sk->sk_prot->unhash(new_sk);
127f4970
UB		 1256 if (isk->clcsock) {
1257 sock_release(isk->clcsock);
1258 isk->clcsock = NULL;
1259 }
51f1de79 1260 sock_put(new_sk); /* final */
a046d57d
UB		 1261 continue;
1262 }
07603b23 1263 if (new_sock) {
a046d57d 1264 sock_graft(new_sk, new_sock);
07603b23
UB		 1265 if (isk->use_fallback) {
1266 smc_sk(new_sk)->clcsock->file = new_sock->file;
1267 isk->clcsock->file->private_data = isk->clcsock;
1268 }
1269 }
a046d57d
UB		 1270 return new_sk;
1271 }
1272 return NULL;
1273}
1274
1275/* clean up for a created but never accepted sock */
b38d7324 1276void smc_close_non_accepted(struct sock *sk)
a046d57d
UB		 1277{
1278 struct smc_sock *smc = smc_sk(sk);
1279
81cf4f47 1280 sock_hold(sk); /* sock_put below */
b38d7324
UB		 1281 lock_sock(sk);
1282 if (!sk->sk_lingertime)
1283 /* wait for peer closing */
1284 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
39f41f36 1285 __smc_release(smc);
b38d7324 1286 release_sock(sk);
81cf4f47 1287 sock_put(sk); /* sock_hold above */
51f1de79 1288 sock_put(sk); /* final sock_put */
a046d57d
UB		 1289}
1290
b9247544 1291static int smcr_serv_conf_first_link(struct smc_sock *smc)
9bf9abea 1292{
387707fd 1293 struct smc_link *link = smc->conn.lnk;
4667bb4a 1294 struct smc_llc_qentry *qentry;
9bf9abea
UB		 1295 int rc;
1296
7562a13d 1297 if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
603cc149 1298 return SMC_CLC_DECL_ERR_REGRMB;
652a1e41 1299
9bf9abea 1300 /* send CONFIRM LINK request to client over the RoCE fabric */
947541f3 1301 rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
9bf9abea 1302 if (rc < 0)
603cc149 1303 return SMC_CLC_DECL_TIMEOUT_CL;
9bf9abea
UB		 1304
1305 /* receive CONFIRM LINK response from client over the RoCE fabric */
4667bb4a
KG		 1306 qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
1307 SMC_LLC_CONFIRM_LINK);
1308 if (!qentry) {
9bf9abea
UB		 1309 struct smc_clc_msg_decline dclc;
1310
1311 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
2b59f58e 1312 SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
9ed28556 1313 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
9bf9abea 1314 }
649758ff 1315 smc_llc_save_peer_uid(qentry);
4667bb4a
KG		 1316 rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP);
1317 smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
1318 if (rc)
75d320d6
KG		 1319 return SMC_CLC_DECL_RMBE_EC;
1320
4667bb4a
KG		 1321 /* confirm_rkey is implicit on 1st contact */
1322 smc->conn.rmb_desc->is_conf_rkey = true;
52bedf37 1323
00a049cf 1324 smc_llc_link_active(link);
0a99be43 1325 smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
52bedf37 1326
4667bb4a 1327 /* initial contact - try to establish second link */
2d2209f2 1328 smc_llc_srv_add_link(link);
75d320d6 1329 return 0;
9bf9abea
UB		 1330}
1331
3b2dec26
HW		 1332/* listen worker: finish */
1333static void smc_listen_out(struct smc_sock *new_smc)
a046d57d 1334{
a046d57d 1335 struct smc_sock *lsmc = new_smc->listen_smc;
a046d57d 1336 struct sock *newsmcsk = &new_smc->sk;
a046d57d 1337
3b2dec26 1338 if (lsmc->sk.sk_state == SMC_LISTEN) {
fd57770d 1339 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
3b2dec26 1340 smc_accept_enqueue(&lsmc->sk, newsmcsk);
fd57770d 1341 release_sock(&lsmc->sk);
3b2dec26
HW		 1342 } else { /* no longer listening */
1343 smc_close_non_accepted(newsmcsk);
c5c1cc9c
UB		 1344 }
1345
3b2dec26
HW		 1346 /* Wake up accept */
1347 lsmc->sk.sk_data_ready(&lsmc->sk);
1348 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1349}
a046d57d 1350
3b2dec26
HW		 1351/* listen worker: finish in state connected */
1352static void smc_listen_out_connected(struct smc_sock *new_smc)
1353{
1354 struct sock *newsmcsk = &new_smc->sk;
a046d57d 1355
3b2dec26
HW		 1356 sk_refcnt_debug_inc(newsmcsk);
1357 if (newsmcsk->sk_state == SMC_INIT)
1358 newsmcsk->sk_state = SMC_ACTIVE;
1359
1360 smc_listen_out(new_smc);
1361}
1362
1363/* listen worker: finish in error state */
1364static void smc_listen_out_err(struct smc_sock *new_smc)
1365{
1366 struct sock *newsmcsk = &new_smc->sk;
194730a9 1367 struct net *net = sock_net(newsmcsk);
3b2dec26 1368
194730a9 1369 this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt);
3b2dec26
HW		 1370 if (newsmcsk->sk_state == SMC_INIT)
1371 sock_put(&new_smc->sk); /* passive closing */
1372 newsmcsk->sk_state = SMC_CLOSED;
3b2dec26
HW		 1373
1374 smc_listen_out(new_smc);
1375}
1376
1377/* listen worker: decline and fall back if possible */
1378static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
4a9baf45 1379 int local_first, u8 version)
3b2dec26
HW		 1380{
1381 /* RDMA setup failed, switch back to TCP */
8cf3f3e4 1382 smc_conn_abort(new_smc, local_first);
3b2dec26
HW		 1383 if (reason_code < 0) { /* error, no fallback possible */
1384 smc_listen_out_err(new_smc);
1385 return;
1386 }
e0e4b8fa 1387 smc_switch_to_fallback(new_smc, reason_code);
603cc149 1388 if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
e8d726c8 1389 if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
3b2dec26
HW		 1390 smc_listen_out_err(new_smc);
1391 return;
1392 }
a046d57d 1393 }
3b2dec26
HW		 1394 smc_listen_out_connected(new_smc);
1395}
1396
5c21c4cc
UB		 1397/* listen worker: version checking */
1398static int smc_listen_v2_check(struct smc_sock *new_smc,
1399 struct smc_clc_msg_proposal *pclc,
1400 struct smc_init_info *ini)
1401{
1402 struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
1403 struct smc_clc_v2_extension *pclc_v2_ext;
3752404a 1404 int rc = SMC_CLC_DECL_PEERNOSMC;
5c21c4cc
UB		 1405
1406 ini->smc_type_v1 = pclc->hdr.typev1;
1407 ini->smc_type_v2 = pclc->hdr.typev2;
1408 ini->smcd_version = ini->smc_type_v1 != SMC_TYPE_N ? SMC_V1 : 0;
1409 if (pclc->hdr.version > SMC_V1)
1410 ini->smcd_version |=
1411 ini->smc_type_v2 != SMC_TYPE_N ? SMC_V2 : 0;
3752404a
KG		 1412 if (!(ini->smcd_version & SMC_V2)) {
1413 rc = SMC_CLC_DECL_PEERNOSMC;
1414 goto out;
1415 }
49407ae2 1416 if (!smc_ism_is_v2_capable()) {
5c21c4cc 1417 ini->smcd_version &= ~SMC_V2;
3752404a 1418 rc = SMC_CLC_DECL_NOISM2SUPP;
5c21c4cc
UB		 1419 goto out;
1420 }
1421 pclc_v2_ext = smc_get_clc_v2_ext(pclc);
1422 if (!pclc_v2_ext) {
1423 ini->smcd_version &= ~SMC_V2;
3752404a 1424 rc = SMC_CLC_DECL_NOV2EXT;
5c21c4cc
UB		 1425 goto out;
1426 }
1427 pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext);
3752404a 1428 if (!pclc_smcd_v2_ext) {
5c21c4cc 1429 ini->smcd_version &= ~SMC_V2;
3752404a
KG		 1430 rc = SMC_CLC_DECL_NOV2DEXT;
1431 }
5c21c4cc
UB		 1432
1433out:
3752404a
KG		 1434 if (!ini->smcd_version)
1435 return rc;
5c21c4cc
UB		 1436
1437 return 0;
1438}
1439
3b2dec26 1440/* listen worker: check prefixes */
59886697 1441static int smc_listen_prfx_check(struct smc_sock *new_smc,
3b2dec26
HW		 1442 struct smc_clc_msg_proposal *pclc)
1443{
1444 struct smc_clc_msg_proposal_prefix *pclc_prfx;
1445 struct socket *newclcsock = new_smc->clcsock;
a046d57d 1446
5c21c4cc
UB		 1447 if (pclc->hdr.typev1 == SMC_TYPE_N)
1448 return 0;
e7b7a64a 1449 pclc_prfx = smc_clc_proposal_get_prefix(pclc);
3b2dec26 1450 if (smc_clc_prfx_match(newclcsock, pclc_prfx))
59886697 1451 return SMC_CLC_DECL_DIFFPREFIX;
c246d942 1452
3b2dec26
HW		 1453 return 0;
1454}
a046d57d 1455
3b2dec26
HW		 1456/* listen worker: initialize connection and buffers */
1457static int smc_listen_rdma_init(struct smc_sock *new_smc,
7a62725a 1458 struct smc_init_info *ini)
3b2dec26 1459{
7a62725a
KG		 1460 int rc;
1461
0cfdd8f9 1462 /* allocate connection / link group */
7a62725a
KG		 1463 rc = smc_conn_create(new_smc, ini);
1464 if (rc)
1465 return rc;
a046d57d 1466
3e034725 1467 /* create send buffer and rmb */
c6ba7c9b 1468 if (smc_buf_create(new_smc, false))
3b2dec26 1469 return SMC_CLC_DECL_MEM;
a046d57d 1470
3b2dec26
HW		 1471 return 0;
1472}
1473
41349844
HW		 1474/* listen worker: initialize connection and buffers for SMC-D */
1475static int smc_listen_ism_init(struct smc_sock *new_smc,
7a62725a 1476 struct smc_init_info *ini)
41349844 1477{
7a62725a 1478 int rc;
41349844 1479
7a62725a
KG		 1480 rc = smc_conn_create(new_smc, ini);
1481 if (rc)
1482 return rc;
41349844 1483
41349844 1484 /* Create send and receive buffers */
72b7f6c4
KG		 1485 rc = smc_buf_create(new_smc, true);
1486 if (rc) {
8cf3f3e4 1487 smc_conn_abort(new_smc, ini->first_contact_local);
72b7f6c4
KG		 1488 return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
1489 SMC_CLC_DECL_MEM;
41349844
HW		 1490 }
1491
1492 return 0;
1493}
1494
5c21c4cc
UB		 1495static bool smc_is_already_selected(struct smcd_dev *smcd,
1496 struct smc_init_info *ini,
1497 int matches)
1498{
1499 int i;
1500
1501 for (i = 0; i < matches; i++)
1502 if (smcd == ini->ism_dev[i])
1503 return true;
1504
1505 return false;
1506}
1507
1508/* check for ISM devices matching proposed ISM devices */
1509static void smc_check_ism_v2_match(struct smc_init_info *ini,
1510 u16 proposed_chid, u64 proposed_gid,
1511 unsigned int *matches)
1512{
1513 struct smcd_dev *smcd;
1514
1515 list_for_each_entry(smcd, &smcd_dev_list.list, list) {
1516 if (smcd->going_away)
1517 continue;
1518 if (smc_is_already_selected(smcd, ini, *matches))
1519 continue;
1520 if (smc_ism_get_chid(smcd) == proposed_chid &&
1521 !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
1522 ini->ism_peer_gid[*matches] = proposed_gid;
1523 ini->ism_dev[*matches] = smcd;
1524 (*matches)++;
1525 break;
1526 }
1527 }
1528}
1529
3752404a
KG		 1530static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini)
1531{
1532 if (!ini->rc)
1533 ini->rc = rc;
1534}
1535
5c21c4cc
UB		 1536static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
1537 struct smc_clc_msg_proposal *pclc,
1538 struct smc_init_info *ini)
1539{
1540 struct smc_clc_smcd_v2_extension *smcd_v2_ext;
1541 struct smc_clc_v2_extension *smc_v2_ext;
1542 struct smc_clc_msg_smcd *pclc_smcd;
1543 unsigned int matches = 0;
f29fa003 1544 u8 smcd_version;
5c21c4cc 1545 u8 *eid = NULL;
3752404a 1546 int i, rc;
5c21c4cc
UB		 1547
1548 if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
f29fa003 1549 goto not_found;
5c21c4cc
UB		 1550
1551 pclc_smcd = smc_get_clc_msg_smcd(pclc);
1552 smc_v2_ext = smc_get_clc_v2_ext(pclc);
1553 smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
1554 if (!smcd_v2_ext ||
3752404a
KG		 1555 !smc_v2_ext->hdr.flag.seid) { /* no system EID support for SMCD */
1556 smc_find_ism_store_rc(SMC_CLC_DECL_NOSEID, ini);
5c21c4cc 1557 goto not_found;
3752404a 1558 }
5c21c4cc
UB		 1559
1560 mutex_lock(&smcd_dev_list.mutex);
1561 if (pclc_smcd->ism.chid)
1562 /* check for ISM device matching proposed native ISM device */
1563 smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
1564 ntohll(pclc_smcd->ism.gid), &matches);
1565 for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
1566 /* check for ISM devices matching proposed non-native ISM
1567 * devices
1568 */
1569 smc_check_ism_v2_match(ini,
1570 ntohs(smcd_v2_ext->gidchid[i - 1].chid),
1571 ntohll(smcd_v2_ext->gidchid[i - 1].gid),
1572 &matches);
1573 }
1574 mutex_unlock(&smcd_dev_list.mutex);
1575
1576 if (ini->ism_dev[0]) {
1577 smc_ism_get_system_eid(ini->ism_dev[0], &eid);
1578 if (memcmp(eid, smcd_v2_ext->system_eid, SMC_MAX_EID_LEN))
1579 goto not_found;
1580 } else {
1581 goto not_found;
1582 }
1583
1584 /* separate - outside the smcd_dev_list.lock */
f29fa003 1585 smcd_version = ini->smcd_version;
5c21c4cc
UB		 1586 for (i = 0; i < matches; i++) {
1587 ini->smcd_version = SMC_V2;
1588 ini->is_smcd = true;
1589 ini->ism_selected = i;
3752404a
KG		 1590 rc = smc_listen_ism_init(new_smc, ini);
1591 if (rc) {
1592 smc_find_ism_store_rc(rc, ini);
5c21c4cc
UB		 1593 /* try next active ISM device */
1594 continue;
3752404a 1595 }
5c21c4cc
UB		 1596 return; /* matching and usable V2 ISM device found */
1597 }
f29fa003
KG		 1598 /* no V2 ISM device could be initialized */
1599 ini->smcd_version = smcd_version; /* restore original value */
5c21c4cc
UB		 1600
1601not_found:
1602 ini->smcd_version &= ~SMC_V2;
1603 ini->ism_dev[0] = NULL;
1604 ini->is_smcd = false;
1605}
1606
1607static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
1608 struct smc_clc_msg_proposal *pclc,
1609 struct smc_init_info *ini)
7affc809
UB		 1610{
1611 struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc);
3752404a 1612 int rc = 0;
7affc809 1613
5c21c4cc
UB		 1614 /* check if ISM V1 is available */
1615 if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
7affc809
UB		 1616 goto not_found;
1617 ini->is_smcd = true; /* prepare ISM check */
8c3dca34 1618 ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
3752404a
KG		 1619 rc = smc_find_ism_device(new_smc, ini);
1620 if (rc)
7affc809 1621 goto not_found;
5c21c4cc 1622 ini->ism_selected = 0;
3752404a
KG		 1623 rc = smc_listen_ism_init(new_smc, ini);
1624 if (!rc)
5c21c4cc 1625 return; /* V1 ISM device found */
7affc809
UB		 1626
1627not_found:
3752404a 1628 smc_find_ism_store_rc(rc, ini);
3fc64937 1629 ini->ism_dev[0] = NULL;
7affc809
UB		 1630 ini->is_smcd = false;
1631}
1632
3b2dec26 1633/* listen worker: register buffers */
5ac54d87 1634static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
3b2dec26 1635{
b9247544 1636 struct smc_connection *conn = &new_smc->conn;
46c28dbd 1637
5ac54d87 1638 if (!local_first) {
7562a13d 1639 if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
c7674c00 1640 return SMC_CLC_DECL_ERR_REGRMB;
652a1e41 1641 }
10428dd8 1642 smc_rmb_sync_sg_for_device(&new_smc->conn);
652a1e41 1643
3b2dec26
HW		 1644 return 0;
1645}
1646
5c21c4cc
UB		 1647static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
1648 struct smc_clc_msg_proposal *pclc,
1649 struct smc_init_info *ini)
7affc809
UB		 1650{
1651 int rc;
1652
5c21c4cc 1653 if (!smcr_indicated(ini->smc_type_v1))
7affc809
UB		 1654 return SMC_CLC_DECL_NOSMCDEV;
1655
1656 /* prepare RDMA check */
1657 ini->ib_lcl = &pclc->lcl;
1658 rc = smc_find_rdma_device(new_smc, ini);
1659 if (rc) {
1660 /* no RDMA device found */
5c21c4cc 1661 if (ini->smc_type_v1 == SMC_TYPE_B)
7affc809
UB		 1662 /* neither ISM nor RDMA device found */
1663 rc = SMC_CLC_DECL_NOSMCDEV;
1664 return rc;
1665 }
1666 rc = smc_listen_rdma_init(new_smc, ini);
1667 if (rc)
1668 return rc;
1669 return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
1670}
1671
1672/* determine the local device matching to proposal */
1673static int smc_listen_find_device(struct smc_sock *new_smc,
1674 struct smc_clc_msg_proposal *pclc,
1675 struct smc_init_info *ini)
1676{
5c21c4cc
UB		 1677 int rc;
1678
1679 /* check for ISM device matching V2 proposed device */
1680 smc_find_ism_v2_device_serv(new_smc, pclc, ini);
1681 if (ini->ism_dev[0])
7affc809 1682 return 0;
5c21c4cc
UB		 1683
1684 if (!(ini->smcd_version & SMC_V1))
3752404a 1685 return ini->rc ?: SMC_CLC_DECL_NOSMCD2DEV;
5c21c4cc
UB		 1686
1687 /* check for matching IP prefix and subnet length */
1688 rc = smc_listen_prfx_check(new_smc, pclc);
1689 if (rc)
3752404a 1690 return ini->rc ?: rc;
5c21c4cc
UB		 1691
1692 /* get vlan id from IP device */
1693 if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
3752404a 1694 return ini->rc ?: SMC_CLC_DECL_GETVLANERR;
5c21c4cc
UB		 1695
1696 /* check for ISM device matching V1 proposed device */
1697 smc_find_ism_v1_device_serv(new_smc, pclc, ini);
1698 if (ini->ism_dev[0])
1699 return 0;
1700
7affc809 1701 if (pclc->hdr.typev1 == SMC_TYPE_D)
3752404a
KG		 1702 /* skip RDMA and decline */
1703 return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV;
7affc809
UB		 1704
1705 /* check if RDMA is available */
3752404a
KG		 1706 rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
1707 smc_find_ism_store_rc(rc, ini);
1708
1709 return (!rc) ? 0 : ini->rc;
7affc809
UB		 1710}
1711
3b2dec26 1712/* listen worker: finish RDMA setup */
1ca52fcf
UB		 1713static int smc_listen_rdma_finish(struct smc_sock *new_smc,
1714 struct smc_clc_msg_accept_confirm *cclc,
5ac54d87 1715 bool local_first)
3b2dec26 1716{
387707fd 1717 struct smc_link *link = new_smc->conn.lnk;
3b2dec26 1718 int reason_code = 0;
a046d57d 1719
5ac54d87 1720 if (local_first)
3b2dec26 1721 smc_link_save_peer_info(link, cclc);
a046d57d 1722
0c881ada
UB		 1723 if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
1724 return SMC_CLC_DECL_ERR_RTOK;
bd4ad577 1725
5ac54d87 1726 if (local_first) {
0c881ada
UB		 1727 if (smc_ib_ready_link(link))
1728 return SMC_CLC_DECL_ERR_RDYLNK;
9bf9abea 1729 /* QP confirmation over RoCE fabric */
4667bb4a 1730 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
b9247544 1731 reason_code = smcr_serv_conf_first_link(new_smc);
4667bb4a 1732 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
bd4ad577 1733 }
1ca52fcf 1734 return reason_code;
3b2dec26 1735}
e6727f39 1736
7affc809 1737/* setup for connection of server */
3b2dec26
HW		 1738static void smc_listen_work(struct work_struct *work)
1739{
1740 struct smc_sock *new_smc = container_of(work, struct smc_sock,
1741 smc_listen_work);
49407ae2 1742 u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
3b2dec26 1743 struct socket *newclcsock = new_smc->clcsock;
a7c9c5f4 1744 struct smc_clc_msg_accept_confirm *cclc;
6bb14e48 1745 struct smc_clc_msg_proposal_area *buf;
3b2dec26 1746 struct smc_clc_msg_proposal *pclc;
3fc64937 1747 struct smc_init_info *ini = NULL;
3b2dec26 1748 int rc = 0;
3b2dec26 1749
fd57770d
KG		 1750 if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
1751 return smc_listen_out_err(new_smc);
1752
3b2dec26
HW		 1753 if (new_smc->use_fallback) {
1754 smc_listen_out_connected(new_smc);
1755 return;
a046d57d 1756 }
a046d57d 1757
3b2dec26
HW		 1758 /* check if peer is smc capable */
1759 if (!tcp_sk(newclcsock->sk)->syn_smc) {
e0e4b8fa 1760 smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC);
3b2dec26
HW		 1761 smc_listen_out_connected(new_smc);
1762 return;
1763 }
a046d57d 1764
3b2dec26
HW		 1765 /* do inband token exchange -
1766 * wait for and receive SMC Proposal CLC message
1767 */
6bb14e48
UB		 1768 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
1769 if (!buf) {
1770 rc = SMC_CLC_DECL_MEM;
1771 goto out_decl;
1772 }
1773 pclc = (struct smc_clc_msg_proposal *)buf;
1774 rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf),
228bae05 1775 SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
9aa68d29
KG		 1776 if (rc)
1777 goto out_decl;
e8d726c8 1778 version = pclc->hdr.version == SMC_V1 ? SMC_V1 : version;
a046d57d 1779
e8d726c8 1780 /* IPSec connections opt out of SMC optimizations */
3b2dec26 1781 if (using_ipsec(new_smc)) {
9aa68d29
KG		 1782 rc = SMC_CLC_DECL_IPSEC;
1783 goto out_decl;
3b2dec26
HW		 1784 }
1785
3fc64937
UB		 1786 ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1787 if (!ini) {
1788 rc = SMC_CLC_DECL_MEM;
1789 goto out_decl;
1790 }
1791
5c21c4cc
UB		 1792 /* initial version checking */
1793 rc = smc_listen_v2_check(new_smc, pclc, ini);
1794 if (rc)
9aa68d29 1795 goto out_decl;
fba7e8ef 1796
72a36a8a 1797 mutex_lock(&smc_server_lgr_pending);
3b2dec26
HW		 1798 smc_close_init(new_smc);
1799 smc_rx_init(new_smc);
1800 smc_tx_init(new_smc);
1801
7affc809 1802 /* determine ISM or RoCE device used for connection */
3fc64937 1803 rc = smc_listen_find_device(new_smc, pclc, ini);
7affc809
UB		 1804 if (rc)
1805 goto out_unlock;
3b2dec26
HW		 1806
1807 /* send SMC Accept CLC message */
a7c9c5f4
UB		 1808 rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
1809 ini->smcd_version == SMC_V2 ? SMC_V2 : SMC_V1);
9aa68d29
KG		 1810 if (rc)
1811 goto out_unlock;
3b2dec26 1812
62c7139f 1813 /* SMC-D does not need this lock any more */
3fc64937 1814 if (ini->is_smcd)
72a36a8a 1815 mutex_unlock(&smc_server_lgr_pending);
62c7139f 1816
3b2dec26 1817 /* receive SMC Confirm CLC message */
9047a617
KG		 1818 memset(buf, 0, sizeof(*buf));
1819 cclc = (struct smc_clc_msg_accept_confirm *)buf;
1820 rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf),
228bae05
KG		 1821 SMC_CLC_CONFIRM, CLC_WAIT_TIME);
1822 if (rc) {
3fc64937 1823 if (!ini->is_smcd)
9aa68d29
KG		 1824 goto out_unlock;
1825 goto out_decl;
3b2dec26
HW		 1826 }
1827
1828 /* finish worker */
3fc64937 1829 if (!ini->is_smcd) {
a7c9c5f4 1830 rc = smc_listen_rdma_finish(new_smc, cclc,
3fc64937 1831 ini->first_contact_local);
62c7139f 1832 if (rc)
0c881ada
UB		 1833 goto out_unlock;
1834 mutex_unlock(&smc_server_lgr_pending);
1ca52fcf 1835 }
a7c9c5f4 1836 smc_conn_save_peer_info(new_smc, cclc);
3b2dec26 1837 smc_listen_out_connected(new_smc);
194730a9 1838 SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
ac679364 1839 goto out_free;
9aa68d29
KG		 1840
1841out_unlock:
1842 mutex_unlock(&smc_server_lgr_pending);
1843out_decl:
4a9baf45
KG		 1844 smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
1845 version);
ac679364 1846out_free:
3fc64937 1847 kfree(ini);
6bb14e48 1848 kfree(buf);
a046d57d
UB		 1849}
1850
1851static void smc_tcp_listen_work(struct work_struct *work)
1852{
1853 struct smc_sock *lsmc = container_of(work, struct smc_sock,
1854 tcp_listen_work);
3163c507 1855 struct sock *lsk = &lsmc->sk;
a046d57d
UB		 1856 struct smc_sock *new_smc;
1857 int rc = 0;
1858
3163c507
UB		 1859 lock_sock(lsk);
1860 while (lsk->sk_state == SMC_LISTEN) {
a046d57d 1861 rc = smc_clcsock_accept(lsmc, &new_smc);
a60a2b1e 1862 if (rc) /* clcsock accept queue empty or error */
a046d57d
UB		 1863 goto out;
1864 if (!new_smc)
1865 continue;
1866
1867 new_smc->listen_smc = lsmc;
ee9dfbef 1868 new_smc->use_fallback = lsmc->use_fallback;
603cc149 1869 new_smc->fallback_rsn = lsmc->fallback_rsn;
3163c507 1870 sock_hold(lsk); /* sock_put in smc_listen_work */
a046d57d
UB		 1871 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1872 smc_copy_sock_settings_to_smc(new_smc);
bd58c7e0
UB		 1873 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1874 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
51f1de79 1875 sock_hold(&new_smc->sk); /* sock_put in passive closing */
22ef473d 1876 if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
51f1de79 1877 sock_put(&new_smc->sk);
a046d57d
UB		 1878 }
1879
1880out:
3163c507 1881 release_sock(lsk);
a60a2b1e
UB		 1882 sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
1883}
1884
1885static void smc_clcsock_data_ready(struct sock *listen_clcsock)
1886{
1887 struct smc_sock *lsmc;
1888
1889 lsmc = (struct smc_sock *)
1890 ((uintptr_t)listen_clcsock->sk_user_data & ~SK_USER_DATA_NOCOPY);
1891 if (!lsmc)
1892 return;
1893 lsmc->clcsk_data_ready(listen_clcsock);
1894 if (lsmc->sk.sk_state == SMC_LISTEN) {
1895 sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
22ef473d 1896 if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work))
a60a2b1e
UB		 1897 sock_put(&lsmc->sk);
1898 }
a046d57d
UB		 1899}
1900
ac713874
UB		 1901static int smc_listen(struct socket *sock, int backlog)
1902{
1903 struct sock *sk = sock->sk;
1904 struct smc_sock *smc;
1905 int rc;
1906
1907 smc = smc_sk(sk);
1908 lock_sock(sk);
1909
1910 rc = -EINVAL;
cd206360
UB		 1911 if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
1912 smc->connect_nonblock)
ac713874
UB		 1913 goto out;
1914
1915 rc = 0;
1916 if (sk->sk_state == SMC_LISTEN) {
1917 sk->sk_max_ack_backlog = backlog;
1918 goto out;
1919 }
1920 /* some socket options are handled in core, so we could not apply
1921 * them to the clc socket -- copy smc socket options to clc socket
1922 */
1923 smc_copy_sock_settings_to_clc(smc);
ee9dfbef
UB		 1924 if (!smc->use_fallback)
1925 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
ac713874 1926
a60a2b1e
UB		 1927 /* save original sk_data_ready function and establish
1928 * smc-specific sk_data_ready function
1929 */
1930 smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready;
1931 smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready;
1932 smc->clcsock->sk->sk_user_data =
1933 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
ac713874 1934 rc = kernel_listen(smc->clcsock, backlog);
6ce3642c
GD		 1935 if (rc) {
1936 smc->clcsock->sk->sk_data_ready = smc->clcsk_data_ready;
ac713874 1937 goto out;
6ce3642c 1938 }
ac713874
UB		 1939 sk->sk_max_ack_backlog = backlog;
1940 sk->sk_ack_backlog = 0;
1941 sk->sk_state = SMC_LISTEN;
1942
1943out:
1944 release_sock(sk);
1945 return rc;
1946}
1947
1948static int smc_accept(struct socket *sock, struct socket *new_sock,
cdfbabfb 1949 int flags, bool kern)
ac713874 1950{
a046d57d
UB		 1951 struct sock *sk = sock->sk, *nsk;
1952 DECLARE_WAITQUEUE(wait, current);
ac713874 1953 struct smc_sock *lsmc;
a046d57d
UB		 1954 long timeo;
1955 int rc = 0;
ac713874
UB		 1956
1957 lsmc = smc_sk(sk);
51f1de79 1958 sock_hold(sk); /* sock_put below */
ac713874
UB		 1959 lock_sock(sk);
1960
1961 if (lsmc->sk.sk_state != SMC_LISTEN) {
1962 rc = -EINVAL;
abb190f1 1963 release_sock(sk);
ac713874
UB		 1964 goto out;
1965 }
1966
a046d57d
UB		 1967 /* Wait for an incoming connection */
1968 timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1969 add_wait_queue_exclusive(sk_sleep(sk), &wait);
1970 while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1971 set_current_state(TASK_INTERRUPTIBLE);
1972 if (!timeo) {
1973 rc = -EAGAIN;
1974 break;
1975 }
1976 release_sock(sk);
1977 timeo = schedule_timeout(timeo);
1978 /* wakeup by sk_data_ready in smc_listen_work() */
1979 sched_annotate_sleep();
1980 lock_sock(sk);
1981 if (signal_pending(current)) {
1982 rc = sock_intr_errno(timeo);
1983 break;
1984 }
1985 }
1986 set_current_state(TASK_RUNNING);
1987 remove_wait_queue(sk_sleep(sk), &wait);
ac713874 1988
a046d57d
UB		 1989 if (!rc)
1990 rc = sock_error(nsk);
abb190f1
UB		 1991 release_sock(sk);
1992 if (rc)
1993 goto out;
1994
1995 if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1996 /* wait till data arrives on the socket */
1997 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1998 MSEC_PER_SEC);
1999 if (smc_sk(nsk)->use_fallback) {
2000 struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
2001
2002 lock_sock(clcsk);
2003 if (skb_queue_empty(&clcsk->sk_receive_queue))
2004 sk_wait_data(clcsk, &timeo, NULL);
2005 release_sock(clcsk);
2006 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
2007 lock_sock(nsk);
b51fa1b1 2008 smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
abb190f1
UB		 2009 release_sock(nsk);
2010 }
2011 }
ac713874
UB		 2012
2013out:
51f1de79 2014 sock_put(sk); /* sock_hold above */
ac713874
UB		 2015 return rc;
2016}
2017
2018static int smc_getname(struct socket *sock, struct sockaddr *addr,
9b2c45d4 2019 int peer)
ac713874
UB		 2020{
2021 struct smc_sock *smc;
2022
b38d7324
UB		 2023 if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
2024 (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
ac713874
UB		 2025 return -ENOTCONN;
2026
2027 smc = smc_sk(sock->sk);
2028
9b2c45d4 2029 return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
ac713874
UB		 2030}
2031
2032static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2033{
2034 struct sock *sk = sock->sk;
2035 struct smc_sock *smc;
2036 int rc = -EPIPE;
2037
2038 smc = smc_sk(sk);
2039 lock_sock(sk);
b38d7324
UB		 2040 if ((sk->sk_state != SMC_ACTIVE) &&
2041 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2042 (sk->sk_state != SMC_INIT))
ac713874 2043 goto out;
ee9dfbef
UB		 2044
2045 if (msg->msg_flags & MSG_FASTOPEN) {
cd206360 2046 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
e0e4b8fa 2047 smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
ee9dfbef
UB		 2048 } else {
2049 rc = -EINVAL;
2050 goto out;
2051 }
2052 }
2053
e0e4b8fa 2054 if (smc->use_fallback) {
ac713874 2055 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
e0e4b8fa 2056 } else {
e6727f39 2057 rc = smc_tx_sendmsg(smc, msg, len);
e0e4b8fa
GG		 2058 SMC_STAT_TX_PAYLOAD(smc, len, rc);
2059 }
ac713874
UB		 2060out:
2061 release_sock(sk);
2062 return rc;
2063}
2064
2065static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2066 int flags)
2067{
2068 struct sock *sk = sock->sk;
2069 struct smc_sock *smc;
2070 int rc = -ENOTCONN;
2071
2072 smc = smc_sk(sk);
2073 lock_sock(sk);
51c5aba3
KG		 2074 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2075 /* socket was connected before, no more data to read */
2076 rc = 0;
2077 goto out;
2078 }
b38d7324
UB		 2079 if ((sk->sk_state == SMC_INIT) ||
2080 (sk->sk_state == SMC_LISTEN) ||
2081 (sk->sk_state == SMC_CLOSED))
ac713874
UB		 2082 goto out;
2083
b38d7324
UB		 2084 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2085 rc = 0;
2086 goto out;
2087 }
2088
9014db20 2089 if (smc->use_fallback) {
ac713874 2090 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
9014db20
SR		 2091 } else {
2092 msg->msg_namelen = 0;
2093 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
e0e4b8fa 2094 SMC_STAT_RX_PAYLOAD(smc, rc, rc);
9014db20 2095 }
b38d7324 2096
ac713874
UB		 2097out:
2098 release_sock(sk);
2099 return rc;
2100}
2101
ade994f4 2102static __poll_t smc_accept_poll(struct sock *parent)
a046d57d 2103{
8dce2786 2104 struct smc_sock *isk = smc_sk(parent);
63e2480c 2105 __poll_t mask = 0;
a046d57d 2106
8dce2786
UB		 2107 spin_lock(&isk->accept_q_lock);
2108 if (!list_empty(&isk->accept_q))
a9a08845 2109 mask = EPOLLIN | EPOLLRDNORM;
8dce2786 2110 spin_unlock(&isk->accept_q_lock);
a046d57d 2111
8dce2786 2112 return mask;
a046d57d
UB		 2113}
2114
a11e1d43
LT		 2115static __poll_t smc_poll(struct file *file, struct socket *sock,
2116 poll_table *wait)
ac713874
UB		 2117{
2118 struct sock *sk = sock->sk;
ac713874 2119 struct smc_sock *smc;
50717a37 2120 __poll_t mask = 0;
ac713874 2121
8dce2786 2122 if (!sk)
a9a08845 2123 return EPOLLNVAL;
8dce2786 2124
ac713874 2125 smc = smc_sk(sock->sk);
648a5a7a 2126 if (smc->use_fallback) {
a046d57d 2127 /* delegate to CLC child sock */
a11e1d43 2128 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
784813ae 2129 sk->sk_err = smc->clcsock->sk->sk_err;
ac713874 2130 } else {
410da1e1 2131 if (sk->sk_state != SMC_CLOSED)
89ab066d 2132 sock_poll_wait(file, sock, wait);
a046d57d 2133 if (sk->sk_err)
a9a08845 2134 mask |= EPOLLERR;
b38d7324
UB		 2135 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2136 (sk->sk_state == SMC_CLOSED))
a9a08845 2137 mask |= EPOLLHUP;
8dce2786
UB		 2138 if (sk->sk_state == SMC_LISTEN) {
2139 /* woken up by sk_data_ready in smc_listen_work() */
50717a37
UB		 2140 mask |= smc_accept_poll(sk);
2141 } else if (smc->use_fallback) { /* as result of connect_work()*/
2142 mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2143 wait);
2144 sk->sk_err = smc->clcsock->sk->sk_err;
8dce2786 2145 } else {
50717a37
UB		 2146 if ((sk->sk_state != SMC_INIT &&
2147 atomic_read(&smc->conn.sndbuf_space)) ||
8dce2786 2148 sk->sk_shutdown & SEND_SHUTDOWN) {
a9a08845 2149 mask |= EPOLLOUT | EPOLLWRNORM;
8dce2786
UB		 2150 } else {
2151 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2152 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2153 }
2154 if (atomic_read(&smc->conn.bytes_to_rcv))
a9a08845 2155 mask |= EPOLLIN | EPOLLRDNORM;
8dce2786 2156 if (sk->sk_shutdown & RCV_SHUTDOWN)
a9a08845 2157 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
8dce2786 2158 if (sk->sk_state == SMC_APPCLOSEWAIT1)
a9a08845 2159 mask |= EPOLLIN;
71d117f5
KG		 2160 if (smc->conn.urg_state == SMC_URG_VALID)
2161 mask |= EPOLLPRI;
8dce2786 2162 }
ac713874
UB		 2163 }
2164
2165 return mask;
2166}
2167
2168static int smc_shutdown(struct socket *sock, int how)
2169{
2170 struct sock *sk = sock->sk;
76aa7d8c 2171 bool do_shutdown = true;
ac713874
UB		 2172 struct smc_sock *smc;
2173 int rc = -EINVAL;
76aa7d8c 2174 int old_state;
b38d7324 2175 int rc1 = 0;
ac713874
UB		 2176
2177 smc = smc_sk(sk);
2178
2179 if ((how < SHUT_RD) || (how > SHUT_RDWR))
b38d7324 2180 return rc;
ac713874
UB		 2181
2182 lock_sock(sk);
2183
2184 rc = -ENOTCONN;
caa21e19 2185 if ((sk->sk_state != SMC_ACTIVE) &&
b38d7324
UB		 2186 (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2187 (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2188 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2189 (sk->sk_state != SMC_APPCLOSEWAIT2) &&
2190 (sk->sk_state != SMC_APPFINCLOSEWAIT))
ac713874
UB		 2191 goto out;
2192 if (smc->use_fallback) {
2193 rc = kernel_sock_shutdown(smc->clcsock, how);
2194 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2195 if (sk->sk_shutdown == SHUTDOWN_MASK)
2196 sk->sk_state = SMC_CLOSED;
b38d7324 2197 goto out;
ac713874 2198 }
b38d7324
UB		 2199 switch (how) {
2200 case SHUT_RDWR: /* shutdown in both directions */
76aa7d8c 2201 old_state = sk->sk_state;
b38d7324 2202 rc = smc_close_active(smc);
76aa7d8c
TL		 2203 if (old_state == SMC_ACTIVE &&
2204 sk->sk_state == SMC_PEERCLOSEWAIT1)
2205 do_shutdown = false;
b38d7324
UB		 2206 break;
2207 case SHUT_WR:
2208 rc = smc_close_shutdown_write(smc);
2209 break;
2210 case SHUT_RD:
1255fcb2
UB		 2211 rc = 0;
2212 /* nothing more to do because peer is not involved */
b38d7324
UB		 2213 break;
2214 }
76aa7d8c 2215 if (do_shutdown && smc->clcsock)
1255fcb2 2216 rc1 = kernel_sock_shutdown(smc->clcsock, how);
b38d7324
UB		 2217 /* map sock_shutdown_cmd constants to sk_shutdown value range */
2218 sk->sk_shutdown |= how + 1;
ac713874
UB		 2219
2220out:
2221 release_sock(sk);
b38d7324 2222 return rc ? rc : rc1;
ac713874
UB		 2223}
2224
2225static int smc_setsockopt(struct socket *sock, int level, int optname,
a7b75c5a 2226 sockptr_t optval, unsigned int optlen)
ac713874
UB		 2227{
2228 struct sock *sk = sock->sk;
2229 struct smc_sock *smc;
01d2f7e2 2230 int val, rc;
ac713874 2231
86214366
CW		 2232 if (level == SOL_TCP && optname == TCP_ULP)
2233 return -EOPNOTSUPP;
2234
ac713874
UB		 2235 smc = smc_sk(sk);
2236
2237 /* generic setsockopts reaching us here always apply to the
2238 * CLC socket
2239 */
a44d9e72
CH		 2240 if (unlikely(!smc->clcsock->ops->setsockopt))
2241 rc = -EOPNOTSUPP;
2242 else
2243 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
2244 optval, optlen);
ee9dfbef
UB		 2245 if (smc->clcsock->sk->sk_err) {
2246 sk->sk_err = smc->clcsock->sk->sk_err;
e3ae2365 2247 sk_error_report(sk);
ee9dfbef 2248 }
ee9dfbef 2249
01d2f7e2 2250 if (optlen < sizeof(int))
3dc9f558 2251 return -EINVAL;
a7b75c5a 2252 if (copy_from_sockptr(&val, optval, sizeof(int)))
ac0107ed 2253 return -EFAULT;
01d2f7e2 2254
ee9dfbef 2255 lock_sock(sk);
86434744
UB		 2256 if (rc || smc->use_fallback)
2257 goto out;
ee9dfbef 2258 switch (optname) {
ee9dfbef
UB		 2259 case TCP_FASTOPEN:
2260 case TCP_FASTOPEN_CONNECT:
2261 case TCP_FASTOPEN_KEY:
2262 case TCP_FASTOPEN_NO_COOKIE:
2263 /* option not supported by SMC */
8204df72 2264 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
e0e4b8fa 2265 smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
ee9dfbef 2266 } else {
86434744 2267 rc = -EINVAL;
ee9dfbef
UB		 2268 }
2269 break;
01d2f7e2 2270 case TCP_NODELAY:
f9cedf1a
UB		 2271 if (sk->sk_state != SMC_INIT &&
2272 sk->sk_state != SMC_LISTEN &&
2273 sk->sk_state != SMC_CLOSED) {
e0e4b8fa 2274 if (val) {
194730a9 2275 SMC_STAT_INC(smc, ndly_cnt);
22ef473d
KG		 2276 mod_delayed_work(smc->conn.lgr->tx_wq,
2277 &smc->conn.tx_work, 0);
e0e4b8fa 2278 }
01d2f7e2
UB		 2279 }
2280 break;
2281 case TCP_CORK:
f9cedf1a
UB		 2282 if (sk->sk_state != SMC_INIT &&
2283 sk->sk_state != SMC_LISTEN &&
2284 sk->sk_state != SMC_CLOSED) {
e0e4b8fa 2285 if (!val) {
194730a9 2286 SMC_STAT_INC(smc, cork_cnt);
22ef473d
KG		 2287 mod_delayed_work(smc->conn.lgr->tx_wq,
2288 &smc->conn.tx_work, 0);
e0e4b8fa 2289 }
01d2f7e2
UB		 2290 }
2291 break;
abb190f1
UB		 2292 case TCP_DEFER_ACCEPT:
2293 smc->sockopt_defer_accept = val;
2294 break;
ee9dfbef
UB		 2295 default:
2296 break;
2297 }
86434744 2298out:
ee9dfbef
UB		 2299 release_sock(sk);
2300
2301 return rc;
ac713874
UB		 2302}
2303
2304static int smc_getsockopt(struct socket *sock, int level, int optname,
2305 char __user *optval, int __user *optlen)
2306{
2307 struct smc_sock *smc;
2308
2309 smc = smc_sk(sock->sk);
2310 /* socket options apply to the CLC socket */
a44d9e72
CH		 2311 if (unlikely(!smc->clcsock->ops->getsockopt))
2312 return -EOPNOTSUPP;
ac713874
UB		 2313 return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
2314 optval, optlen);
2315}
2316
2317static int smc_ioctl(struct socket *sock, unsigned int cmd,
2318 unsigned long arg)
2319{
de8474eb
SR		 2320 union smc_host_cursor cons, urg;
2321 struct smc_connection *conn;
ac713874 2322 struct smc_sock *smc;
9b67e26f 2323 int answ;
ac713874
UB		 2324
2325 smc = smc_sk(sock->sk);
de8474eb 2326 conn = &smc->conn;
7311d665 2327 lock_sock(&smc->sk);
9b67e26f 2328 if (smc->use_fallback) {
7311d665
UB		 2329 if (!smc->clcsock) {
2330 release_sock(&smc->sk);
9b67e26f 2331 return -EBADF;
7311d665
UB		 2332 }
2333 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
2334 release_sock(&smc->sk);
2335 return answ;
9b67e26f
UB		 2336 }
2337 switch (cmd) {
2338 case SIOCINQ: /* same as FIONREAD */
1992d998
UB		 2339 if (smc->sk.sk_state == SMC_LISTEN) {
2340 release_sock(&smc->sk);
9b67e26f 2341 return -EINVAL;
1992d998 2342 }
2351abe6
UB		 2343 if (smc->sk.sk_state == SMC_INIT ||
2344 smc->sk.sk_state == SMC_CLOSED)
2345 answ = 0;
2346 else
2347 answ = atomic_read(&smc->conn.bytes_to_rcv);
9b67e26f
UB		 2348 break;
2349 case SIOCOUTQ:
2350 /* output queue size (not send + not acked) */
1992d998
UB		 2351 if (smc->sk.sk_state == SMC_LISTEN) {
2352 release_sock(&smc->sk);
9b67e26f 2353 return -EINVAL;
1992d998 2354 }
2351abe6
UB		 2355 if (smc->sk.sk_state == SMC_INIT ||
2356 smc->sk.sk_state == SMC_CLOSED)
2357 answ = 0;
2358 else
2359 answ = smc->conn.sndbuf_desc->len -
9b67e26f
UB		 2360 atomic_read(&smc->conn.sndbuf_space);
2361 break;
2362 case SIOCOUTQNSD:
2363 /* output queue size (not send only) */
1992d998
UB		 2364 if (smc->sk.sk_state == SMC_LISTEN) {
2365 release_sock(&smc->sk);
9b67e26f 2366 return -EINVAL;
1992d998 2367 }
2351abe6
UB		 2368 if (smc->sk.sk_state == SMC_INIT ||
2369 smc->sk.sk_state == SMC_CLOSED)
2370 answ = 0;
2371 else
2372 answ = smc_tx_prepared_sends(&smc->conn);
9b67e26f 2373 break;
de8474eb 2374 case SIOCATMARK:
1992d998
UB		 2375 if (smc->sk.sk_state == SMC_LISTEN) {
2376 release_sock(&smc->sk);
de8474eb 2377 return -EINVAL;
1992d998 2378 }
de8474eb
SR		 2379 if (smc->sk.sk_state == SMC_INIT ||
2380 smc->sk.sk_state == SMC_CLOSED) {
2381 answ = 0;
2382 } else {
bac6de7b
SR		 2383 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
2384 smc_curs_copy(&urg, &conn->urg_curs, conn);
de8474eb
SR		 2385 answ = smc_curs_diff(conn->rmb_desc->len,
2386 &cons, &urg) == 1;
2387 }
2388 break;
9b67e26f 2389 default:
1992d998 2390 release_sock(&smc->sk);
9b67e26f
UB		 2391 return -ENOIOCTLCMD;
2392 }
1992d998 2393 release_sock(&smc->sk);
9b67e26f
UB		 2394
2395 return put_user(answ, (int __user *)arg);
ac713874
UB		 2396}
2397
2398static ssize_t smc_sendpage(struct socket *sock, struct page *page,
2399 int offset, size_t size, int flags)
2400{
2401 struct sock *sk = sock->sk;
2402 struct smc_sock *smc;
2403 int rc = -EPIPE;
2404
2405 smc = smc_sk(sk);
2406 lock_sock(sk);
bda27ff5
SR		 2407 if (sk->sk_state != SMC_ACTIVE) {
2408 release_sock(sk);
ac713874 2409 goto out;
bda27ff5
SR		 2410 }
2411 release_sock(sk);
e0e4b8fa 2412 if (smc->use_fallback) {
ac713874
UB		 2413 rc = kernel_sendpage(smc->clcsock, page, offset,
2414 size, flags);
e0e4b8fa 2415 } else {
194730a9 2416 SMC_STAT_INC(smc, sendpage_cnt);
ac713874 2417 rc = sock_no_sendpage(sock, page, offset, size, flags);
e0e4b8fa 2418 }
ac713874
UB		 2419
2420out:
ac713874
UB		 2421 return rc;
2422}
2423
9014db20
SR		 2424/* Map the affected portions of the rmbe into an spd, note the number of bytes
2425 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
2426 * updates till whenever a respective page has been fully processed.
2427 * Note that subsequent recv() calls have to wait till all splice() processing
2428 * completed.
2429 */
ac713874
UB		 2430static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
2431 struct pipe_inode_info *pipe, size_t len,
9014db20 2432 unsigned int flags)
ac713874
UB		 2433{
2434 struct sock *sk = sock->sk;
2435 struct smc_sock *smc;
2436 int rc = -ENOTCONN;
2437
2438 smc = smc_sk(sk);
2439 lock_sock(sk);
51c5aba3
KG		 2440 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2441 /* socket was connected before, no more data to read */
2442 rc = 0;
2443 goto out;
2444 }
9014db20
SR		 2445 if (sk->sk_state == SMC_INIT ||
2446 sk->sk_state == SMC_LISTEN ||
2447 sk->sk_state == SMC_CLOSED)
2448 goto out;
2449
2450 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2451 rc = 0;
ac713874 2452 goto out;
9014db20
SR		 2453 }
2454
ac713874
UB		 2455 if (smc->use_fallback) {
2456 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
2457 pipe, len, flags);
2458 } else {
9014db20
SR		 2459 if (*ppos) {
2460 rc = -ESPIPE;
2461 goto out;
2462 }
2463 if (flags & SPLICE_F_NONBLOCK)
2464 flags = MSG_DONTWAIT;
2465 else
2466 flags = 0;
194730a9 2467 SMC_STAT_INC(smc, splice_cnt);
9014db20 2468 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
ac713874
UB		 2469 }
2470out:
2471 release_sock(sk);
9014db20 2472
ac713874
UB		 2473 return rc;
2474}
2475
2476/* must look like tcp */
2477static const struct proto_ops smc_sock_ops = {
2478 .family = PF_SMC,
2479 .owner = THIS_MODULE,
2480 .release = smc_release,
2481 .bind = smc_bind,
2482 .connect = smc_connect,
2483 .socketpair = sock_no_socketpair,
2484 .accept = smc_accept,
2485 .getname = smc_getname,
a11e1d43 2486 .poll = smc_poll,
ac713874
UB		 2487 .ioctl = smc_ioctl,
2488 .listen = smc_listen,
2489 .shutdown = smc_shutdown,
2490 .setsockopt = smc_setsockopt,
2491 .getsockopt = smc_getsockopt,
2492 .sendmsg = smc_sendmsg,
2493 .recvmsg = smc_recvmsg,
2494 .mmap = sock_no_mmap,
2495 .sendpage = smc_sendpage,
2496 .splice_read = smc_splice_read,
2497};
2498
2499static int smc_create(struct net *net, struct socket *sock, int protocol,
2500 int kern)
2501{
aaa4d33f 2502 int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
ac713874
UB		 2503 struct smc_sock *smc;
2504 struct sock *sk;
2505 int rc;
2506
2507 rc = -ESOCKTNOSUPPORT;
2508 if (sock->type != SOCK_STREAM)
2509 goto out;
2510
2511 rc = -EPROTONOSUPPORT;
aaa4d33f 2512 if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
ac713874
UB		 2513 goto out;
2514
2515 rc = -ENOBUFS;
2516 sock->ops = &smc_sock_ops;
aaa4d33f 2517 sk = smc_sock_alloc(net, sock, protocol);
ac713874
UB		 2518 if (!sk)
2519 goto out;
2520
2521 /* create internal TCP socket for CLC handshake and fallback */
2522 smc = smc_sk(sk);
a046d57d 2523 smc->use_fallback = false; /* assume rdma capability first */
603cc149 2524 smc->fallback_rsn = 0;
aaa4d33f
KG		 2525 rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
2526 &smc->clcsock);
a5dcb73b 2527 if (rc) {
ac713874 2528 sk_common_release(sk);
a5dcb73b
DC		 2529 goto out;
2530 }
cd6851f3
UB		 2531 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2532 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
ac713874
UB		 2533
2534out:
2535 return rc;
2536}
2537
2538static const struct net_proto_family smc_sock_family_ops = {
2539 .family = PF_SMC,
2540 .owner = THIS_MODULE,
2541 .create = smc_create,
2542};
2543
64e28b52
HW		 2544unsigned int smc_net_id;
2545
2546static __net_init int smc_net_init(struct net *net)
2547{
2548 return smc_pnet_net_init(net);
2549}
2550
2551static void __net_exit smc_net_exit(struct net *net)
2552{
2553 smc_pnet_net_exit(net);
2554}
2555
194730a9
GG		 2556static __net_init int smc_net_stat_init(struct net *net)
2557{
2558 return smc_stats_init(net);
2559}
2560
2561static void __net_exit smc_net_stat_exit(struct net *net)
2562{
2563 smc_stats_exit(net);
2564}
2565
64e28b52
HW		 2566static struct pernet_operations smc_net_ops = {
2567 .init = smc_net_init,
2568 .exit = smc_net_exit,
2569 .id = &smc_net_id,
2570 .size = sizeof(struct smc_net),
2571};
2572
194730a9
GG		 2573static struct pernet_operations smc_net_stat_ops = {
2574 .init = smc_net_stat_init,
2575 .exit = smc_net_stat_exit,
2576};
2577
ac713874
UB		 2578static int __init smc_init(void)
2579{
2580 int rc;
2581
64e28b52
HW		 2582 rc = register_pernet_subsys(&smc_net_ops);
2583 if (rc)
2584 return rc;
2585
194730a9
GG		 2586 rc = register_pernet_subsys(&smc_net_stat_ops);
2587 if (rc)
2588 return rc;
2589
201091eb 2590 smc_ism_init();
b81a5eb7 2591 smc_clc_init();
201091eb 2592
e8372d9d 2593 rc = smc_nl_init();
6812baab 2594 if (rc)
8c33bf1b 2595 goto out_pernet_subsys;
6812baab 2596
e8372d9d
GG		 2597 rc = smc_pnet_init();
2598 if (rc)
2599 goto out_nl;
2600
22ef473d
KG		 2601 rc = -ENOMEM;
2602 smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
2603 if (!smc_hs_wq)
2604 goto out_pnet;
2605
2606 smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
2607 if (!smc_close_wq)
2608 goto out_alloc_hs_wq;
2609
6dabd405
UB		 2610 rc = smc_core_init();
2611 if (rc) {
2612 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
194730a9 2613 goto out_alloc_wqs;
6dabd405
UB		 2614 }
2615
9bf9abea
UB		 2616 rc = smc_llc_init();
2617 if (rc) {
2618 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
6dabd405 2619 goto out_core;
9bf9abea
UB		 2620 }
2621
5f08318f
UB		 2622 rc = smc_cdc_init();
2623 if (rc) {
2624 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
6dabd405 2625 goto out_core;
5f08318f
UB		 2626 }
2627
ac713874
UB		 2628 rc = proto_register(&smc_proto, 1);
2629 if (rc) {
aaa4d33f 2630 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
6dabd405 2631 goto out_core;
ac713874
UB		 2632 }
2633
aaa4d33f
KG		 2634 rc = proto_register(&smc_proto6, 1);
2635 if (rc) {
2636 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2637 goto out_proto;
2638 }
2639
ac713874
UB		 2640 rc = sock_register(&smc_sock_family_ops);
2641 if (rc) {
2642 pr_err("%s: sock_register fails with %d\n", __func__, rc);
aaa4d33f 2643 goto out_proto6;
ac713874 2644 }
f16a7dd5 2645 INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
aaa4d33f 2646 INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
ac713874 2647
a4cf0443
UB		 2648 rc = smc_ib_register_client();
2649 if (rc) {
2650 pr_err("%s: ib_register fails with %d\n", __func__, rc);
2651 goto out_sock;
2652 }
2653
c5c1cc9c 2654 static_branch_enable(&tcp_have_smc);
ac713874
UB		 2655 return 0;
2656
a4cf0443
UB		 2657out_sock:
2658 sock_unregister(PF_SMC);
aaa4d33f
KG		 2659out_proto6:
2660 proto_unregister(&smc_proto6);
ac713874
UB		 2661out_proto:
2662 proto_unregister(&smc_proto);
6dabd405
UB		 2663out_core:
2664 smc_core_exit();
22ef473d
KG		 2665out_alloc_wqs:
2666 destroy_workqueue(smc_close_wq);
2667out_alloc_hs_wq:
2668 destroy_workqueue(smc_hs_wq);
6812baab
TR		 2669out_pnet:
2670 smc_pnet_exit();
e8372d9d
GG		 2671out_nl:
2672 smc_nl_exit();
8c33bf1b
Y		 2673out_pernet_subsys:
2674 unregister_pernet_subsys(&smc_net_ops);
2675
ac713874
UB		 2676 return rc;
2677}
2678
2679static void __exit smc_exit(void)
2680{
c5c1cc9c 2681 static_branch_disable(&tcp_have_smc);
ac713874 2682 sock_unregister(PF_SMC);
6dabd405
UB		 2683 smc_core_exit();
2684 smc_ib_unregister_client();
22ef473d
KG		 2685 destroy_workqueue(smc_close_wq);
2686 destroy_workqueue(smc_hs_wq);
aaa4d33f 2687 proto_unregister(&smc_proto6);
ac713874 2688 proto_unregister(&smc_proto);
6812baab 2689 smc_pnet_exit();
e8372d9d 2690 smc_nl_exit();
194730a9 2691 unregister_pernet_subsys(&smc_net_stat_ops);
64e28b52 2692 unregister_pernet_subsys(&smc_net_ops);
4ead9c96 2693 rcu_barrier();
ac713874
UB		 2694}
2695
2696module_init(smc_init);
2697module_exit(smc_exit);
2698
2699MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
2700MODULE_DESCRIPTION("smc socket address family");
2701MODULE_LICENSE("GPL");
2702MODULE_ALIAS_NETPROTO(PF_SMC);
Ubuntu Jammy Linux kernel mirror