]> git.proxmox.com Git - mirror_ubuntu-jammy-kernel.git/blame - net/smc/af_smc.c
projects / mirror_ubuntu-jammy-kernel.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/smc: Send directly when TCP_CORK is cleared
[mirror_ubuntu-jammy-kernel.git] / net / smc / af_smc.c
CommitLineData
09c434b8 1// SPDX-License-Identifier: GPL-2.0-only
ac713874
UB		 2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
6 * applies to SOCK_STREAM sockets only
7 * offers an alternative communication option for TCP-protocol sockets
8 * applicable with RoCE-cards only
9 *
a046d57d 10 * Initial restrictions:
a046d57d 11 * - support for alternate links postponed
a046d57d 12 *
aaa4d33f 13 * Copyright IBM Corp. 2016, 2018
ac713874
UB		 14 *
15 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
16 * based on prototype from Frank Blaschka
17 */
18
19#define KMSG_COMPONENT "smc"
20#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21
22#include <linux/module.h>
23#include <linux/socket.h>
a046d57d 24#include <linux/workqueue.h>
5f08318f 25#include <linux/in.h>
c3edc401 26#include <linux/sched/signal.h>
41349844 27#include <linux/if_vlan.h>
4ead9c96 28#include <linux/rcupdate_wait.h>
b81a5eb7 29#include <linux/ctype.h>
c3edc401 30
ac713874 31#include <net/sock.h>
a046d57d 32#include <net/tcp.h>
f16a7dd5 33#include <net/smc.h>
9b67e26f 34#include <asm/ioctls.h>
ac713874 35
64e28b52
HW		 36#include <net/net_namespace.h>
37#include <net/netns/generic.h>
38#include "smc_netns.h"
39
ac713874 40#include "smc.h"
a046d57d 41#include "smc_clc.h"
9bf9abea 42#include "smc_llc.h"
5f08318f 43#include "smc_cdc.h"
0cfdd8f9 44#include "smc_core.h"
a4cf0443 45#include "smc_ib.h"
41349844 46#include "smc_ism.h"
6812baab 47#include "smc_pnet.h"
e8372d9d 48#include "smc_netlink.h"
e6727f39 49#include "smc_tx.h"
952310cc 50#include "smc_rx.h"
b38d7324 51#include "smc_close.h"
e0e4b8fa 52#include "smc_stats.h"
ac713874 53
72a36a8a
HW		 54static DEFINE_MUTEX(smc_server_lgr_pending); /* serialize link group
55 * creation on server
56 */
57static DEFINE_MUTEX(smc_client_lgr_pending); /* serialize link group
58 * creation on client
0cfdd8f9
UB		 59 */
60
22ef473d
KG		 61struct workqueue_struct *smc_hs_wq; /* wq for handshake work */
62struct workqueue_struct *smc_close_wq; /* wq for close work */
63
a046d57d 64static void smc_tcp_listen_work(struct work_struct *);
24ac3a08 65static void smc_connect_work(struct work_struct *);
a046d57d 66
ac713874
UB		 67static void smc_set_keepalive(struct sock *sk, int val)
68{
69 struct smc_sock *smc = smc_sk(sk);
70
71 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
72}
73
f16a7dd5
UB		 74static struct smc_hashinfo smc_v4_hashinfo = {
75 .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
76};
77
aaa4d33f
KG		 78static struct smc_hashinfo smc_v6_hashinfo = {
79 .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
80};
81
f16a7dd5
UB		 82int smc_hash_sk(struct sock *sk)
83{
84 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
85 struct hlist_head *head;
86
87 head = &h->ht;
88
89 write_lock_bh(&h->lock);
90 sk_add_node(sk, head);
91 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
92 write_unlock_bh(&h->lock);
93
94 return 0;
95}
96EXPORT_SYMBOL_GPL(smc_hash_sk);
97
98void smc_unhash_sk(struct sock *sk)
99{
100 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
101
102 write_lock_bh(&h->lock);
103 if (sk_del_node_init(sk))
104 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
105 write_unlock_bh(&h->lock);
106}
107EXPORT_SYMBOL_GPL(smc_unhash_sk);
108
109struct proto smc_proto = {
ac713874
UB		 110 .name = "SMC",
111 .owner = THIS_MODULE,
112 .keepalive = smc_set_keepalive,
f16a7dd5
UB		 113 .hash = smc_hash_sk,
114 .unhash = smc_unhash_sk,
ac713874 115 .obj_size = sizeof(struct smc_sock),
f16a7dd5 116 .h.smc_hash = &smc_v4_hashinfo,
5f0d5a3a 117 .slab_flags = SLAB_TYPESAFE_BY_RCU,
ac713874 118};
f16a7dd5 119EXPORT_SYMBOL_GPL(smc_proto);
ac713874 120
aaa4d33f
KG		 121struct proto smc_proto6 = {
122 .name = "SMC6",
123 .owner = THIS_MODULE,
124 .keepalive = smc_set_keepalive,
125 .hash = smc_hash_sk,
126 .unhash = smc_unhash_sk,
127 .obj_size = sizeof(struct smc_sock),
128 .h.smc_hash = &smc_v6_hashinfo,
129 .slab_flags = SLAB_TYPESAFE_BY_RCU,
130};
131EXPORT_SYMBOL_GPL(smc_proto6);
132
f536dffc
UB		 133static void smc_restore_fallback_changes(struct smc_sock *smc)
134{
1ad24058
KG		 135 if (smc->clcsock->file) { /* non-accepted sockets have no file yet */
136 smc->clcsock->file->private_data = smc->sk.sk_socket;
137 smc->clcsock->file = NULL;
138 }
f536dffc
UB		 139}
140
39f41f36 141static int __smc_release(struct smc_sock *smc)
ac713874 142{
39f41f36 143 struct sock *sk = &smc->sk;
b38d7324 144 int rc = 0;
ac713874 145
51f1de79 146 if (!smc->use_fallback) {
b38d7324
UB		 147 rc = smc_close_active(smc);
148 sock_set_flag(sk, SOCK_DEAD);
149 sk->sk_shutdown |= SHUTDOWN_MASK;
b03faa1f 150 } else {
0cad438e
DL		 151 if (sk->sk_state != SMC_CLOSED) {
152 if (sk->sk_state != SMC_LISTEN &&
153 sk->sk_state != SMC_INIT)
154 sock_put(sk); /* passive closing */
155 if (sk->sk_state == SMC_LISTEN) {
156 /* wake up clcsock accept */
157 rc = kernel_sock_shutdown(smc->clcsock,
158 SHUT_RDWR);
159 }
160 sk->sk_state = SMC_CLOSED;
161 sk->sk_state_change(sk);
78abe3d0 162 }
f536dffc 163 smc_restore_fallback_changes(smc);
51f1de79 164 }
ac713874 165
b03faa1f
UB		 166 sk->sk_prot->unhash(sk);
167
168 if (sk->sk_state == SMC_CLOSED) {
169 if (smc->clcsock) {
fd57770d
KG		 170 release_sock(sk);
171 smc_clcsock_release(smc);
172 lock_sock(sk);
b03faa1f
UB		 173 }
174 if (!smc->use_fallback)
175 smc_conn_free(&smc->conn);
176 }
177
39f41f36
UB		 178 return rc;
179}
180
181static int smc_release(struct socket *sock)
182{
183 struct sock *sk = sock->sk;
184 struct smc_sock *smc;
f1274744 185 int old_state, rc = 0;
39f41f36
UB		 186
187 if (!sk)
188 goto out;
189
81cf4f47 190 sock_hold(sk); /* sock_put below */
39f41f36
UB		 191 smc = smc_sk(sk);
192
f1274744
W		 193 old_state = sk->sk_state;
194
39f41f36 195 /* cleanup for a dangling non-blocking connect */
f1274744 196 if (smc->connect_nonblock && old_state == SMC_INIT)
39f41f36 197 tcp_abort(smc->clcsock->sk, ECONNABORTED);
98cd11a6
W		 198
199 if (cancel_work_sync(&smc->connect_work))
200 sock_put(&smc->sk); /* sock_hold in smc_connect for passive closing */
39f41f36
UB		 201
202 if (sk->sk_state == SMC_LISTEN)
203 /* smc_close_non_accepted() is called and acquires
204 * sock lock for child sockets again
205 */
206 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
207 else
208 lock_sock(sk);
209
f1274744
W		 210 if (old_state == SMC_INIT && sk->sk_state == SMC_ACTIVE &&
211 !smc->use_fallback)
212 smc_close_active_abort(smc);
213
39f41f36
UB		 214 rc = __smc_release(smc);
215
ac713874
UB		 216 /* detach socket */
217 sock_orphan(sk);
218 sock->sk = NULL;
219 release_sock(sk);
220
81cf4f47 221 sock_put(sk); /* sock_hold above */
51f1de79 222 sock_put(sk); /* final sock_put */
ac713874 223out:
b38d7324 224 return rc;
ac713874
UB		 225}
226
227static void smc_destruct(struct sock *sk)
228{
229 if (sk->sk_state != SMC_CLOSED)
230 return;
231 if (!sock_flag(sk, SOCK_DEAD))
232 return;
233
234 sk_refcnt_debug_dec(sk);
235}
236
aaa4d33f
KG		 237static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
238 int protocol)
ac713874
UB		 239{
240 struct smc_sock *smc;
aaa4d33f 241 struct proto *prot;
ac713874
UB		 242 struct sock *sk;
243
aaa4d33f
KG		 244 prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
245 sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
ac713874
UB		 246 if (!sk)
247 return NULL;
248
249 sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
250 sk->sk_state = SMC_INIT;
251 sk->sk_destruct = smc_destruct;
aaa4d33f 252 sk->sk_protocol = protocol;
ac713874 253 smc = smc_sk(sk);
a046d57d 254 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
24ac3a08 255 INIT_WORK(&smc->connect_work, smc_connect_work);
be7f3e59 256 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
a046d57d
UB		 257 INIT_LIST_HEAD(&smc->accept_q);
258 spin_lock_init(&smc->accept_q_lock);
be7f3e59 259 spin_lock_init(&smc->conn.send_lock);
f16a7dd5 260 sk->sk_prot->hash(sk);
a046d57d 261 sk_refcnt_debug_inc(sk);
78abe3d0 262 mutex_init(&smc->clcsock_release_lock);
ac713874
UB		 263
264 return sk;
265}
266
267static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
268 int addr_len)
269{
270 struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
271 struct sock *sk = sock->sk;
272 struct smc_sock *smc;
273 int rc;
274
275 smc = smc_sk(sk);
276
277 /* replicate tests from inet_bind(), to be safe wrt. future changes */
278 rc = -EINVAL;
279 if (addr_len < sizeof(struct sockaddr_in))
280 goto out;
281
282 rc = -EAFNOSUPPORT;
aaa4d33f
KG		 283 if (addr->sin_family != AF_INET &&
284 addr->sin_family != AF_INET6 &&
285 addr->sin_family != AF_UNSPEC)
286 goto out;
ac713874 287 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
aaa4d33f
KG		 288 if (addr->sin_family == AF_UNSPEC &&
289 addr->sin_addr.s_addr != htonl(INADDR_ANY))
ac713874
UB		 290 goto out;
291
292 lock_sock(sk);
293
294 /* Check if socket is already active */
295 rc = -EINVAL;
cd206360 296 if (sk->sk_state != SMC_INIT || smc->connect_nonblock)
ac713874
UB		 297 goto out_rel;
298
299 smc->clcsock->sk->sk_reuse = sk->sk_reuse;
300 rc = kernel_bind(smc->clcsock, uaddr, addr_len);
301
302out_rel:
303 release_sock(sk);
304out:
305 return rc;
306}
307
308static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
309 unsigned long mask)
310{
311 /* options we don't get control via setsockopt for */
312 nsk->sk_type = osk->sk_type;
313 nsk->sk_sndbuf = osk->sk_sndbuf;
314 nsk->sk_rcvbuf = osk->sk_rcvbuf;
315 nsk->sk_sndtimeo = osk->sk_sndtimeo;
316 nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
317 nsk->sk_mark = osk->sk_mark;
318 nsk->sk_priority = osk->sk_priority;
319 nsk->sk_rcvlowat = osk->sk_rcvlowat;
320 nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
321 nsk->sk_err = osk->sk_err;
322
323 nsk->sk_flags &= ~mask;
324 nsk->sk_flags |= osk->sk_flags & mask;
325}
326
327#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
328 (1UL << SOCK_KEEPOPEN) | \
329 (1UL << SOCK_LINGER) | \
330 (1UL << SOCK_BROADCAST) | \
331 (1UL << SOCK_TIMESTAMP) | \
332 (1UL << SOCK_DBG) | \
333 (1UL << SOCK_RCVTSTAMP) | \
334 (1UL << SOCK_RCVTSTAMPNS) | \
335 (1UL << SOCK_LOCALROUTE) | \
336 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
337 (1UL << SOCK_RXQ_OVFL) | \
338 (1UL << SOCK_WIFI_STATUS) | \
339 (1UL << SOCK_NOFCS) | \
9718475e
DD		 340 (1UL << SOCK_FILTER_LOCKED) | \
341 (1UL << SOCK_TSTAMP_NEW))
ac713874
UB		 342/* copy only relevant settings and flags of SOL_SOCKET level from smc to
343 * clc socket (since smc is not called for these options from net/core)
344 */
345static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
346{
347 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
348}
349
350#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
351 (1UL << SOCK_KEEPOPEN) | \
352 (1UL << SOCK_LINGER) | \
353 (1UL << SOCK_DBG))
354/* copy only settings and flags relevant for smc from clc to smc socket */
355static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
356{
357 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
358}
359
b9247544 360/* register the new rmb on all links */
7562a13d 361static int smcr_lgr_reg_rmbs(struct smc_link *link,
b9247544
KG		 362 struct smc_buf_desc *rmb_desc)
363{
7562a13d
KG		 364 struct smc_link_group *lgr = link->lgr;
365 int i, rc = 0;
b9247544 366
d5500667
KG		 367 rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
368 if (rc)
369 return rc;
370 /* protect against parallel smc_llc_cli_rkey_exchange() and
371 * parallel smcr_link_reg_rmb()
372 */
373 mutex_lock(&lgr->llc_conf_mutex);
b9247544 374 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
741a49a4 375 if (!smc_link_active(&lgr->lnk[i]))
b9247544 376 continue;
7562a13d 377 rc = smcr_link_reg_rmb(&lgr->lnk[i], rmb_desc);
b9247544 378 if (rc)
7562a13d 379 goto out;
44aa81ce 380 }
7562a13d
KG		 381
382 /* exchange confirm_rkey msg with peer */
383 rc = smc_llc_do_confirm_rkey(link, rmb_desc);
384 if (rc) {
385 rc = -EFAULT;
386 goto out;
387 }
388 rmb_desc->is_conf_rkey = true;
389out:
d5500667
KG		 390 mutex_unlock(&lgr->llc_conf_mutex);
391 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
7562a13d 392 return rc;
e63a5f8c
KG		 393}
394
b9247544 395static int smcr_clnt_conf_first_link(struct smc_sock *smc)
9bf9abea 396{
387707fd 397 struct smc_link *link = smc->conn.lnk;
0fb0b02b 398 struct smc_llc_qentry *qentry;
9bf9abea
UB		 399 int rc;
400
9bf9abea 401 /* receive CONFIRM LINK request from server over RoCE fabric */
0fb0b02b
KG		 402 qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
403 SMC_LLC_CONFIRM_LINK);
404 if (!qentry) {
9bf9abea
UB		 405 struct smc_clc_msg_decline dclc;
406
407 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
2b59f58e 408 SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
9ed28556 409 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
9bf9abea 410 }
649758ff 411 smc_llc_save_peer_uid(qentry);
0fb0b02b
KG		 412 rc = smc_llc_eval_conf_link(qentry, SMC_LLC_REQ);
413 smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
414 if (rc)
75d320d6
KG		 415 return SMC_CLC_DECL_RMBE_EC;
416
9bf9abea
UB		 417 rc = smc_ib_modify_qp_rts(link);
418 if (rc)
603cc149 419 return SMC_CLC_DECL_ERR_RDYLNK;
9bf9abea
UB		 420
421 smc_wr_remember_qp_attr(link);
652a1e41 422
7562a13d 423 if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
603cc149 424 return SMC_CLC_DECL_ERR_REGRMB;
652a1e41 425
0fb0b02b
KG		 426 /* confirm_rkey is implicit on 1st contact */
427 smc->conn.rmb_desc->is_conf_rkey = true;
428
9bf9abea 429 /* send CONFIRM LINK response over RoCE fabric */
947541f3 430 rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
9bf9abea 431 if (rc < 0)
603cc149 432 return SMC_CLC_DECL_TIMEOUT_CL;
9bf9abea 433
0fb0b02b 434 smc_llc_link_active(link);
0a99be43 435 smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
0fb0b02b
KG		 436
437 /* optional 2nd link, receive ADD LINK request from server */
438 qentry = smc_llc_wait(link->lgr, NULL, SMC_LLC_WAIT_TIME,
439 SMC_LLC_ADD_LINK);
440 if (!qentry) {
52bedf37
KG		 441 struct smc_clc_msg_decline dclc;
442
443 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
2b59f58e 444 SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
0fb0b02b
KG		 445 if (rc == -EAGAIN)
446 rc = 0; /* no DECLINE received, go with one link */
447 return rc;
52bedf37 448 }
0fb0b02b 449 smc_llc_flow_qentry_clr(&link->lgr->llc_flow_lcl);
b1570a87 450 smc_llc_cli_add_link(link, qentry);
75d320d6 451 return 0;
9bf9abea
UB		 452}
453
ea9478b7
KG		 454static bool smc_isascii(char *hostname)
455{
456 int i;
457
458 for (i = 0; i < SMC_MAX_HOSTNAME_LEN; i++)
459 if (!isascii(hostname[i]))
460 return false;
461 return true;
462}
463
464static void smc_conn_save_peer_info_fce(struct smc_sock *smc,
465 struct smc_clc_msg_accept_confirm *clc)
466{
467 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
468 (struct smc_clc_msg_accept_confirm_v2 *)clc;
469 struct smc_clc_first_contact_ext *fce;
470 int clc_v2_len;
471
472 if (clc->hdr.version == SMC_V1 ||
473 !(clc->hdr.typev2 & SMC_FIRST_CONTACT_MASK))
474 return;
475
476 if (smc->conn.lgr->is_smcd) {
477 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->d1.eid,
478 SMC_MAX_EID_LEN);
479 clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
480 d1);
481 } else {
482 memcpy(smc->conn.lgr->negotiated_eid, clc_v2->r1.eid,
483 SMC_MAX_EID_LEN);
484 clc_v2_len = offsetofend(struct smc_clc_msg_accept_confirm_v2,
485 r1);
486 }
487 fce = (struct smc_clc_first_contact_ext *)(((u8 *)clc_v2) + clc_v2_len);
488 smc->conn.lgr->peer_os = fce->os_type;
489 smc->conn.lgr->peer_smc_release = fce->release;
490 if (smc_isascii(fce->hostname))
491 memcpy(smc->conn.lgr->peer_hostname, fce->hostname,
492 SMC_MAX_HOSTNAME_LEN);
493}
494
41349844
HW		 495static void smcr_conn_save_peer_info(struct smc_sock *smc,
496 struct smc_clc_msg_accept_confirm *clc)
0cfdd8f9 497{
3d9725a6 498 int bufsize = smc_uncompress_bufsize(clc->r0.rmbe_size);
95d8d263 499
3d9725a6
UB		 500 smc->conn.peer_rmbe_idx = clc->r0.rmbe_idx;
501 smc->conn.local_tx_ctrl.token = ntohl(clc->r0.rmbe_alert_token);
95d8d263 502 smc->conn.peer_rmbe_size = bufsize;
cd6851f3 503 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
95d8d263 504 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
0cfdd8f9
UB		 505}
506
41349844
HW		 507static void smcd_conn_save_peer_info(struct smc_sock *smc,
508 struct smc_clc_msg_accept_confirm *clc)
509{
3d9725a6 510 int bufsize = smc_uncompress_bufsize(clc->d0.dmbe_size);
41349844 511
3d9725a6
UB		 512 smc->conn.peer_rmbe_idx = clc->d0.dmbe_idx;
513 smc->conn.peer_token = clc->d0.token;
41349844
HW		 514 /* msg header takes up space in the buffer */
515 smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
516 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
517 smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
518}
519
520static void smc_conn_save_peer_info(struct smc_sock *smc,
521 struct smc_clc_msg_accept_confirm *clc)
522{
523 if (smc->conn.lgr->is_smcd)
524 smcd_conn_save_peer_info(smc, clc);
525 else
526 smcr_conn_save_peer_info(smc, clc);
ea9478b7 527 smc_conn_save_peer_info_fce(smc, clc);
41349844
HW		 528}
529
0cfdd8f9 530static void smc_link_save_peer_info(struct smc_link *link,
ea9478b7
KG		 531 struct smc_clc_msg_accept_confirm *clc,
532 struct smc_init_info *ini)
0cfdd8f9 533{
3d9725a6 534 link->peer_qpn = ntoh24(clc->r0.qpn);
ea9478b7
KG		 535 memcpy(link->peer_gid, ini->peer_gid, SMC_GID_SIZE);
536 memcpy(link->peer_mac, ini->peer_mac, sizeof(link->peer_mac));
3d9725a6
UB		 537 link->peer_psn = ntoh24(clc->r0.psn);
538 link->peer_mtu = clc->r0.qp_mtu;
0cfdd8f9
UB		 539}
540
e0e4b8fa
GG		 541static void smc_stat_inc_fback_rsn_cnt(struct smc_sock *smc,
542 struct smc_stats_fback *fback_arr)
543{
544 int cnt;
545
546 for (cnt = 0; cnt < SMC_MAX_FBACK_RSN_CNT; cnt++) {
547 if (fback_arr[cnt].fback_code == smc->fallback_rsn) {
548 fback_arr[cnt].count++;
549 break;
550 }
551 if (!fback_arr[cnt].fback_code) {
552 fback_arr[cnt].fback_code = smc->fallback_rsn;
553 fback_arr[cnt].count++;
554 break;
555 }
556 }
557}
558
559static void smc_stat_fallback(struct smc_sock *smc)
560{
194730a9
GG		 561 struct net *net = sock_net(&smc->sk);
562
563 mutex_lock(&net->smc.mutex_fback_rsn);
e0e4b8fa 564 if (smc->listen_smc) {
194730a9
GG		 565 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->srv);
566 net->smc.fback_rsn->srv_fback_cnt++;
e0e4b8fa 567 } else {
194730a9
GG		 568 smc_stat_inc_fback_rsn_cnt(smc, net->smc.fback_rsn->clnt);
569 net->smc.fback_rsn->clnt_fback_cnt++;
e0e4b8fa 570 }
194730a9 571 mutex_unlock(&net->smc.mutex_fback_rsn);
e0e4b8fa
GG		 572}
573
1ddd4b02
WG		 574/* must be called under rcu read lock */
575static void smc_fback_wakeup_waitqueue(struct smc_sock *smc, void *key)
576{
577 struct socket_wq *wq;
578 __poll_t flags;
579
580 wq = rcu_dereference(smc->sk.sk_wq);
581 if (!skwq_has_sleeper(wq))
582 return;
583
584 /* wake up smc sk->sk_wq */
585 if (!key) {
586 /* sk_state_change */
587 wake_up_interruptible_all(&wq->wait);
588 } else {
589 flags = key_to_poll(key);
590 if (flags & (EPOLLIN | EPOLLOUT))
591 /* sk_data_ready or sk_write_space */
592 wake_up_interruptible_sync_poll(&wq->wait, flags);
593 else if (flags & EPOLLERR)
594 /* sk_error_report */
595 wake_up_interruptible_poll(&wq->wait, flags);
596 }
597}
598
599static int smc_fback_mark_woken(wait_queue_entry_t *wait,
600 unsigned int mode, int sync, void *key)
601{
602 struct smc_mark_woken *mark =
603 container_of(wait, struct smc_mark_woken, wait_entry);
604
605 mark->woken = true;
606 mark->key = key;
607 return 0;
608}
609
610static void smc_fback_forward_wakeup(struct smc_sock *smc, struct sock *clcsk,
611 void (*clcsock_callback)(struct sock *sk))
612{
613 struct smc_mark_woken mark = { .woken = false };
614 struct socket_wq *wq;
615
616 init_waitqueue_func_entry(&mark.wait_entry,
617 smc_fback_mark_woken);
618 rcu_read_lock();
619 wq = rcu_dereference(clcsk->sk_wq);
620 if (!wq)
621 goto out;
622 add_wait_queue(sk_sleep(clcsk), &mark.wait_entry);
623 clcsock_callback(clcsk);
624 remove_wait_queue(sk_sleep(clcsk), &mark.wait_entry);
625
626 if (mark.woken)
627 smc_fback_wakeup_waitqueue(smc, mark.key);
628out:
629 rcu_read_unlock();
630}
631
632static void smc_fback_state_change(struct sock *clcsk)
633{
634 struct smc_sock *smc =
635 smc_clcsock_user_data(clcsk);
636
637 if (!smc)
638 return;
639 smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_state_change);
640}
641
642static void smc_fback_data_ready(struct sock *clcsk)
643{
644 struct smc_sock *smc =
645 smc_clcsock_user_data(clcsk);
646
647 if (!smc)
648 return;
649 smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_data_ready);
650}
651
652static void smc_fback_write_space(struct sock *clcsk)
653{
654 struct smc_sock *smc =
655 smc_clcsock_user_data(clcsk);
656
657 if (!smc)
658 return;
659 smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_write_space);
660}
661
662static void smc_fback_error_report(struct sock *clcsk)
663{
664 struct smc_sock *smc =
665 smc_clcsock_user_data(clcsk);
666
667 if (!smc)
668 return;
669 smc_fback_forward_wakeup(smc, clcsk, smc->clcsk_error_report);
670}
671
3924702d 672static int smc_switch_to_fallback(struct smc_sock *smc, int reason_code)
07603b23 673{
1ddd4b02 674 struct sock *clcsk;
90edcd88 675 int rc = 0;
bd6b2124 676
3924702d
WG		 677 mutex_lock(&smc->clcsock_release_lock);
678 if (!smc->clcsock) {
90edcd88
WG		 679 rc = -EBADF;
680 goto out;
3924702d 681 }
1ddd4b02
WG		 682 clcsk = smc->clcsock->sk;
683
90edcd88
WG		 684 if (smc->use_fallback)
685 goto out;
07603b23 686 smc->use_fallback = true;
e0e4b8fa
GG		 687 smc->fallback_rsn = reason_code;
688 smc_stat_fallback(smc);
07603b23
UB		 689 if (smc->sk.sk_socket && smc->sk.sk_socket->file) {
690 smc->clcsock->file = smc->sk.sk_socket->file;
691 smc->clcsock->file->private_data = smc->clcsock;
67f562e3
UB		 692 smc->clcsock->wq.fasync_list =
693 smc->sk.sk_socket->wq.fasync_list;
bd6b2124 694
1ddd4b02
WG		 695 /* There might be some wait entries remaining
696 * in smc sk->sk_wq and they should be woken up
697 * as clcsock's wait queue is woken up.
bd6b2124 698 */
1ddd4b02
WG		 699 smc->clcsk_state_change = clcsk->sk_state_change;
700 smc->clcsk_data_ready = clcsk->sk_data_ready;
701 smc->clcsk_write_space = clcsk->sk_write_space;
702 smc->clcsk_error_report = clcsk->sk_error_report;
703
704 clcsk->sk_state_change = smc_fback_state_change;
705 clcsk->sk_data_ready = smc_fback_data_ready;
706 clcsk->sk_write_space = smc_fback_write_space;
707 clcsk->sk_error_report = smc_fback_error_report;
708
709 smc->clcsock->sk->sk_user_data =
710 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
07603b23 711 }
90edcd88 712out:
3924702d 713 mutex_unlock(&smc->clcsock_release_lock);
90edcd88 714 return rc;
07603b23
UB		 715}
716
3b2dec26 717/* fall back during connect */
603cc149 718static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
a046d57d 719{
3924702d
WG		 720 struct net *net = sock_net(&smc->sk);
721 int rc = 0;
722
723 rc = smc_switch_to_fallback(smc, reason_code);
724 if (rc) { /* fallback fails */
725 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
726 if (smc->sk.sk_state == SMC_INIT)
727 sock_put(&smc->sk); /* passive closing */
728 return rc;
729 }
3b2dec26 730 smc_copy_sock_settings_to_clc(smc);
50717a37 731 smc->connect_nonblock = 0;
3b2dec26
HW		 732 if (smc->sk.sk_state == SMC_INIT)
733 smc->sk.sk_state = SMC_ACTIVE;
734 return 0;
735}
51f1de79 736
3b2dec26 737/* decline and fall back during connect */
e8d726c8
UB		 738static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code,
739 u8 version)
3b2dec26 740{
194730a9 741 struct net *net = sock_net(&smc->sk);
3b2dec26 742 int rc;
ee9dfbef 743
e1bbdd57 744 if (reason_code < 0) { /* error, fallback is not possible */
194730a9 745 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
e1bbdd57
UB		 746 if (smc->sk.sk_state == SMC_INIT)
747 sock_put(&smc->sk); /* passive closing */
3b2dec26 748 return reason_code;
e1bbdd57 749 }
603cc149 750 if (reason_code != SMC_CLC_DECL_PEERDECL) {
e8d726c8 751 rc = smc_clc_send_decline(smc, reason_code, version);
e1bbdd57 752 if (rc < 0) {
194730a9 753 this_cpu_inc(net->smc.smc_stats->clnt_hshake_err_cnt);
e1bbdd57
UB		 754 if (smc->sk.sk_state == SMC_INIT)
755 sock_put(&smc->sk); /* passive closing */
3b2dec26 756 return rc;
e1bbdd57 757 }
c5c1cc9c 758 }
603cc149 759 return smc_connect_fallback(smc, reason_code);
3b2dec26 760}
c5c1cc9c 761
8cf3f3e4 762static void smc_conn_abort(struct smc_sock *smc, int local_first)
3b2dec26 763{
07b4dc34
WG		 764 struct smc_connection *conn = &smc->conn;
765 struct smc_link_group *lgr = conn->lgr;
766
767 smc_conn_free(conn);
5ac54d87 768 if (local_first)
07b4dc34 769 smc_lgr_cleanup_early(lgr);
3b2dec26
HW		 770}
771
772/* check if there is a rdma device available for this connection. */
773/* called for connect and listen */
228bae05 774static int smc_find_rdma_device(struct smc_sock *smc, struct smc_init_info *ini)
3b2dec26 775{
a046d57d
UB		 776 /* PNET table look up: search active ib_device and port
777 * within same PNETID that also contains the ethernet device
778 * used for the internal TCP socket
779 */
bc36d2fc 780 smc_pnet_find_roce_resource(smc->clcsock->sk, ini);
dfc62b12
KG		 781 if (!ini->check_smcrv2 && !ini->ib_dev)
782 return SMC_CLC_DECL_NOSMCRDEV;
783 if (ini->check_smcrv2 && !ini->smcrv2.ib_dev_v2)
9aa68d29 784 return SMC_CLC_DECL_NOSMCRDEV;
bc36d2fc 785 return 0;
3b2dec26
HW		 786}
787
41349844
HW		 788/* check if there is an ISM device available for this connection. */
789/* called for connect and listen */
228bae05 790static int smc_find_ism_device(struct smc_sock *smc, struct smc_init_info *ini)
41349844
HW		 791{
792 /* Find ISM device with same PNETID as connecting interface */
bc36d2fc 793 smc_pnet_find_ism_resource(smc->clcsock->sk, ini);
3fc64937 794 if (!ini->ism_dev[0])
9aa68d29 795 return SMC_CLC_DECL_NOSMCDDEV;
8caaccf5
UB		 796 else
797 ini->ism_chid[0] = smc_ism_get_chid(ini->ism_dev[0]);
41349844
HW		 798 return 0;
799}
800
839d696f
KG		 801/* is chid unique for the ism devices that are already determined? */
802static bool smc_find_ism_v2_is_unique_chid(u16 chid, struct smc_init_info *ini,
803 int cnt)
804{
805 int i = (!ini->ism_dev[0]) ? 1 : 0;
806
807 for (; i < cnt; i++)
808 if (ini->ism_chid[i] == chid)
809 return false;
810 return true;
811}
812
d70bf4f7
UB		 813/* determine possible V2 ISM devices (either without PNETID or with PNETID plus
814 * PNETID matching net_device)
815 */
816static int smc_find_ism_v2_device_clnt(struct smc_sock *smc,
817 struct smc_init_info *ini)
818{
819 int rc = SMC_CLC_DECL_NOSMCDDEV;
820 struct smcd_dev *smcd;
821 int i = 1;
839d696f 822 u16 chid;
d70bf4f7
UB		 823
824 if (smcd_indicated(ini->smc_type_v1))
825 rc = 0; /* already initialized for V1 */
826 mutex_lock(&smcd_dev_list.mutex);
827 list_for_each_entry(smcd, &smcd_dev_list.list, list) {
828 if (smcd->going_away || smcd == ini->ism_dev[0])
829 continue;
839d696f
KG		 830 chid = smc_ism_get_chid(smcd);
831 if (!smc_find_ism_v2_is_unique_chid(chid, ini, i))
832 continue;
d70bf4f7
UB		 833 if (!smc_pnet_is_pnetid_set(smcd->pnetid) ||
834 smc_pnet_is_ndev_pnetid(sock_net(&smc->sk), smcd->pnetid)) {
835 ini->ism_dev[i] = smcd;
839d696f 836 ini->ism_chid[i] = chid;
d70bf4f7
UB		 837 ini->is_smcd = true;
838 rc = 0;
839 i++;
840 if (i > SMC_MAX_ISM_DEVS)
841 break;
842 }
843 }
844 mutex_unlock(&smcd_dev_list.mutex);
845 ini->ism_offered_cnt = i - 1;
846 if (!ini->ism_dev[0] && !ini->ism_dev[1])
847 ini->smcd_version = 0;
848
849 return rc;
850}
851
41349844
HW		 852/* Check for VLAN ID and register it on ISM device just for CLC handshake */
853static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
bc36d2fc 854 struct smc_init_info *ini)
41349844 855{
3fc64937 856 if (ini->vlan_id && smc_ism_get_vlan(ini->ism_dev[0], ini->vlan_id))
7a62725a 857 return SMC_CLC_DECL_ISMVLANERR;
41349844
HW		 858 return 0;
859}
860
d70bf4f7
UB		 861static int smc_find_proposal_devices(struct smc_sock *smc,
862 struct smc_init_info *ini)
863{
864 int rc = 0;
865
866 /* check if there is an ism device available */
dfc62b12
KG		 867 if (!(ini->smcd_version & SMC_V1) ||
868 smc_find_ism_device(smc, ini) ||
869 smc_connect_ism_vlan_setup(smc, ini))
870 ini->smcd_version &= ~SMC_V1;
871 /* else ISM V1 is supported for this connection */
872
873 /* check if there is an rdma device available */
874 if (!(ini->smcr_version & SMC_V1) ||
875 smc_find_rdma_device(smc, ini))
876 ini->smcr_version &= ~SMC_V1;
877 /* else RDMA is supported for this connection */
878
879 ini->smc_type_v1 = smc_indicated_type(ini->smcd_version & SMC_V1,
880 ini->smcr_version & SMC_V1);
881
882 /* check if there is an ism v2 device available */
883 if (!(ini->smcd_version & SMC_V2) ||
884 !smc_ism_is_v2_capable() ||
885 smc_find_ism_v2_device_clnt(smc, ini))
886 ini->smcd_version &= ~SMC_V2;
887
888 /* check if there is an rdma v2 device available */
889 ini->check_smcrv2 = true;
890 ini->smcrv2.saddr = smc->clcsock->sk->sk_rcv_saddr;
891 if (!(ini->smcr_version & SMC_V2) ||
892 smc->clcsock->sk->sk_family != AF_INET ||
893 !smc_clc_ueid_count() ||
894 smc_find_rdma_device(smc, ini))
895 ini->smcr_version &= ~SMC_V2;
896 ini->check_smcrv2 = false;
897
898 ini->smc_type_v2 = smc_indicated_type(ini->smcd_version & SMC_V2,
899 ini->smcr_version & SMC_V2);
d70bf4f7
UB		 900
901 /* if neither ISM nor RDMA are supported, fallback */
dfc62b12 902 if (ini->smc_type_v1 == SMC_TYPE_N && ini->smc_type_v2 == SMC_TYPE_N)
d70bf4f7
UB		 903 rc = SMC_CLC_DECL_NOSMCDEV;
904
905 return rc;
906}
907
41349844
HW		 908/* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
909 * used, the VLAN ID will be registered again during the connection setup.
910 */
d70bf4f7 911static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc,
bc36d2fc 912 struct smc_init_info *ini)
41349844 913{
d70bf4f7 914 if (!smcd_indicated(ini->smc_type_v1))
41349844 915 return 0;
3fc64937 916 if (ini->vlan_id && smc_ism_put_vlan(ini->ism_dev[0], ini->vlan_id))
41349844
HW		 917 return SMC_CLC_DECL_CNFERR;
918 return 0;
919}
920
a7c9c5f4
UB		 921#define SMC_CLC_MAX_ACCEPT_LEN \
922 (sizeof(struct smc_clc_msg_accept_confirm_v2) + \
b81a5eb7 923 sizeof(struct smc_clc_first_contact_ext) + \
a7c9c5f4
UB		 924 sizeof(struct smc_clc_msg_trail))
925
3b2dec26 926/* CLC handshake during connect */
d70bf4f7 927static int smc_connect_clc(struct smc_sock *smc,
a7c9c5f4 928 struct smc_clc_msg_accept_confirm_v2 *aclc2,
bc36d2fc 929 struct smc_init_info *ini)
3b2dec26
HW		 930{
931 int rc = 0;
a046d57d
UB		 932
933 /* do inband token exchange */
d70bf4f7 934 rc = smc_clc_send_proposal(smc, ini);
3b2dec26
HW		 935 if (rc)
936 return rc;
a046d57d 937 /* receive SMC Accept CLC message */
a7c9c5f4
UB		 938 return smc_clc_wait_msg(smc, aclc2, SMC_CLC_MAX_ACCEPT_LEN,
939 SMC_CLC_ACCEPT, CLC_WAIT_TIME);
3b2dec26
HW		 940}
941
d3bdc25d
KG		 942void smc_fill_gid_list(struct smc_link_group *lgr,
943 struct smc_gidlist *gidlist,
944 struct smc_ib_device *known_dev, u8 *known_gid)
ea9478b7
KG		 945{
946 struct smc_init_info *alt_ini = NULL;
947
948 memset(gidlist, 0, sizeof(*gidlist));
949 memcpy(gidlist->list[gidlist->len++], known_gid, SMC_GID_SIZE);
950
951 alt_ini = kzalloc(sizeof(*alt_ini), GFP_KERNEL);
952 if (!alt_ini)
953 goto out;
954
955 alt_ini->vlan_id = lgr->vlan_id;
956 alt_ini->check_smcrv2 = true;
957 alt_ini->smcrv2.saddr = lgr->saddr;
958 smc_pnet_find_alt_roce(lgr, alt_ini, known_dev);
959
960 if (!alt_ini->smcrv2.ib_dev_v2)
961 goto out;
962
963 memcpy(gidlist->list[gidlist->len++], alt_ini->smcrv2.ib_gid_v2,
964 SMC_GID_SIZE);
965
966out:
967 kfree(alt_ini);
968}
969
970static int smc_connect_rdma_v2_prepare(struct smc_sock *smc,
971 struct smc_clc_msg_accept_confirm *aclc,
972 struct smc_init_info *ini)
973{
974 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
975 (struct smc_clc_msg_accept_confirm_v2 *)aclc;
976 struct smc_clc_first_contact_ext *fce =
977 (struct smc_clc_first_contact_ext *)
978 (((u8 *)clc_v2) + sizeof(*clc_v2));
979
980 if (!ini->first_contact_peer || aclc->hdr.version == SMC_V1)
981 return 0;
982
983 if (fce->v2_direct) {
984 memcpy(ini->smcrv2.nexthop_mac, &aclc->r0.lcl.mac, ETH_ALEN);
985 ini->smcrv2.uses_gateway = false;
986 } else {
987 if (smc_ib_find_route(smc->clcsock->sk->sk_rcv_saddr,
988 smc_ib_gid_to_ipv4(aclc->r0.lcl.gid),
989 ini->smcrv2.nexthop_mac,
990 &ini->smcrv2.uses_gateway))
991 return SMC_CLC_DECL_NOROUTE;
992 if (!ini->smcrv2.uses_gateway) {
993 /* mismatch: peer claims indirect, but its direct */
994 return SMC_CLC_DECL_NOINDIRECT;
995 }
996 }
997 return 0;
998}
999
3b2dec26
HW		 1000/* setup for RDMA connection of client */
1001static int smc_connect_rdma(struct smc_sock *smc,
1002 struct smc_clc_msg_accept_confirm *aclc,
bc36d2fc 1003 struct smc_init_info *ini)
3b2dec26 1004{
0fb0b02b 1005 int i, reason_code = 0;
3b2dec26 1006 struct smc_link *link;
ea9478b7 1007 u8 *eid = NULL;
a046d57d 1008
bc36d2fc 1009 ini->is_smcd = false;
3d9725a6 1010 ini->ib_clcqpn = ntoh24(aclc->r0.qpn);
f1eb02f9 1011 ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
ea9478b7
KG		 1012 memcpy(ini->peer_systemid, aclc->r0.lcl.id_for_peer, SMC_SYSTEMID_LEN);
1013 memcpy(ini->peer_gid, aclc->r0.lcl.gid, SMC_GID_SIZE);
1014 memcpy(ini->peer_mac, aclc->r0.lcl.mac, ETH_ALEN);
1015
1016 reason_code = smc_connect_rdma_v2_prepare(smc, aclc, ini);
1017 if (reason_code)
1018 return reason_code;
bc36d2fc 1019
72a36a8a 1020 mutex_lock(&smc_client_lgr_pending);
7a62725a
KG		 1021 reason_code = smc_conn_create(smc, ini);
1022 if (reason_code) {
72a36a8a
HW		 1023 mutex_unlock(&smc_client_lgr_pending);
1024 return reason_code;
0cfdd8f9 1025 }
a046d57d 1026
3b2dec26 1027 smc_conn_save_peer_info(smc, aclc);
cd6851f3 1028
5ac54d87 1029 if (ini->first_contact_local) {
0fb0b02b
KG		 1030 link = smc->conn.lnk;
1031 } else {
1032 /* set link that was assigned by server */
1033 link = NULL;
1034 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1035 struct smc_link *l = &smc->conn.lgr->lnk[i];
1036
3d9725a6
UB		 1037 if (l->peer_qpn == ntoh24(aclc->r0.qpn) &&
1038 !memcmp(l->peer_gid, &aclc->r0.lcl.gid,
1039 SMC_GID_SIZE) &&
ea9478b7
KG		 1040 (aclc->hdr.version > SMC_V1 ||
1041 !memcmp(l->peer_mac, &aclc->r0.lcl.mac,
1042 sizeof(l->peer_mac)))) {
0fb0b02b
KG		 1043 link = l;
1044 break;
1045 }
1046 }
c60a2cef
KG		 1047 if (!link) {
1048 reason_code = SMC_CLC_DECL_NOSRVLINK;
1049 goto connect_abort;
1050 }
64513d26 1051 smc_switch_link_and_count(&smc->conn, link);
0fb0b02b
KG		 1052 }
1053
3e034725 1054 /* create send buffer and rmb */
c60a2cef
KG		 1055 if (smc_buf_create(smc, false)) {
1056 reason_code = SMC_CLC_DECL_MEM;
1057 goto connect_abort;
1058 }
cd6851f3 1059
5ac54d87 1060 if (ini->first_contact_local)
ea9478b7 1061 smc_link_save_peer_info(link, aclc, ini);
bd4ad577 1062
c60a2cef
KG		 1063 if (smc_rmb_rtoken_handling(&smc->conn, link, aclc)) {
1064 reason_code = SMC_CLC_DECL_ERR_RTOK;
1065 goto connect_abort;
1066 }
bd4ad577 1067
46c28dbd
UB		 1068 smc_close_init(smc);
1069 smc_rx_init(smc);
1070
5ac54d87 1071 if (ini->first_contact_local) {
c60a2cef
KG		 1072 if (smc_ib_ready_link(link)) {
1073 reason_code = SMC_CLC_DECL_ERR_RDYLNK;
1074 goto connect_abort;
1075 }
652a1e41 1076 } else {
c60a2cef
KG		 1077 if (smcr_lgr_reg_rmbs(link, smc->conn.rmb_desc)) {
1078 reason_code = SMC_CLC_DECL_ERR_REGRMB;
1079 goto connect_abort;
1080 }
bd4ad577 1081 }
10428dd8 1082 smc_rmb_sync_sg_for_device(&smc->conn);
a046d57d 1083
ea9478b7
KG		 1084 if (aclc->hdr.version > SMC_V1) {
1085 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1086 (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1087
1088 eid = clc_v2->r1.eid;
1089 if (ini->first_contact_local)
1090 smc_fill_gid_list(link->lgr, &ini->smcrv2.gidlist,
1091 link->smcibdev, link->gid);
1092 }
1093
a7c9c5f4 1094 reason_code = smc_clc_send_confirm(smc, ini->first_contact_local,
ea9478b7 1095 aclc->hdr.version, eid, ini);
3b2dec26 1096 if (reason_code)
c60a2cef 1097 goto connect_abort;
3b2dec26
HW		 1098
1099 smc_tx_init(smc);
a046d57d 1100
5ac54d87 1101 if (ini->first_contact_local) {
9bf9abea 1102 /* QP confirmation over RoCE fabric */
0fb0b02b 1103 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
b9247544 1104 reason_code = smcr_clnt_conf_first_link(smc);
0fb0b02b 1105 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
3b2dec26 1106 if (reason_code)
c60a2cef 1107 goto connect_abort;
9bf9abea 1108 }
72a36a8a 1109 mutex_unlock(&smc_client_lgr_pending);
e6727f39 1110
a046d57d 1111 smc_copy_sock_settings_to_clc(smc);
50717a37 1112 smc->connect_nonblock = 0;
b38d7324
UB		 1113 if (smc->sk.sk_state == SMC_INIT)
1114 smc->sk.sk_state = SMC_ACTIVE;
a046d57d 1115
3b2dec26 1116 return 0;
c60a2cef 1117connect_abort:
8cf3f3e4 1118 smc_conn_abort(smc, ini->first_contact_local);
c60a2cef
KG		 1119 mutex_unlock(&smc_client_lgr_pending);
1120 smc->connect_nonblock = 0;
1121
1122 return reason_code;
3b2dec26 1123}
a046d57d 1124
a7c9c5f4
UB		 1125/* The server has chosen one of the proposed ISM devices for the communication.
1126 * Determine from the CHID of the received CLC ACCEPT the ISM device chosen.
1127 */
1128static int
1129smc_v2_determine_accepted_chid(struct smc_clc_msg_accept_confirm_v2 *aclc,
1130 struct smc_init_info *ini)
1131{
1132 int i;
1133
1134 for (i = 0; i < ini->ism_offered_cnt + 1; i++) {
ea9478b7 1135 if (ini->ism_chid[i] == ntohs(aclc->d1.chid)) {
a7c9c5f4
UB		 1136 ini->ism_selected = i;
1137 return 0;
1138 }
1139 }
1140
1141 return -EPROTO;
1142}
1143
41349844
HW		 1144/* setup for ISM connection of client */
1145static int smc_connect_ism(struct smc_sock *smc,
1146 struct smc_clc_msg_accept_confirm *aclc,
bc36d2fc 1147 struct smc_init_info *ini)
41349844 1148{
fefefe10 1149 u8 *eid = NULL;
41349844
HW		 1150 int rc = 0;
1151
bc36d2fc 1152 ini->is_smcd = true;
f1eb02f9 1153 ini->first_contact_peer = aclc->hdr.typev2 & SMC_FIRST_CONTACT_MASK;
bc36d2fc 1154
a7c9c5f4
UB		 1155 if (aclc->hdr.version == SMC_V2) {
1156 struct smc_clc_msg_accept_confirm_v2 *aclc_v2 =
1157 (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1158
1159 rc = smc_v2_determine_accepted_chid(aclc_v2, ini);
1160 if (rc)
1161 return rc;
1162 }
1163 ini->ism_peer_gid[ini->ism_selected] = aclc->d0.gid;
1164
72a36a8a
HW		 1165 /* there is only one lgr role for SMC-D; use server lock */
1166 mutex_lock(&smc_server_lgr_pending);
7a62725a
KG		 1167 rc = smc_conn_create(smc, ini);
1168 if (rc) {
72a36a8a 1169 mutex_unlock(&smc_server_lgr_pending);
7a62725a 1170 return rc;
72a36a8a 1171 }
41349844
HW		 1172
1173 /* Create send and receive buffers */
72b7f6c4 1174 rc = smc_buf_create(smc, true);
c60a2cef
KG		 1175 if (rc) {
1176 rc = (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB : SMC_CLC_DECL_MEM;
1177 goto connect_abort;
1178 }
41349844
HW		 1179
1180 smc_conn_save_peer_info(smc, aclc);
1181 smc_close_init(smc);
1182 smc_rx_init(smc);
1183 smc_tx_init(smc);
1184
fefefe10
KG		 1185 if (aclc->hdr.version > SMC_V1) {
1186 struct smc_clc_msg_accept_confirm_v2 *clc_v2 =
1187 (struct smc_clc_msg_accept_confirm_v2 *)aclc;
1188
ea9478b7 1189 eid = clc_v2->d1.eid;
fefefe10
KG		 1190 }
1191
a7c9c5f4 1192 rc = smc_clc_send_confirm(smc, ini->first_contact_local,
ea9478b7 1193 aclc->hdr.version, eid, NULL);
41349844 1194 if (rc)
c60a2cef 1195 goto connect_abort;
72a36a8a 1196 mutex_unlock(&smc_server_lgr_pending);
41349844
HW		 1197
1198 smc_copy_sock_settings_to_clc(smc);
50717a37 1199 smc->connect_nonblock = 0;
41349844
HW		 1200 if (smc->sk.sk_state == SMC_INIT)
1201 smc->sk.sk_state = SMC_ACTIVE;
1202
1203 return 0;
c60a2cef 1204connect_abort:
8cf3f3e4 1205 smc_conn_abort(smc, ini->first_contact_local);
c60a2cef
KG		 1206 mutex_unlock(&smc_server_lgr_pending);
1207 smc->connect_nonblock = 0;
1208
1209 return rc;
41349844
HW		 1210}
1211
d70bf4f7
UB		 1212/* check if received accept type and version matches a proposed one */
1213static int smc_connect_check_aclc(struct smc_init_info *ini,
1214 struct smc_clc_msg_accept_confirm *aclc)
1215{
dfc62b12
KG		 1216 if (aclc->hdr.typev1 != SMC_TYPE_R &&
1217 aclc->hdr.typev1 != SMC_TYPE_D)
d70bf4f7
UB		 1218 return SMC_CLC_DECL_MODEUNSUPP;
1219
dfc62b12
KG		 1220 if (aclc->hdr.version >= SMC_V2) {
1221 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1222 !smcr_indicated(ini->smc_type_v2)) ||
1223 (aclc->hdr.typev1 == SMC_TYPE_D &&
1224 !smcd_indicated(ini->smc_type_v2)))
1225 return SMC_CLC_DECL_MODEUNSUPP;
1226 } else {
1227 if ((aclc->hdr.typev1 == SMC_TYPE_R &&
1228 !smcr_indicated(ini->smc_type_v1)) ||
1229 (aclc->hdr.typev1 == SMC_TYPE_D &&
1230 !smcd_indicated(ini->smc_type_v1)))
1231 return SMC_CLC_DECL_MODEUNSUPP;
1232 }
1233
d70bf4f7
UB		 1234 return 0;
1235}
1236
3b2dec26
HW		 1237/* perform steps before actually connecting */
1238static int __smc_connect(struct smc_sock *smc)
1239{
49407ae2 1240 u8 version = smc_ism_is_v2_capable() ? SMC_V2 : SMC_V1;
a7c9c5f4
UB		 1241 struct smc_clc_msg_accept_confirm_v2 *aclc2;
1242 struct smc_clc_msg_accept_confirm *aclc;
3fc64937 1243 struct smc_init_info *ini = NULL;
a7c9c5f4 1244 u8 *buf = NULL;
3b2dec26 1245 int rc = 0;
a046d57d 1246
3b2dec26 1247 if (smc->use_fallback)
603cc149 1248 return smc_connect_fallback(smc, smc->fallback_rsn);
3b2dec26
HW		 1249
1250 /* if peer has not signalled SMC-capability, fall back */
1251 if (!tcp_sk(smc->clcsock->sk)->syn_smc)
603cc149 1252 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
3b2dec26 1253
e8d726c8 1254 /* IPSec connections opt out of SMC optimizations */
3b2dec26 1255 if (using_ipsec(smc))
e8d726c8
UB		 1256 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC,
1257 version);
3b2dec26 1258
3fc64937
UB		 1259 ini = kzalloc(sizeof(*ini), GFP_KERNEL);
1260 if (!ini)
e8d726c8
UB		 1261 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_MEM,
1262 version);
3fc64937 1263
dfc62b12
KG		 1264 ini->smcd_version = SMC_V1 | SMC_V2;
1265 ini->smcr_version = SMC_V1 | SMC_V2;
d70bf4f7 1266 ini->smc_type_v1 = SMC_TYPE_B;
dfc62b12 1267 ini->smc_type_v2 = SMC_TYPE_B;
d70bf4f7 1268
fba7e8ef 1269 /* get vlan id from IP device */
3fc64937 1270 if (smc_vlan_by_tcpsk(smc->clcsock, ini)) {
d70bf4f7 1271 ini->smcd_version &= ~SMC_V1;
dfc62b12 1272 ini->smcr_version = 0;
d70bf4f7
UB		 1273 ini->smc_type_v1 = SMC_TYPE_N;
1274 if (!ini->smcd_version) {
1275 rc = SMC_CLC_DECL_GETVLANERR;
1276 goto fallback;
1277 }
41349844
HW		 1278 }
1279
d70bf4f7
UB		 1280 rc = smc_find_proposal_devices(smc, ini);
1281 if (rc)
1282 goto fallback;
3b2dec26 1283
a7c9c5f4
UB		 1284 buf = kzalloc(SMC_CLC_MAX_ACCEPT_LEN, GFP_KERNEL);
1285 if (!buf) {
1286 rc = SMC_CLC_DECL_MEM;
1287 goto fallback;
1288 }
1289 aclc2 = (struct smc_clc_msg_accept_confirm_v2 *)buf;
1290 aclc = (struct smc_clc_msg_accept_confirm *)aclc2;
1291
3b2dec26 1292 /* perform CLC handshake */
a7c9c5f4 1293 rc = smc_connect_clc(smc, aclc2, ini);
d70bf4f7
UB		 1294 if (rc)
1295 goto vlan_cleanup;
1296
1297 /* check if smc modes and versions of CLC proposal and accept match */
a7c9c5f4 1298 rc = smc_connect_check_aclc(ini, aclc);
0530bd6e 1299 version = aclc->hdr.version == SMC_V1 ? SMC_V1 : SMC_V2;
d70bf4f7
UB		 1300 if (rc)
1301 goto vlan_cleanup;
3b2dec26 1302
41349844 1303 /* depending on previous steps, connect using rdma or ism */
dfc62b12
KG		 1304 if (aclc->hdr.typev1 == SMC_TYPE_R) {
1305 ini->smcr_version = version;
a7c9c5f4 1306 rc = smc_connect_rdma(smc, aclc, ini);
dfc62b12
KG		 1307 } else if (aclc->hdr.typev1 == SMC_TYPE_D) {
1308 ini->smcd_version = version;
a7c9c5f4 1309 rc = smc_connect_ism(smc, aclc, ini);
dfc62b12 1310 }
d70bf4f7
UB		 1311 if (rc)
1312 goto vlan_cleanup;
3b2dec26 1313
194730a9 1314 SMC_STAT_CLNT_SUCC_INC(sock_net(smc->clcsock->sk), aclc);
d70bf4f7 1315 smc_connect_ism_vlan_cleanup(smc, ini);
a7c9c5f4 1316 kfree(buf);
3fc64937 1317 kfree(ini);
3b2dec26 1318 return 0;
d70bf4f7
UB		 1319
1320vlan_cleanup:
1321 smc_connect_ism_vlan_cleanup(smc, ini);
a7c9c5f4 1322 kfree(buf);
d70bf4f7
UB		 1323fallback:
1324 kfree(ini);
e8d726c8 1325 return smc_connect_decline_fallback(smc, rc, version);
a046d57d
UB		 1326}
1327
24ac3a08
UB		 1328static void smc_connect_work(struct work_struct *work)
1329{
1330 struct smc_sock *smc = container_of(work, struct smc_sock,
1331 connect_work);
50717a37
UB		 1332 long timeo = smc->sk.sk_sndtimeo;
1333 int rc = 0;
24ac3a08 1334
50717a37
UB		 1335 if (!timeo)
1336 timeo = MAX_SCHEDULE_TIMEOUT;
1337 lock_sock(smc->clcsock->sk);
24ac3a08
UB		 1338 if (smc->clcsock->sk->sk_err) {
1339 smc->sk.sk_err = smc->clcsock->sk->sk_err;
50717a37 1340 } else if ((1 << smc->clcsock->sk->sk_state) &
f3a3a0fe 1341 (TCPF_SYN_SENT | TCPF_SYN_RECV)) {
50717a37
UB		 1342 rc = sk_stream_wait_connect(smc->clcsock->sk, &timeo);
1343 if ((rc == -EPIPE) &&
1344 ((1 << smc->clcsock->sk->sk_state) &
1345 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)))
1346 rc = 0;
24ac3a08 1347 }
50717a37
UB		 1348 release_sock(smc->clcsock->sk);
1349 lock_sock(&smc->sk);
1350 if (rc != 0 || smc->sk.sk_err) {
1351 smc->sk.sk_state = SMC_CLOSED;
1352 if (rc == -EPIPE || rc == -EAGAIN)
1353 smc->sk.sk_err = EPIPE;
1354 else if (signal_pending(current))
1355 smc->sk.sk_err = -sock_intr_errno(timeo);
6d6dd528 1356 sock_put(&smc->sk); /* passive closing */
24ac3a08
UB		 1357 goto out;
1358 }
1359
1360 rc = __smc_connect(smc);
1361 if (rc < 0)
1362 smc->sk.sk_err = -rc;
1363
1364out:
07603b23
UB		 1365 if (!sock_flag(&smc->sk, SOCK_DEAD)) {
1366 if (smc->sk.sk_err) {
1367 smc->sk.sk_state_change(&smc->sk);
1368 } else { /* allow polling before and after fallback decision */
1369 smc->clcsock->sk->sk_write_space(smc->clcsock->sk);
1370 smc->sk.sk_write_space(&smc->sk);
1371 }
1372 }
24ac3a08
UB		 1373 release_sock(&smc->sk);
1374}
1375
ac713874
UB		 1376static int smc_connect(struct socket *sock, struct sockaddr *addr,
1377 int alen, int flags)
1378{
1379 struct sock *sk = sock->sk;
1380 struct smc_sock *smc;
1381 int rc = -EINVAL;
1382
1383 smc = smc_sk(sk);
1384
1385 /* separate smc parameter checking to be safe */
1386 if (alen < sizeof(addr->sa_family))
1387 goto out_err;
aaa4d33f 1388 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
ac713874
UB		 1389 goto out_err;
1390
1391 lock_sock(sk);
1392 switch (sk->sk_state) {
1393 default:
1394 goto out;
1395 case SMC_ACTIVE:
1396 rc = -EISCONN;
1397 goto out;
1398 case SMC_INIT:
ac713874
UB		 1399 break;
1400 }
1401
1402 smc_copy_sock_settings_to_clc(smc);
c5c1cc9c 1403 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
50717a37
UB		 1404 if (smc->connect_nonblock) {
1405 rc = -EALREADY;
1406 goto out;
1407 }
1408 rc = kernel_connect(smc->clcsock, addr, alen, flags);
1409 if (rc && rc != -EINPROGRESS)
1410 goto out;
301428ea
UB		 1411
1412 sock_hold(&smc->sk); /* sock put in passive closing */
86434744
UB		 1413 if (smc->use_fallback)
1414 goto out;
24ac3a08 1415 if (flags & O_NONBLOCK) {
22ef473d 1416 if (queue_work(smc_hs_wq, &smc->connect_work))
50717a37 1417 smc->connect_nonblock = 1;
24ac3a08
UB		 1418 rc = -EINPROGRESS;
1419 } else {
24ac3a08
UB		 1420 rc = __smc_connect(smc);
1421 if (rc < 0)
1422 goto out;
1423 else
1424 rc = 0; /* success cases including fallback */
1425 }
ac713874
UB		 1426
1427out:
1428 release_sock(sk);
1429out_err:
1430 return rc;
1431}
1432
1433static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
1434{
3163c507
UB		 1435 struct socket *new_clcsock = NULL;
1436 struct sock *lsk = &lsmc->sk;
ac713874 1437 struct sock *new_sk;
78abe3d0 1438 int rc = -EINVAL;
ac713874 1439
3163c507 1440 release_sock(lsk);
aaa4d33f 1441 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
ac713874
UB		 1442 if (!new_sk) {
1443 rc = -ENOMEM;
3163c507 1444 lsk->sk_err = ENOMEM;
ac713874 1445 *new_smc = NULL;
3163c507 1446 lock_sock(lsk);
ac713874
UB		 1447 goto out;
1448 }
1449 *new_smc = smc_sk(new_sk);
1450
78abe3d0
MJ		 1451 mutex_lock(&lsmc->clcsock_release_lock);
1452 if (lsmc->clcsock)
a60a2b1e 1453 rc = kernel_accept(lsmc->clcsock, &new_clcsock, SOCK_NONBLOCK);
78abe3d0 1454 mutex_unlock(&lsmc->clcsock_release_lock);
3163c507 1455 lock_sock(lsk);
a60a2b1e 1456 if (rc < 0 && rc != -EAGAIN)
3163c507 1457 lsk->sk_err = -rc;
35a6b178 1458 if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
f61bca58 1459 new_sk->sk_prot->unhash(new_sk);
a046d57d
UB		 1460 if (new_clcsock)
1461 sock_release(new_clcsock);
1462 new_sk->sk_state = SMC_CLOSED;
1463 sock_set_flag(new_sk, SOCK_DEAD);
51f1de79 1464 sock_put(new_sk); /* final */
ac713874
UB		 1465 *new_smc = NULL;
1466 goto out;
1467 }
1468
a60a2b1e
UB		 1469 /* new clcsock has inherited the smc listen-specific sk_data_ready
1470 * function; switch it back to the original sk_data_ready function
1471 */
1472 new_clcsock->sk->sk_data_ready = lsmc->clcsk_data_ready;
ac713874
UB		 1473 (*new_smc)->clcsock = new_clcsock;
1474out:
1475 return rc;
1476}
1477
a046d57d
UB		 1478/* add a just created sock to the accept queue of the listen sock as
1479 * candidate for a following socket accept call from user space
1480 */
1481static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
1482{
1483 struct smc_sock *par = smc_sk(parent);
1484
51f1de79 1485 sock_hold(sk); /* sock_put in smc_accept_unlink () */
a046d57d
UB		 1486 spin_lock(&par->accept_q_lock);
1487 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
1488 spin_unlock(&par->accept_q_lock);
1489 sk_acceptq_added(parent);
1490}
1491
1492/* remove a socket from the accept queue of its parental listening socket */
1493static void smc_accept_unlink(struct sock *sk)
1494{
1495 struct smc_sock *par = smc_sk(sk)->listen_smc;
1496
1497 spin_lock(&par->accept_q_lock);
1498 list_del_init(&smc_sk(sk)->accept_q);
1499 spin_unlock(&par->accept_q_lock);
1500 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
51f1de79 1501 sock_put(sk); /* sock_hold in smc_accept_enqueue */
a046d57d
UB		 1502}
1503
1504/* remove a sock from the accept queue to bind it to a new socket created
1505 * for a socket accept call from user space
1506 */
b38d7324
UB		 1507struct sock *smc_accept_dequeue(struct sock *parent,
1508 struct socket *new_sock)
a046d57d
UB		 1509{
1510 struct smc_sock *isk, *n;
1511 struct sock *new_sk;
1512
1513 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
1514 new_sk = (struct sock *)isk;
1515
1516 smc_accept_unlink(new_sk);
1517 if (new_sk->sk_state == SMC_CLOSED) {
f61bca58 1518 new_sk->sk_prot->unhash(new_sk);
127f4970
UB		 1519 if (isk->clcsock) {
1520 sock_release(isk->clcsock);
1521 isk->clcsock = NULL;
1522 }
51f1de79 1523 sock_put(new_sk); /* final */
a046d57d
UB		 1524 continue;
1525 }
07603b23 1526 if (new_sock) {
a046d57d 1527 sock_graft(new_sk, new_sock);
07603b23
UB		 1528 if (isk->use_fallback) {
1529 smc_sk(new_sk)->clcsock->file = new_sock->file;
1530 isk->clcsock->file->private_data = isk->clcsock;
1531 }
1532 }
a046d57d
UB		 1533 return new_sk;
1534 }
1535 return NULL;
1536}
1537
1538/* clean up for a created but never accepted sock */
b38d7324 1539void smc_close_non_accepted(struct sock *sk)
a046d57d
UB		 1540{
1541 struct smc_sock *smc = smc_sk(sk);
1542
81cf4f47 1543 sock_hold(sk); /* sock_put below */
b38d7324
UB		 1544 lock_sock(sk);
1545 if (!sk->sk_lingertime)
1546 /* wait for peer closing */
1547 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
39f41f36 1548 __smc_release(smc);
b38d7324 1549 release_sock(sk);
81cf4f47 1550 sock_put(sk); /* sock_hold above */
51f1de79 1551 sock_put(sk); /* final sock_put */
a046d57d
UB		 1552}
1553
b9247544 1554static int smcr_serv_conf_first_link(struct smc_sock *smc)
9bf9abea 1555{
387707fd 1556 struct smc_link *link = smc->conn.lnk;
4667bb4a 1557 struct smc_llc_qentry *qentry;
9bf9abea
UB		 1558 int rc;
1559
7562a13d 1560 if (smcr_link_reg_rmb(link, smc->conn.rmb_desc))
603cc149 1561 return SMC_CLC_DECL_ERR_REGRMB;
652a1e41 1562
9bf9abea 1563 /* send CONFIRM LINK request to client over the RoCE fabric */
947541f3 1564 rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
9bf9abea 1565 if (rc < 0)
603cc149 1566 return SMC_CLC_DECL_TIMEOUT_CL;
9bf9abea
UB		 1567
1568 /* receive CONFIRM LINK response from client over the RoCE fabric */
4667bb4a
KG		 1569 qentry = smc_llc_wait(link->lgr, link, SMC_LLC_WAIT_TIME,
1570 SMC_LLC_CONFIRM_LINK);
1571 if (!qentry) {
9bf9abea
UB		 1572 struct smc_clc_msg_decline dclc;
1573
1574 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
2b59f58e 1575 SMC_CLC_DECLINE, CLC_WAIT_TIME_SHORT);
9ed28556 1576 return rc == -EAGAIN ? SMC_CLC_DECL_TIMEOUT_CL : rc;
9bf9abea 1577 }
649758ff 1578 smc_llc_save_peer_uid(qentry);
4667bb4a
KG		 1579 rc = smc_llc_eval_conf_link(qentry, SMC_LLC_RESP);
1580 smc_llc_flow_qentry_del(&link->lgr->llc_flow_lcl);
1581 if (rc)
75d320d6
KG		 1582 return SMC_CLC_DECL_RMBE_EC;
1583
4667bb4a
KG		 1584 /* confirm_rkey is implicit on 1st contact */
1585 smc->conn.rmb_desc->is_conf_rkey = true;
52bedf37 1586
00a049cf 1587 smc_llc_link_active(link);
0a99be43 1588 smcr_lgr_set_type(link->lgr, SMC_LGR_SINGLE);
52bedf37 1589
4667bb4a 1590 /* initial contact - try to establish second link */
d3bdc25d 1591 smc_llc_srv_add_link(link, NULL);
75d320d6 1592 return 0;
9bf9abea
UB		 1593}
1594
3b2dec26
HW		 1595/* listen worker: finish */
1596static void smc_listen_out(struct smc_sock *new_smc)
a046d57d 1597{
a046d57d 1598 struct smc_sock *lsmc = new_smc->listen_smc;
a046d57d 1599 struct sock *newsmcsk = &new_smc->sk;
a046d57d 1600
3b2dec26 1601 if (lsmc->sk.sk_state == SMC_LISTEN) {
fd57770d 1602 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
3b2dec26 1603 smc_accept_enqueue(&lsmc->sk, newsmcsk);
fd57770d 1604 release_sock(&lsmc->sk);
3b2dec26
HW		 1605 } else { /* no longer listening */
1606 smc_close_non_accepted(newsmcsk);
c5c1cc9c
UB		 1607 }
1608
3b2dec26
HW		 1609 /* Wake up accept */
1610 lsmc->sk.sk_data_ready(&lsmc->sk);
1611 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1612}
a046d57d 1613
3b2dec26
HW		 1614/* listen worker: finish in state connected */
1615static void smc_listen_out_connected(struct smc_sock *new_smc)
1616{
1617 struct sock *newsmcsk = &new_smc->sk;
a046d57d 1618
3b2dec26
HW		 1619 sk_refcnt_debug_inc(newsmcsk);
1620 if (newsmcsk->sk_state == SMC_INIT)
1621 newsmcsk->sk_state = SMC_ACTIVE;
1622
1623 smc_listen_out(new_smc);
1624}
1625
1626/* listen worker: finish in error state */
1627static void smc_listen_out_err(struct smc_sock *new_smc)
1628{
1629 struct sock *newsmcsk = &new_smc->sk;
194730a9 1630 struct net *net = sock_net(newsmcsk);
3b2dec26 1631
194730a9 1632 this_cpu_inc(net->smc.smc_stats->srv_hshake_err_cnt);
3b2dec26
HW		 1633 if (newsmcsk->sk_state == SMC_INIT)
1634 sock_put(&new_smc->sk); /* passive closing */
1635 newsmcsk->sk_state = SMC_CLOSED;
3b2dec26
HW		 1636
1637 smc_listen_out(new_smc);
1638}
1639
1640/* listen worker: decline and fall back if possible */
1641static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
4a9baf45 1642 int local_first, u8 version)
3b2dec26
HW		 1643{
1644 /* RDMA setup failed, switch back to TCP */
8cf3f3e4 1645 smc_conn_abort(new_smc, local_first);
3924702d
WG		 1646 if (reason_code < 0 ||
1647 smc_switch_to_fallback(new_smc, reason_code)) {
1648 /* error, no fallback possible */
3b2dec26
HW		 1649 smc_listen_out_err(new_smc);
1650 return;
1651 }
603cc149 1652 if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
e8d726c8 1653 if (smc_clc_send_decline(new_smc, reason_code, version) < 0) {
3b2dec26
HW		 1654 smc_listen_out_err(new_smc);
1655 return;
1656 }
a046d57d 1657 }
3b2dec26
HW		 1658 smc_listen_out_connected(new_smc);
1659}
1660
5c21c4cc
UB		 1661/* listen worker: version checking */
1662static int smc_listen_v2_check(struct smc_sock *new_smc,
1663 struct smc_clc_msg_proposal *pclc,
1664 struct smc_init_info *ini)
1665{
1666 struct smc_clc_smcd_v2_extension *pclc_smcd_v2_ext;
1667 struct smc_clc_v2_extension *pclc_v2_ext;
3752404a 1668 int rc = SMC_CLC_DECL_PEERNOSMC;
5c21c4cc
UB		 1669
1670 ini->smc_type_v1 = pclc->hdr.typev1;
1671 ini->smc_type_v2 = pclc->hdr.typev2;
9f1e0d61
KG		 1672 ini->smcd_version = smcd_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1673 ini->smcr_version = smcr_indicated(ini->smc_type_v1) ? SMC_V1 : 0;
1674 if (pclc->hdr.version > SMC_V1) {
1675 if (smcd_indicated(ini->smc_type_v2))
1676 ini->smcd_version |= SMC_V2;
1677 if (smcr_indicated(ini->smc_type_v2))
1678 ini->smcr_version |= SMC_V2;
1679 }
1680 if (!(ini->smcd_version & SMC_V2) && !(ini->smcr_version & SMC_V2)) {
3752404a
KG		 1681 rc = SMC_CLC_DECL_PEERNOSMC;
1682 goto out;
1683 }
5c21c4cc
UB		 1684 pclc_v2_ext = smc_get_clc_v2_ext(pclc);
1685 if (!pclc_v2_ext) {
1686 ini->smcd_version &= ~SMC_V2;
9f1e0d61 1687 ini->smcr_version &= ~SMC_V2;
3752404a 1688 rc = SMC_CLC_DECL_NOV2EXT;
5c21c4cc
UB		 1689 goto out;
1690 }
1691 pclc_smcd_v2_ext = smc_get_clc_smcd_v2_ext(pclc_v2_ext);
9f1e0d61
KG		 1692 if (ini->smcd_version & SMC_V2) {
1693 if (!smc_ism_is_v2_capable()) {
1694 ini->smcd_version &= ~SMC_V2;
1695 rc = SMC_CLC_DECL_NOISM2SUPP;
1696 } else if (!pclc_smcd_v2_ext) {
1697 ini->smcd_version &= ~SMC_V2;
1698 rc = SMC_CLC_DECL_NOV2DEXT;
1699 } else if (!pclc_v2_ext->hdr.eid_cnt &&
1700 !pclc_v2_ext->hdr.flag.seid) {
1701 ini->smcd_version &= ~SMC_V2;
1702 rc = SMC_CLC_DECL_NOUEID;
1703 }
1704 }
1705 if (ini->smcr_version & SMC_V2) {
1706 if (!pclc_v2_ext->hdr.eid_cnt) {
1707 ini->smcr_version &= ~SMC_V2;
1708 rc = SMC_CLC_DECL_NOUEID;
1709 }
3752404a 1710 }
5c21c4cc
UB		 1711
1712out:
9f1e0d61 1713 if (!ini->smcd_version && !ini->smcr_version)
3752404a 1714 return rc;
5c21c4cc
UB		 1715
1716 return 0;
1717}
1718
3b2dec26 1719/* listen worker: check prefixes */
59886697 1720static int smc_listen_prfx_check(struct smc_sock *new_smc,
3b2dec26
HW		 1721 struct smc_clc_msg_proposal *pclc)
1722{
1723 struct smc_clc_msg_proposal_prefix *pclc_prfx;
1724 struct socket *newclcsock = new_smc->clcsock;
a046d57d 1725
5c21c4cc
UB		 1726 if (pclc->hdr.typev1 == SMC_TYPE_N)
1727 return 0;
e7b7a64a 1728 pclc_prfx = smc_clc_proposal_get_prefix(pclc);
3b2dec26 1729 if (smc_clc_prfx_match(newclcsock, pclc_prfx))
59886697 1730 return SMC_CLC_DECL_DIFFPREFIX;
c246d942 1731
3b2dec26
HW		 1732 return 0;
1733}
a046d57d 1734
3b2dec26
HW		 1735/* listen worker: initialize connection and buffers */
1736static int smc_listen_rdma_init(struct smc_sock *new_smc,
7a62725a 1737 struct smc_init_info *ini)
3b2dec26 1738{
7a62725a
KG		 1739 int rc;
1740
0cfdd8f9 1741 /* allocate connection / link group */
7a62725a
KG		 1742 rc = smc_conn_create(new_smc, ini);
1743 if (rc)
1744 return rc;
a046d57d 1745
3e034725 1746 /* create send buffer and rmb */
c6ba7c9b 1747 if (smc_buf_create(new_smc, false))
3b2dec26 1748 return SMC_CLC_DECL_MEM;
a046d57d 1749
3b2dec26
HW		 1750 return 0;
1751}
1752
41349844
HW		 1753/* listen worker: initialize connection and buffers for SMC-D */
1754static int smc_listen_ism_init(struct smc_sock *new_smc,
7a62725a 1755 struct smc_init_info *ini)
41349844 1756{
7a62725a 1757 int rc;
41349844 1758
7a62725a
KG		 1759 rc = smc_conn_create(new_smc, ini);
1760 if (rc)
1761 return rc;
41349844 1762
41349844 1763 /* Create send and receive buffers */
72b7f6c4
KG		 1764 rc = smc_buf_create(new_smc, true);
1765 if (rc) {
8cf3f3e4 1766 smc_conn_abort(new_smc, ini->first_contact_local);
72b7f6c4
KG		 1767 return (rc == -ENOSPC) ? SMC_CLC_DECL_MAX_DMB :
1768 SMC_CLC_DECL_MEM;
41349844
HW		 1769 }
1770
1771 return 0;
1772}
1773
5c21c4cc
UB		 1774static bool smc_is_already_selected(struct smcd_dev *smcd,
1775 struct smc_init_info *ini,
1776 int matches)
1777{
1778 int i;
1779
1780 for (i = 0; i < matches; i++)
1781 if (smcd == ini->ism_dev[i])
1782 return true;
1783
1784 return false;
1785}
1786
1787/* check for ISM devices matching proposed ISM devices */
1788static void smc_check_ism_v2_match(struct smc_init_info *ini,
1789 u16 proposed_chid, u64 proposed_gid,
1790 unsigned int *matches)
1791{
1792 struct smcd_dev *smcd;
1793
1794 list_for_each_entry(smcd, &smcd_dev_list.list, list) {
1795 if (smcd->going_away)
1796 continue;
1797 if (smc_is_already_selected(smcd, ini, *matches))
1798 continue;
1799 if (smc_ism_get_chid(smcd) == proposed_chid &&
1800 !smc_ism_cantalk(proposed_gid, ISM_RESERVED_VLANID, smcd)) {
1801 ini->ism_peer_gid[*matches] = proposed_gid;
1802 ini->ism_dev[*matches] = smcd;
1803 (*matches)++;
1804 break;
1805 }
1806 }
1807}
1808
3752404a
KG		 1809static void smc_find_ism_store_rc(u32 rc, struct smc_init_info *ini)
1810{
1811 if (!ini->rc)
1812 ini->rc = rc;
1813}
1814
5c21c4cc
UB		 1815static void smc_find_ism_v2_device_serv(struct smc_sock *new_smc,
1816 struct smc_clc_msg_proposal *pclc,
1817 struct smc_init_info *ini)
1818{
1819 struct smc_clc_smcd_v2_extension *smcd_v2_ext;
1820 struct smc_clc_v2_extension *smc_v2_ext;
1821 struct smc_clc_msg_smcd *pclc_smcd;
1822 unsigned int matches = 0;
f29fa003 1823 u8 smcd_version;
5c21c4cc 1824 u8 *eid = NULL;
3752404a 1825 int i, rc;
5c21c4cc
UB		 1826
1827 if (!(ini->smcd_version & SMC_V2) || !smcd_indicated(ini->smc_type_v2))
f29fa003 1828 goto not_found;
5c21c4cc
UB		 1829
1830 pclc_smcd = smc_get_clc_msg_smcd(pclc);
1831 smc_v2_ext = smc_get_clc_v2_ext(pclc);
1832 smcd_v2_ext = smc_get_clc_smcd_v2_ext(smc_v2_ext);
5c21c4cc
UB		 1833
1834 mutex_lock(&smcd_dev_list.mutex);
1835 if (pclc_smcd->ism.chid)
1836 /* check for ISM device matching proposed native ISM device */
1837 smc_check_ism_v2_match(ini, ntohs(pclc_smcd->ism.chid),
1838 ntohll(pclc_smcd->ism.gid), &matches);
1839 for (i = 1; i <= smc_v2_ext->hdr.ism_gid_cnt; i++) {
1840 /* check for ISM devices matching proposed non-native ISM
1841 * devices
1842 */
1843 smc_check_ism_v2_match(ini,
1844 ntohs(smcd_v2_ext->gidchid[i - 1].chid),
1845 ntohll(smcd_v2_ext->gidchid[i - 1].gid),
1846 &matches);
1847 }
1848 mutex_unlock(&smcd_dev_list.mutex);
1849
9f1e0d61
KG		 1850 if (!ini->ism_dev[0]) {
1851 smc_find_ism_store_rc(SMC_CLC_DECL_NOSMCD2DEV, ini);
fefefe10 1852 goto not_found;
9f1e0d61 1853 }
fefefe10 1854
9927aab9 1855 smc_ism_get_system_eid(&eid);
fefefe10
KG		 1856 if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext,
1857 smcd_v2_ext->system_eid, eid))
5c21c4cc 1858 goto not_found;
5c21c4cc
UB		 1859
1860 /* separate - outside the smcd_dev_list.lock */
f29fa003 1861 smcd_version = ini->smcd_version;
5c21c4cc
UB		 1862 for (i = 0; i < matches; i++) {
1863 ini->smcd_version = SMC_V2;
1864 ini->is_smcd = true;
1865 ini->ism_selected = i;
3752404a
KG		 1866 rc = smc_listen_ism_init(new_smc, ini);
1867 if (rc) {
1868 smc_find_ism_store_rc(rc, ini);
5c21c4cc
UB		 1869 /* try next active ISM device */
1870 continue;
3752404a 1871 }
5c21c4cc
UB		 1872 return; /* matching and usable V2 ISM device found */
1873 }
f29fa003
KG		 1874 /* no V2 ISM device could be initialized */
1875 ini->smcd_version = smcd_version; /* restore original value */
fefefe10 1876 ini->negotiated_eid[0] = 0;
5c21c4cc
UB		 1877
1878not_found:
1879 ini->smcd_version &= ~SMC_V2;
1880 ini->ism_dev[0] = NULL;
1881 ini->is_smcd = false;
1882}
1883
1884static void smc_find_ism_v1_device_serv(struct smc_sock *new_smc,
1885 struct smc_clc_msg_proposal *pclc,
1886 struct smc_init_info *ini)
7affc809
UB		 1887{
1888 struct smc_clc_msg_smcd *pclc_smcd = smc_get_clc_msg_smcd(pclc);
3752404a 1889 int rc = 0;
7affc809 1890
5c21c4cc
UB		 1891 /* check if ISM V1 is available */
1892 if (!(ini->smcd_version & SMC_V1) || !smcd_indicated(ini->smc_type_v1))
7affc809
UB		 1893 goto not_found;
1894 ini->is_smcd = true; /* prepare ISM check */
8c3dca34 1895 ini->ism_peer_gid[0] = ntohll(pclc_smcd->ism.gid);
3752404a
KG		 1896 rc = smc_find_ism_device(new_smc, ini);
1897 if (rc)
7affc809 1898 goto not_found;
5c21c4cc 1899 ini->ism_selected = 0;
3752404a
KG		 1900 rc = smc_listen_ism_init(new_smc, ini);
1901 if (!rc)
5c21c4cc 1902 return; /* V1 ISM device found */
7affc809
UB		 1903
1904not_found:
3752404a 1905 smc_find_ism_store_rc(rc, ini);
9f1e0d61 1906 ini->smcd_version &= ~SMC_V1;
3fc64937 1907 ini->ism_dev[0] = NULL;
7affc809
UB		 1908 ini->is_smcd = false;
1909}
1910
3b2dec26 1911/* listen worker: register buffers */
5ac54d87 1912static int smc_listen_rdma_reg(struct smc_sock *new_smc, bool local_first)
3b2dec26 1913{
b9247544 1914 struct smc_connection *conn = &new_smc->conn;
46c28dbd 1915
5ac54d87 1916 if (!local_first) {
7562a13d 1917 if (smcr_lgr_reg_rmbs(conn->lnk, conn->rmb_desc))
c7674c00 1918 return SMC_CLC_DECL_ERR_REGRMB;
652a1e41 1919 }
10428dd8 1920 smc_rmb_sync_sg_for_device(&new_smc->conn);
652a1e41 1921
3b2dec26
HW		 1922 return 0;
1923}
1924
9f1e0d61
KG		 1925static void smc_find_rdma_v2_device_serv(struct smc_sock *new_smc,
1926 struct smc_clc_msg_proposal *pclc,
1927 struct smc_init_info *ini)
1928{
1929 struct smc_clc_v2_extension *smc_v2_ext;
1930 u8 smcr_version;
1931 int rc;
1932
1933 if (!(ini->smcr_version & SMC_V2) || !smcr_indicated(ini->smc_type_v2))
1934 goto not_found;
1935
1936 smc_v2_ext = smc_get_clc_v2_ext(pclc);
1937 if (!smc_clc_match_eid(ini->negotiated_eid, smc_v2_ext, NULL, NULL))
1938 goto not_found;
1939
1940 /* prepare RDMA check */
1941 memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
1942 memcpy(ini->peer_gid, smc_v2_ext->roce, SMC_GID_SIZE);
1943 memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
1944 ini->check_smcrv2 = true;
1945 ini->smcrv2.clc_sk = new_smc->clcsock->sk;
1946 ini->smcrv2.saddr = new_smc->clcsock->sk->sk_rcv_saddr;
1947 ini->smcrv2.daddr = smc_ib_gid_to_ipv4(smc_v2_ext->roce);
1948 rc = smc_find_rdma_device(new_smc, ini);
1949 if (rc) {
1950 smc_find_ism_store_rc(rc, ini);
1951 goto not_found;
1952 }
1953 if (!ini->smcrv2.uses_gateway)
1954 memcpy(ini->smcrv2.nexthop_mac, pclc->lcl.mac, ETH_ALEN);
1955
1956 smcr_version = ini->smcr_version;
1957 ini->smcr_version = SMC_V2;
1958 rc = smc_listen_rdma_init(new_smc, ini);
1959 if (!rc)
1960 rc = smc_listen_rdma_reg(new_smc, ini->first_contact_local);
1961 if (!rc)
1962 return;
1963 ini->smcr_version = smcr_version;
1964 smc_find_ism_store_rc(rc, ini);
1965
1966not_found:
1967 ini->smcr_version &= ~SMC_V2;
1968 ini->check_smcrv2 = false;
1969}
1970
5c21c4cc
UB		 1971static int smc_find_rdma_v1_device_serv(struct smc_sock *new_smc,
1972 struct smc_clc_msg_proposal *pclc,
1973 struct smc_init_info *ini)
7affc809
UB		 1974{
1975 int rc;
1976
9f1e0d61 1977 if (!(ini->smcr_version & SMC_V1) || !smcr_indicated(ini->smc_type_v1))
7affc809
UB		 1978 return SMC_CLC_DECL_NOSMCDEV;
1979
1980 /* prepare RDMA check */
9f1e0d61
KG		 1981 memcpy(ini->peer_systemid, pclc->lcl.id_for_peer, SMC_SYSTEMID_LEN);
1982 memcpy(ini->peer_gid, pclc->lcl.gid, SMC_GID_SIZE);
1983 memcpy(ini->peer_mac, pclc->lcl.mac, ETH_ALEN);
7affc809
UB		 1984 rc = smc_find_rdma_device(new_smc, ini);
1985 if (rc) {
1986 /* no RDMA device found */
9f1e0d61 1987 return SMC_CLC_DECL_NOSMCDEV;
7affc809
UB		 1988 }
1989 rc = smc_listen_rdma_init(new_smc, ini);
1990 if (rc)
1991 return rc;
1992 return smc_listen_rdma_reg(new_smc, ini->first_contact_local);
1993}
1994
1995/* determine the local device matching to proposal */
1996static int smc_listen_find_device(struct smc_sock *new_smc,
1997 struct smc_clc_msg_proposal *pclc,
1998 struct smc_init_info *ini)
1999{
9f1e0d61 2000 int prfx_rc;
5c21c4cc
UB		 2001
2002 /* check for ISM device matching V2 proposed device */
2003 smc_find_ism_v2_device_serv(new_smc, pclc, ini);
2004 if (ini->ism_dev[0])
7affc809 2005 return 0;
5c21c4cc 2006
9f1e0d61
KG		 2007 /* check for matching IP prefix and subnet length (V1) */
2008 prfx_rc = smc_listen_prfx_check(new_smc, pclc);
2009 if (prfx_rc)
2010 smc_find_ism_store_rc(prfx_rc, ini);
5c21c4cc
UB		 2011
2012 /* get vlan id from IP device */
2013 if (smc_vlan_by_tcpsk(new_smc->clcsock, ini))
3752404a 2014 return ini->rc ?: SMC_CLC_DECL_GETVLANERR;
5c21c4cc
UB		 2015
2016 /* check for ISM device matching V1 proposed device */
9f1e0d61
KG		 2017 if (!prfx_rc)
2018 smc_find_ism_v1_device_serv(new_smc, pclc, ini);
5c21c4cc
UB		 2019 if (ini->ism_dev[0])
2020 return 0;
2021
9f1e0d61
KG		 2022 if (!smcr_indicated(pclc->hdr.typev1) &&
2023 !smcr_indicated(pclc->hdr.typev2))
3752404a
KG		 2024 /* skip RDMA and decline */
2025 return ini->rc ?: SMC_CLC_DECL_NOSMCDDEV;
7affc809 2026
9f1e0d61
KG		 2027 /* check if RDMA V2 is available */
2028 smc_find_rdma_v2_device_serv(new_smc, pclc, ini);
2029 if (ini->smcrv2.ib_dev_v2)
2030 return 0;
3752404a 2031
9f1e0d61
KG		 2032 /* check if RDMA V1 is available */
2033 if (!prfx_rc) {
2034 int rc;
2035
2036 rc = smc_find_rdma_v1_device_serv(new_smc, pclc, ini);
2037 smc_find_ism_store_rc(rc, ini);
2038 return (!rc) ? 0 : ini->rc;
2039 }
2040 return SMC_CLC_DECL_NOSMCDEV;
7affc809
UB		 2041}
2042
3b2dec26 2043/* listen worker: finish RDMA setup */
1ca52fcf
UB		 2044static int smc_listen_rdma_finish(struct smc_sock *new_smc,
2045 struct smc_clc_msg_accept_confirm *cclc,
9f1e0d61
KG		 2046 bool local_first,
2047 struct smc_init_info *ini)
3b2dec26 2048{
387707fd 2049 struct smc_link *link = new_smc->conn.lnk;
3b2dec26 2050 int reason_code = 0;
a046d57d 2051
5ac54d87 2052 if (local_first)
9f1e0d61 2053 smc_link_save_peer_info(link, cclc, ini);
a046d57d 2054
0c881ada
UB		 2055 if (smc_rmb_rtoken_handling(&new_smc->conn, link, cclc))
2056 return SMC_CLC_DECL_ERR_RTOK;
bd4ad577 2057
5ac54d87 2058 if (local_first) {
0c881ada
UB		 2059 if (smc_ib_ready_link(link))
2060 return SMC_CLC_DECL_ERR_RDYLNK;
9bf9abea 2061 /* QP confirmation over RoCE fabric */
4667bb4a 2062 smc_llc_flow_initiate(link->lgr, SMC_LLC_FLOW_ADD_LINK);
b9247544 2063 reason_code = smcr_serv_conf_first_link(new_smc);
4667bb4a 2064 smc_llc_flow_stop(link->lgr, &link->lgr->llc_flow_lcl);
bd4ad577 2065 }
1ca52fcf 2066 return reason_code;
3b2dec26 2067}
e6727f39 2068
7affc809 2069/* setup for connection of server */
3b2dec26
HW		 2070static void smc_listen_work(struct work_struct *work)
2071{
2072 struct smc_sock *new_smc = container_of(work, struct smc_sock,
2073 smc_listen_work);
2074 struct socket *newclcsock = new_smc->clcsock;
a7c9c5f4 2075 struct smc_clc_msg_accept_confirm *cclc;
6bb14e48 2076 struct smc_clc_msg_proposal_area *buf;
3b2dec26 2077 struct smc_clc_msg_proposal *pclc;
3fc64937 2078 struct smc_init_info *ini = NULL;
9f1e0d61
KG		 2079 u8 proposal_version = SMC_V1;
2080 u8 accept_version;
3b2dec26 2081 int rc = 0;
3b2dec26 2082
fd57770d
KG		 2083 if (new_smc->listen_smc->sk.sk_state != SMC_LISTEN)
2084 return smc_listen_out_err(new_smc);
2085
3b2dec26
HW		 2086 if (new_smc->use_fallback) {
2087 smc_listen_out_connected(new_smc);
2088 return;
a046d57d 2089 }
a046d57d 2090
3b2dec26
HW		 2091 /* check if peer is smc capable */
2092 if (!tcp_sk(newclcsock->sk)->syn_smc) {
3924702d
WG		 2093 rc = smc_switch_to_fallback(new_smc, SMC_CLC_DECL_PEERNOSMC);
2094 if (rc)
2095 smc_listen_out_err(new_smc);
2096 else
2097 smc_listen_out_connected(new_smc);
3b2dec26
HW		 2098 return;
2099 }
a046d57d 2100
3b2dec26
HW		 2101 /* do inband token exchange -
2102 * wait for and receive SMC Proposal CLC message
2103 */
6bb14e48
UB		 2104 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
2105 if (!buf) {
2106 rc = SMC_CLC_DECL_MEM;
2107 goto out_decl;
2108 }
2109 pclc = (struct smc_clc_msg_proposal *)buf;
2110 rc = smc_clc_wait_msg(new_smc, pclc, sizeof(*buf),
228bae05 2111 SMC_CLC_PROPOSAL, CLC_WAIT_TIME);
9aa68d29
KG		 2112 if (rc)
2113 goto out_decl;
9f1e0d61
KG		 2114
2115 if (pclc->hdr.version > SMC_V1)
2116 proposal_version = SMC_V2;
a046d57d 2117
e8d726c8 2118 /* IPSec connections opt out of SMC optimizations */
3b2dec26 2119 if (using_ipsec(new_smc)) {
9aa68d29
KG		 2120 rc = SMC_CLC_DECL_IPSEC;
2121 goto out_decl;
3b2dec26
HW		 2122 }
2123
3fc64937
UB		 2124 ini = kzalloc(sizeof(*ini), GFP_KERNEL);
2125 if (!ini) {
2126 rc = SMC_CLC_DECL_MEM;
2127 goto out_decl;
2128 }
2129
5c21c4cc
UB		 2130 /* initial version checking */
2131 rc = smc_listen_v2_check(new_smc, pclc, ini);
2132 if (rc)
9aa68d29 2133 goto out_decl;
fba7e8ef 2134
72a36a8a 2135 mutex_lock(&smc_server_lgr_pending);
3b2dec26
HW		 2136 smc_close_init(new_smc);
2137 smc_rx_init(new_smc);
2138 smc_tx_init(new_smc);
2139
7affc809 2140 /* determine ISM or RoCE device used for connection */
3fc64937 2141 rc = smc_listen_find_device(new_smc, pclc, ini);
7affc809
UB		 2142 if (rc)
2143 goto out_unlock;
3b2dec26
HW		 2144
2145 /* send SMC Accept CLC message */
9f1e0d61 2146 accept_version = ini->is_smcd ? ini->smcd_version : ini->smcr_version;
a7c9c5f4 2147 rc = smc_clc_send_accept(new_smc, ini->first_contact_local,
9f1e0d61 2148 accept_version, ini->negotiated_eid);
9aa68d29
KG		 2149 if (rc)
2150 goto out_unlock;
3b2dec26 2151
62c7139f 2152 /* SMC-D does not need this lock any more */
3fc64937 2153 if (ini->is_smcd)
72a36a8a 2154 mutex_unlock(&smc_server_lgr_pending);
62c7139f 2155
3b2dec26 2156 /* receive SMC Confirm CLC message */
9047a617
KG		 2157 memset(buf, 0, sizeof(*buf));
2158 cclc = (struct smc_clc_msg_accept_confirm *)buf;
2159 rc = smc_clc_wait_msg(new_smc, cclc, sizeof(*buf),
228bae05
KG		 2160 SMC_CLC_CONFIRM, CLC_WAIT_TIME);
2161 if (rc) {
3fc64937 2162 if (!ini->is_smcd)
9aa68d29
KG		 2163 goto out_unlock;
2164 goto out_decl;
3b2dec26
HW		 2165 }
2166
2167 /* finish worker */
3fc64937 2168 if (!ini->is_smcd) {
a7c9c5f4 2169 rc = smc_listen_rdma_finish(new_smc, cclc,
9f1e0d61 2170 ini->first_contact_local, ini);
62c7139f 2171 if (rc)
0c881ada
UB		 2172 goto out_unlock;
2173 mutex_unlock(&smc_server_lgr_pending);
1ca52fcf 2174 }
a7c9c5f4 2175 smc_conn_save_peer_info(new_smc, cclc);
3b2dec26 2176 smc_listen_out_connected(new_smc);
194730a9 2177 SMC_STAT_SERV_SUCC_INC(sock_net(newclcsock->sk), ini);
ac679364 2178 goto out_free;
9aa68d29
KG		 2179
2180out_unlock:
2181 mutex_unlock(&smc_server_lgr_pending);
2182out_decl:
4a9baf45 2183 smc_listen_decline(new_smc, rc, ini ? ini->first_contact_local : 0,
9f1e0d61 2184 proposal_version);
ac679364 2185out_free:
3fc64937 2186 kfree(ini);
6bb14e48 2187 kfree(buf);
a046d57d
UB		 2188}
2189
2190static void smc_tcp_listen_work(struct work_struct *work)
2191{
2192 struct smc_sock *lsmc = container_of(work, struct smc_sock,
2193 tcp_listen_work);
3163c507 2194 struct sock *lsk = &lsmc->sk;
a046d57d
UB		 2195 struct smc_sock *new_smc;
2196 int rc = 0;
2197
3163c507
UB		 2198 lock_sock(lsk);
2199 while (lsk->sk_state == SMC_LISTEN) {
a046d57d 2200 rc = smc_clcsock_accept(lsmc, &new_smc);
a60a2b1e 2201 if (rc) /* clcsock accept queue empty or error */
a046d57d
UB		 2202 goto out;
2203 if (!new_smc)
2204 continue;
2205
2206 new_smc->listen_smc = lsmc;
ee9dfbef 2207 new_smc->use_fallback = lsmc->use_fallback;
603cc149 2208 new_smc->fallback_rsn = lsmc->fallback_rsn;
3163c507 2209 sock_hold(lsk); /* sock_put in smc_listen_work */
a046d57d
UB		 2210 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
2211 smc_copy_sock_settings_to_smc(new_smc);
bd58c7e0
UB		 2212 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
2213 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
51f1de79 2214 sock_hold(&new_smc->sk); /* sock_put in passive closing */
22ef473d 2215 if (!queue_work(smc_hs_wq, &new_smc->smc_listen_work))
51f1de79 2216 sock_put(&new_smc->sk);
a046d57d
UB		 2217 }
2218
2219out:
3163c507 2220 release_sock(lsk);
a60a2b1e
UB		 2221 sock_put(&lsmc->sk); /* sock_hold in smc_clcsock_data_ready() */
2222}
2223
2224static void smc_clcsock_data_ready(struct sock *listen_clcsock)
2225{
1ddd4b02
WG		 2226 struct smc_sock *lsmc =
2227 smc_clcsock_user_data(listen_clcsock);
a60a2b1e 2228
a60a2b1e
UB		 2229 if (!lsmc)
2230 return;
2231 lsmc->clcsk_data_ready(listen_clcsock);
2232 if (lsmc->sk.sk_state == SMC_LISTEN) {
2233 sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
22ef473d 2234 if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work))
a60a2b1e
UB		 2235 sock_put(&lsmc->sk);
2236 }
a046d57d
UB		 2237}
2238
ac713874
UB		 2239static int smc_listen(struct socket *sock, int backlog)
2240{
2241 struct sock *sk = sock->sk;
2242 struct smc_sock *smc;
2243 int rc;
2244
2245 smc = smc_sk(sk);
2246 lock_sock(sk);
2247
2248 rc = -EINVAL;
cd206360
UB		 2249 if ((sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) ||
2250 smc->connect_nonblock)
ac713874
UB		 2251 goto out;
2252
2253 rc = 0;
2254 if (sk->sk_state == SMC_LISTEN) {
2255 sk->sk_max_ack_backlog = backlog;
2256 goto out;
2257 }
2258 /* some socket options are handled in core, so we could not apply
2259 * them to the clc socket -- copy smc socket options to clc socket
2260 */
2261 smc_copy_sock_settings_to_clc(smc);
ee9dfbef
UB		 2262 if (!smc->use_fallback)
2263 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
ac713874 2264
a60a2b1e
UB		 2265 /* save original sk_data_ready function and establish
2266 * smc-specific sk_data_ready function
2267 */
2268 smc->clcsk_data_ready = smc->clcsock->sk->sk_data_ready;
2269 smc->clcsock->sk->sk_data_ready = smc_clcsock_data_ready;
2270 smc->clcsock->sk->sk_user_data =
2271 (void *)((uintptr_t)smc | SK_USER_DATA_NOCOPY);
ac713874 2272 rc = kernel_listen(smc->clcsock, backlog);
6ce3642c
GD		 2273 if (rc) {
2274 smc->clcsock->sk->sk_data_ready = smc->clcsk_data_ready;
ac713874 2275 goto out;
6ce3642c 2276 }
ac713874
UB		 2277 sk->sk_max_ack_backlog = backlog;
2278 sk->sk_ack_backlog = 0;
2279 sk->sk_state = SMC_LISTEN;
2280
2281out:
2282 release_sock(sk);
2283 return rc;
2284}
2285
2286static int smc_accept(struct socket *sock, struct socket *new_sock,
cdfbabfb 2287 int flags, bool kern)
ac713874 2288{
a046d57d
UB		 2289 struct sock *sk = sock->sk, *nsk;
2290 DECLARE_WAITQUEUE(wait, current);
ac713874 2291 struct smc_sock *lsmc;
a046d57d
UB		 2292 long timeo;
2293 int rc = 0;
ac713874
UB		 2294
2295 lsmc = smc_sk(sk);
51f1de79 2296 sock_hold(sk); /* sock_put below */
ac713874
UB		 2297 lock_sock(sk);
2298
2299 if (lsmc->sk.sk_state != SMC_LISTEN) {
2300 rc = -EINVAL;
abb190f1 2301 release_sock(sk);
ac713874
UB		 2302 goto out;
2303 }
2304
a046d57d
UB		 2305 /* Wait for an incoming connection */
2306 timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
2307 add_wait_queue_exclusive(sk_sleep(sk), &wait);
2308 while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
2309 set_current_state(TASK_INTERRUPTIBLE);
2310 if (!timeo) {
2311 rc = -EAGAIN;
2312 break;
2313 }
2314 release_sock(sk);
2315 timeo = schedule_timeout(timeo);
2316 /* wakeup by sk_data_ready in smc_listen_work() */
2317 sched_annotate_sleep();
2318 lock_sock(sk);
2319 if (signal_pending(current)) {
2320 rc = sock_intr_errno(timeo);
2321 break;
2322 }
2323 }
2324 set_current_state(TASK_RUNNING);
2325 remove_wait_queue(sk_sleep(sk), &wait);
ac713874 2326
a046d57d
UB		 2327 if (!rc)
2328 rc = sock_error(nsk);
abb190f1
UB		 2329 release_sock(sk);
2330 if (rc)
2331 goto out;
2332
2333 if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
2334 /* wait till data arrives on the socket */
2335 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
2336 MSEC_PER_SEC);
2337 if (smc_sk(nsk)->use_fallback) {
2338 struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
2339
2340 lock_sock(clcsk);
2341 if (skb_queue_empty(&clcsk->sk_receive_queue))
2342 sk_wait_data(clcsk, &timeo, NULL);
2343 release_sock(clcsk);
2344 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
2345 lock_sock(nsk);
b51fa1b1 2346 smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
abb190f1
UB		 2347 release_sock(nsk);
2348 }
2349 }
ac713874
UB		 2350
2351out:
51f1de79 2352 sock_put(sk); /* sock_hold above */
ac713874
UB		 2353 return rc;
2354}
2355
2356static int smc_getname(struct socket *sock, struct sockaddr *addr,
9b2c45d4 2357 int peer)
ac713874
UB		 2358{
2359 struct smc_sock *smc;
2360
b38d7324
UB		 2361 if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
2362 (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
ac713874
UB		 2363 return -ENOTCONN;
2364
2365 smc = smc_sk(sock->sk);
2366
9b2c45d4 2367 return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
ac713874
UB		 2368}
2369
2370static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
2371{
2372 struct sock *sk = sock->sk;
2373 struct smc_sock *smc;
2374 int rc = -EPIPE;
2375
2376 smc = smc_sk(sk);
2377 lock_sock(sk);
b38d7324
UB		 2378 if ((sk->sk_state != SMC_ACTIVE) &&
2379 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2380 (sk->sk_state != SMC_INIT))
ac713874 2381 goto out;
ee9dfbef
UB		 2382
2383 if (msg->msg_flags & MSG_FASTOPEN) {
cd206360 2384 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
3924702d
WG		 2385 rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
2386 if (rc)
2387 goto out;
ee9dfbef
UB		 2388 } else {
2389 rc = -EINVAL;
2390 goto out;
2391 }
2392 }
2393
e0e4b8fa 2394 if (smc->use_fallback) {
ac713874 2395 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
e0e4b8fa 2396 } else {
e6727f39 2397 rc = smc_tx_sendmsg(smc, msg, len);
e0e4b8fa
GG		 2398 SMC_STAT_TX_PAYLOAD(smc, len, rc);
2399 }
ac713874
UB		 2400out:
2401 release_sock(sk);
2402 return rc;
2403}
2404
2405static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
2406 int flags)
2407{
2408 struct sock *sk = sock->sk;
2409 struct smc_sock *smc;
2410 int rc = -ENOTCONN;
2411
2412 smc = smc_sk(sk);
2413 lock_sock(sk);
51c5aba3
KG		 2414 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2415 /* socket was connected before, no more data to read */
2416 rc = 0;
2417 goto out;
2418 }
b38d7324
UB		 2419 if ((sk->sk_state == SMC_INIT) ||
2420 (sk->sk_state == SMC_LISTEN) ||
2421 (sk->sk_state == SMC_CLOSED))
ac713874
UB		 2422 goto out;
2423
b38d7324
UB		 2424 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2425 rc = 0;
2426 goto out;
2427 }
2428
9014db20 2429 if (smc->use_fallback) {
ac713874 2430 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
9014db20
SR		 2431 } else {
2432 msg->msg_namelen = 0;
2433 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
e0e4b8fa 2434 SMC_STAT_RX_PAYLOAD(smc, rc, rc);
9014db20 2435 }
b38d7324 2436
ac713874
UB		 2437out:
2438 release_sock(sk);
2439 return rc;
2440}
2441
ade994f4 2442static __poll_t smc_accept_poll(struct sock *parent)
a046d57d 2443{
8dce2786 2444 struct smc_sock *isk = smc_sk(parent);
63e2480c 2445 __poll_t mask = 0;
a046d57d 2446
8dce2786
UB		 2447 spin_lock(&isk->accept_q_lock);
2448 if (!list_empty(&isk->accept_q))
a9a08845 2449 mask = EPOLLIN | EPOLLRDNORM;
8dce2786 2450 spin_unlock(&isk->accept_q_lock);
a046d57d 2451
8dce2786 2452 return mask;
a046d57d
UB		 2453}
2454
a11e1d43
LT		 2455static __poll_t smc_poll(struct file *file, struct socket *sock,
2456 poll_table *wait)
ac713874
UB		 2457{
2458 struct sock *sk = sock->sk;
ac713874 2459 struct smc_sock *smc;
50717a37 2460 __poll_t mask = 0;
ac713874 2461
8dce2786 2462 if (!sk)
a9a08845 2463 return EPOLLNVAL;
8dce2786 2464
ac713874 2465 smc = smc_sk(sock->sk);
648a5a7a 2466 if (smc->use_fallback) {
a046d57d 2467 /* delegate to CLC child sock */
a11e1d43 2468 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
784813ae 2469 sk->sk_err = smc->clcsock->sk->sk_err;
ac713874 2470 } else {
410da1e1 2471 if (sk->sk_state != SMC_CLOSED)
89ab066d 2472 sock_poll_wait(file, sock, wait);
a046d57d 2473 if (sk->sk_err)
a9a08845 2474 mask |= EPOLLERR;
b38d7324
UB		 2475 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
2476 (sk->sk_state == SMC_CLOSED))
a9a08845 2477 mask |= EPOLLHUP;
8dce2786
UB		 2478 if (sk->sk_state == SMC_LISTEN) {
2479 /* woken up by sk_data_ready in smc_listen_work() */
50717a37
UB		 2480 mask |= smc_accept_poll(sk);
2481 } else if (smc->use_fallback) { /* as result of connect_work()*/
2482 mask |= smc->clcsock->ops->poll(file, smc->clcsock,
2483 wait);
2484 sk->sk_err = smc->clcsock->sk->sk_err;
8dce2786 2485 } else {
50717a37
UB		 2486 if ((sk->sk_state != SMC_INIT &&
2487 atomic_read(&smc->conn.sndbuf_space)) ||
8dce2786 2488 sk->sk_shutdown & SEND_SHUTDOWN) {
a9a08845 2489 mask |= EPOLLOUT | EPOLLWRNORM;
8dce2786
UB		 2490 } else {
2491 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2492 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2493 }
2494 if (atomic_read(&smc->conn.bytes_to_rcv))
a9a08845 2495 mask |= EPOLLIN | EPOLLRDNORM;
8dce2786 2496 if (sk->sk_shutdown & RCV_SHUTDOWN)
a9a08845 2497 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
8dce2786 2498 if (sk->sk_state == SMC_APPCLOSEWAIT1)
a9a08845 2499 mask |= EPOLLIN;
71d117f5
KG		 2500 if (smc->conn.urg_state == SMC_URG_VALID)
2501 mask |= EPOLLPRI;
8dce2786 2502 }
ac713874
UB		 2503 }
2504
2505 return mask;
2506}
2507
2508static int smc_shutdown(struct socket *sock, int how)
2509{
2510 struct sock *sk = sock->sk;
76aa7d8c 2511 bool do_shutdown = true;
ac713874
UB		 2512 struct smc_sock *smc;
2513 int rc = -EINVAL;
76aa7d8c 2514 int old_state;
b38d7324 2515 int rc1 = 0;
ac713874
UB		 2516
2517 smc = smc_sk(sk);
2518
2519 if ((how < SHUT_RD) || (how > SHUT_RDWR))
b38d7324 2520 return rc;
ac713874
UB		 2521
2522 lock_sock(sk);
2523
2524 rc = -ENOTCONN;
caa21e19 2525 if ((sk->sk_state != SMC_ACTIVE) &&
b38d7324
UB		 2526 (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
2527 (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
2528 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
2529 (sk->sk_state != SMC_APPCLOSEWAIT2) &&
2530 (sk->sk_state != SMC_APPFINCLOSEWAIT))
ac713874
UB		 2531 goto out;
2532 if (smc->use_fallback) {
2533 rc = kernel_sock_shutdown(smc->clcsock, how);
2534 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
2535 if (sk->sk_shutdown == SHUTDOWN_MASK)
2536 sk->sk_state = SMC_CLOSED;
b38d7324 2537 goto out;
ac713874 2538 }
b38d7324
UB		 2539 switch (how) {
2540 case SHUT_RDWR: /* shutdown in both directions */
76aa7d8c 2541 old_state = sk->sk_state;
b38d7324 2542 rc = smc_close_active(smc);
76aa7d8c
TL		 2543 if (old_state == SMC_ACTIVE &&
2544 sk->sk_state == SMC_PEERCLOSEWAIT1)
2545 do_shutdown = false;
b38d7324
UB		 2546 break;
2547 case SHUT_WR:
2548 rc = smc_close_shutdown_write(smc);
2549 break;
2550 case SHUT_RD:
1255fcb2
UB		 2551 rc = 0;
2552 /* nothing more to do because peer is not involved */
b38d7324
UB		 2553 break;
2554 }
76aa7d8c 2555 if (do_shutdown && smc->clcsock)
1255fcb2 2556 rc1 = kernel_sock_shutdown(smc->clcsock, how);
b38d7324
UB		 2557 /* map sock_shutdown_cmd constants to sk_shutdown value range */
2558 sk->sk_shutdown |= how + 1;
ac713874
UB		 2559
2560out:
2561 release_sock(sk);
b38d7324 2562 return rc ? rc : rc1;
ac713874
UB		 2563}
2564
2565static int smc_setsockopt(struct socket *sock, int level, int optname,
a7b75c5a 2566 sockptr_t optval, unsigned int optlen)
ac713874
UB		 2567{
2568 struct sock *sk = sock->sk;
2569 struct smc_sock *smc;
01d2f7e2 2570 int val, rc;
ac713874 2571
86214366
CW		 2572 if (level == SOL_TCP && optname == TCP_ULP)
2573 return -EOPNOTSUPP;
2574
ac713874
UB		 2575 smc = smc_sk(sk);
2576
2577 /* generic setsockopts reaching us here always apply to the
2578 * CLC socket
2579 */
3924702d
WG		 2580 mutex_lock(&smc->clcsock_release_lock);
2581 if (!smc->clcsock) {
2582 mutex_unlock(&smc->clcsock_release_lock);
2583 return -EBADF;
2584 }
a44d9e72
CH		 2585 if (unlikely(!smc->clcsock->ops->setsockopt))
2586 rc = -EOPNOTSUPP;
2587 else
2588 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
2589 optval, optlen);
ee9dfbef
UB		 2590 if (smc->clcsock->sk->sk_err) {
2591 sk->sk_err = smc->clcsock->sk->sk_err;
e3ae2365 2592 sk_error_report(sk);
ee9dfbef 2593 }
3924702d 2594 mutex_unlock(&smc->clcsock_release_lock);
ee9dfbef 2595
01d2f7e2 2596 if (optlen < sizeof(int))
3dc9f558 2597 return -EINVAL;
a7b75c5a 2598 if (copy_from_sockptr(&val, optval, sizeof(int)))
ac0107ed 2599 return -EFAULT;
01d2f7e2 2600
ee9dfbef 2601 lock_sock(sk);
86434744
UB		 2602 if (rc || smc->use_fallback)
2603 goto out;
ee9dfbef 2604 switch (optname) {
ee9dfbef
UB		 2605 case TCP_FASTOPEN:
2606 case TCP_FASTOPEN_CONNECT:
2607 case TCP_FASTOPEN_KEY:
2608 case TCP_FASTOPEN_NO_COOKIE:
2609 /* option not supported by SMC */
8204df72 2610 if (sk->sk_state == SMC_INIT && !smc->connect_nonblock) {
3924702d 2611 rc = smc_switch_to_fallback(smc, SMC_CLC_DECL_OPTUNSUPP);
ee9dfbef 2612 } else {
86434744 2613 rc = -EINVAL;
ee9dfbef
UB		 2614 }
2615 break;
01d2f7e2 2616 case TCP_NODELAY:
f9cedf1a
UB		 2617 if (sk->sk_state != SMC_INIT &&
2618 sk->sk_state != SMC_LISTEN &&
2619 sk->sk_state != SMC_CLOSED) {
e0e4b8fa 2620 if (val) {
194730a9 2621 SMC_STAT_INC(smc, ndly_cnt);
22ef473d
KG		 2622 mod_delayed_work(smc->conn.lgr->tx_wq,
2623 &smc->conn.tx_work, 0);
e0e4b8fa 2624 }
01d2f7e2
UB		 2625 }
2626 break;
2627 case TCP_CORK:
f9cedf1a
UB		 2628 if (sk->sk_state != SMC_INIT &&
2629 sk->sk_state != SMC_LISTEN &&
2630 sk->sk_state != SMC_CLOSED) {
e0e4b8fa 2631 if (!val) {
194730a9 2632 SMC_STAT_INC(smc, cork_cnt);
131c0127
TL		 2633 smc_tx_pending(&smc->conn);
2634 cancel_delayed_work(&smc->conn.tx_work);
e0e4b8fa 2635 }
01d2f7e2
UB		 2636 }
2637 break;
abb190f1
UB		 2638 case TCP_DEFER_ACCEPT:
2639 smc->sockopt_defer_accept = val;
2640 break;
ee9dfbef
UB		 2641 default:
2642 break;
2643 }
86434744 2644out:
ee9dfbef
UB		 2645 release_sock(sk);
2646
2647 return rc;
ac713874
UB		 2648}
2649
2650static int smc_getsockopt(struct socket *sock, int level, int optname,
2651 char __user *optval, int __user *optlen)
2652{
2653 struct smc_sock *smc;
3924702d 2654 int rc;
ac713874
UB		 2655
2656 smc = smc_sk(sock->sk);
3924702d
WG		 2657 mutex_lock(&smc->clcsock_release_lock);
2658 if (!smc->clcsock) {
2659 mutex_unlock(&smc->clcsock_release_lock);
2660 return -EBADF;
2661 }
ac713874 2662 /* socket options apply to the CLC socket */
3924702d
WG		 2663 if (unlikely(!smc->clcsock->ops->getsockopt)) {
2664 mutex_unlock(&smc->clcsock_release_lock);
a44d9e72 2665 return -EOPNOTSUPP;
3924702d
WG		 2666 }
2667 rc = smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
2668 optval, optlen);
2669 mutex_unlock(&smc->clcsock_release_lock);
2670 return rc;
ac713874
UB		 2671}
2672
2673static int smc_ioctl(struct socket *sock, unsigned int cmd,
2674 unsigned long arg)
2675{
de8474eb
SR		 2676 union smc_host_cursor cons, urg;
2677 struct smc_connection *conn;
ac713874 2678 struct smc_sock *smc;
9b67e26f 2679 int answ;
ac713874
UB		 2680
2681 smc = smc_sk(sock->sk);
de8474eb 2682 conn = &smc->conn;
7311d665 2683 lock_sock(&smc->sk);
9b67e26f 2684 if (smc->use_fallback) {
7311d665
UB		 2685 if (!smc->clcsock) {
2686 release_sock(&smc->sk);
9b67e26f 2687 return -EBADF;
7311d665
UB		 2688 }
2689 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
2690 release_sock(&smc->sk);
2691 return answ;
9b67e26f
UB		 2692 }
2693 switch (cmd) {
2694 case SIOCINQ: /* same as FIONREAD */
1992d998
UB		 2695 if (smc->sk.sk_state == SMC_LISTEN) {
2696 release_sock(&smc->sk);
9b67e26f 2697 return -EINVAL;
1992d998 2698 }
2351abe6
UB		 2699 if (smc->sk.sk_state == SMC_INIT ||
2700 smc->sk.sk_state == SMC_CLOSED)
2701 answ = 0;
2702 else
2703 answ = atomic_read(&smc->conn.bytes_to_rcv);
9b67e26f
UB		 2704 break;
2705 case SIOCOUTQ:
2706 /* output queue size (not send + not acked) */
1992d998
UB		 2707 if (smc->sk.sk_state == SMC_LISTEN) {
2708 release_sock(&smc->sk);
9b67e26f 2709 return -EINVAL;
1992d998 2710 }
2351abe6
UB		 2711 if (smc->sk.sk_state == SMC_INIT ||
2712 smc->sk.sk_state == SMC_CLOSED)
2713 answ = 0;
2714 else
2715 answ = smc->conn.sndbuf_desc->len -
9b67e26f
UB		 2716 atomic_read(&smc->conn.sndbuf_space);
2717 break;
2718 case SIOCOUTQNSD:
2719 /* output queue size (not send only) */
1992d998
UB		 2720 if (smc->sk.sk_state == SMC_LISTEN) {
2721 release_sock(&smc->sk);
9b67e26f 2722 return -EINVAL;
1992d998 2723 }
2351abe6
UB		 2724 if (smc->sk.sk_state == SMC_INIT ||
2725 smc->sk.sk_state == SMC_CLOSED)
2726 answ = 0;
2727 else
2728 answ = smc_tx_prepared_sends(&smc->conn);
9b67e26f 2729 break;
de8474eb 2730 case SIOCATMARK:
1992d998
UB		 2731 if (smc->sk.sk_state == SMC_LISTEN) {
2732 release_sock(&smc->sk);
de8474eb 2733 return -EINVAL;
1992d998 2734 }
de8474eb
SR		 2735 if (smc->sk.sk_state == SMC_INIT ||
2736 smc->sk.sk_state == SMC_CLOSED) {
2737 answ = 0;
2738 } else {
bac6de7b
SR		 2739 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
2740 smc_curs_copy(&urg, &conn->urg_curs, conn);
de8474eb
SR		 2741 answ = smc_curs_diff(conn->rmb_desc->len,
2742 &cons, &urg) == 1;
2743 }
2744 break;
9b67e26f 2745 default:
1992d998 2746 release_sock(&smc->sk);
9b67e26f
UB		 2747 return -ENOIOCTLCMD;
2748 }
1992d998 2749 release_sock(&smc->sk);
9b67e26f
UB		 2750
2751 return put_user(answ, (int __user *)arg);
ac713874
UB		 2752}
2753
2754static ssize_t smc_sendpage(struct socket *sock, struct page *page,
2755 int offset, size_t size, int flags)
2756{
2757 struct sock *sk = sock->sk;
2758 struct smc_sock *smc;
2759 int rc = -EPIPE;
2760
2761 smc = smc_sk(sk);
2762 lock_sock(sk);
bda27ff5
SR		 2763 if (sk->sk_state != SMC_ACTIVE) {
2764 release_sock(sk);
ac713874 2765 goto out;
bda27ff5
SR		 2766 }
2767 release_sock(sk);
e0e4b8fa 2768 if (smc->use_fallback) {
ac713874
UB		 2769 rc = kernel_sendpage(smc->clcsock, page, offset,
2770 size, flags);
e0e4b8fa 2771 } else {
194730a9 2772 SMC_STAT_INC(smc, sendpage_cnt);
ac713874 2773 rc = sock_no_sendpage(sock, page, offset, size, flags);
e0e4b8fa 2774 }
ac713874
UB		 2775
2776out:
ac713874
UB		 2777 return rc;
2778}
2779
9014db20
SR		 2780/* Map the affected portions of the rmbe into an spd, note the number of bytes
2781 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
2782 * updates till whenever a respective page has been fully processed.
2783 * Note that subsequent recv() calls have to wait till all splice() processing
2784 * completed.
2785 */
ac713874
UB		 2786static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
2787 struct pipe_inode_info *pipe, size_t len,
9014db20 2788 unsigned int flags)
ac713874
UB		 2789{
2790 struct sock *sk = sock->sk;
2791 struct smc_sock *smc;
2792 int rc = -ENOTCONN;
2793
2794 smc = smc_sk(sk);
2795 lock_sock(sk);
51c5aba3
KG		 2796 if (sk->sk_state == SMC_CLOSED && (sk->sk_shutdown & RCV_SHUTDOWN)) {
2797 /* socket was connected before, no more data to read */
2798 rc = 0;
2799 goto out;
2800 }
9014db20
SR		 2801 if (sk->sk_state == SMC_INIT ||
2802 sk->sk_state == SMC_LISTEN ||
2803 sk->sk_state == SMC_CLOSED)
2804 goto out;
2805
2806 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
2807 rc = 0;
ac713874 2808 goto out;
9014db20
SR		 2809 }
2810
ac713874
UB		 2811 if (smc->use_fallback) {
2812 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
2813 pipe, len, flags);
2814 } else {
9014db20
SR		 2815 if (*ppos) {
2816 rc = -ESPIPE;
2817 goto out;
2818 }
2819 if (flags & SPLICE_F_NONBLOCK)
2820 flags = MSG_DONTWAIT;
2821 else
2822 flags = 0;
194730a9 2823 SMC_STAT_INC(smc, splice_cnt);
9014db20 2824 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
ac713874
UB		 2825 }
2826out:
2827 release_sock(sk);
9014db20 2828
ac713874
UB		 2829 return rc;
2830}
2831
2832/* must look like tcp */
2833static const struct proto_ops smc_sock_ops = {
2834 .family = PF_SMC,
2835 .owner = THIS_MODULE,
2836 .release = smc_release,
2837 .bind = smc_bind,
2838 .connect = smc_connect,
2839 .socketpair = sock_no_socketpair,
2840 .accept = smc_accept,
2841 .getname = smc_getname,
a11e1d43 2842 .poll = smc_poll,
ac713874
UB		 2843 .ioctl = smc_ioctl,
2844 .listen = smc_listen,
2845 .shutdown = smc_shutdown,
2846 .setsockopt = smc_setsockopt,
2847 .getsockopt = smc_getsockopt,
2848 .sendmsg = smc_sendmsg,
2849 .recvmsg = smc_recvmsg,
2850 .mmap = sock_no_mmap,
2851 .sendpage = smc_sendpage,
2852 .splice_read = smc_splice_read,
2853};
2854
2855static int smc_create(struct net *net, struct socket *sock, int protocol,
2856 int kern)
2857{
aaa4d33f 2858 int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
ac713874
UB		 2859 struct smc_sock *smc;
2860 struct sock *sk;
2861 int rc;
2862
2863 rc = -ESOCKTNOSUPPORT;
2864 if (sock->type != SOCK_STREAM)
2865 goto out;
2866
2867 rc = -EPROTONOSUPPORT;
aaa4d33f 2868 if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
ac713874
UB		 2869 goto out;
2870
2871 rc = -ENOBUFS;
2872 sock->ops = &smc_sock_ops;
aaa4d33f 2873 sk = smc_sock_alloc(net, sock, protocol);
ac713874
UB		 2874 if (!sk)
2875 goto out;
2876
2877 /* create internal TCP socket for CLC handshake and fallback */
2878 smc = smc_sk(sk);
a046d57d 2879 smc->use_fallback = false; /* assume rdma capability first */
603cc149 2880 smc->fallback_rsn = 0;
aaa4d33f
KG		 2881 rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
2882 &smc->clcsock);
a5dcb73b 2883 if (rc) {
ac713874 2884 sk_common_release(sk);
a5dcb73b
DC		 2885 goto out;
2886 }
cd6851f3
UB		 2887 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
2888 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
ac713874
UB		 2889
2890out:
2891 return rc;
2892}
2893
2894static const struct net_proto_family smc_sock_family_ops = {
2895 .family = PF_SMC,
2896 .owner = THIS_MODULE,
2897 .create = smc_create,
2898};
2899
64e28b52
HW		 2900unsigned int smc_net_id;
2901
2902static __net_init int smc_net_init(struct net *net)
2903{
2904 return smc_pnet_net_init(net);
2905}
2906
2907static void __net_exit smc_net_exit(struct net *net)
2908{
2909 smc_pnet_net_exit(net);
2910}
2911
194730a9
GG		 2912static __net_init int smc_net_stat_init(struct net *net)
2913{
2914 return smc_stats_init(net);
2915}
2916
2917static void __net_exit smc_net_stat_exit(struct net *net)
2918{
2919 smc_stats_exit(net);
2920}
2921
64e28b52
HW		 2922static struct pernet_operations smc_net_ops = {
2923 .init = smc_net_init,
2924 .exit = smc_net_exit,
2925 .id = &smc_net_id,
2926 .size = sizeof(struct smc_net),
2927};
2928
194730a9
GG		 2929static struct pernet_operations smc_net_stat_ops = {
2930 .init = smc_net_stat_init,
2931 .exit = smc_net_stat_exit,
2932};
2933
ac713874
UB		 2934static int __init smc_init(void)
2935{
2936 int rc;
2937
64e28b52
HW		 2938 rc = register_pernet_subsys(&smc_net_ops);
2939 if (rc)
2940 return rc;
2941
194730a9
GG		 2942 rc = register_pernet_subsys(&smc_net_stat_ops);
2943 if (rc)
2944 return rc;
2945
201091eb 2946 smc_ism_init();
b81a5eb7 2947 smc_clc_init();
201091eb 2948
e8372d9d 2949 rc = smc_nl_init();
6812baab 2950 if (rc)
8c33bf1b 2951 goto out_pernet_subsys;
6812baab 2952
e8372d9d
GG		 2953 rc = smc_pnet_init();
2954 if (rc)
2955 goto out_nl;
2956
22ef473d
KG		 2957 rc = -ENOMEM;
2958 smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
2959 if (!smc_hs_wq)
2960 goto out_pnet;
2961
2962 smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
2963 if (!smc_close_wq)
2964 goto out_alloc_hs_wq;
2965
6dabd405
UB		 2966 rc = smc_core_init();
2967 if (rc) {
2968 pr_err("%s: smc_core_init fails with %d\n", __func__, rc);
194730a9 2969 goto out_alloc_wqs;
6dabd405
UB		 2970 }
2971
9bf9abea
UB		 2972 rc = smc_llc_init();
2973 if (rc) {
2974 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
6dabd405 2975 goto out_core;
9bf9abea
UB		 2976 }
2977
5f08318f
UB		 2978 rc = smc_cdc_init();
2979 if (rc) {
2980 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
6dabd405 2981 goto out_core;
5f08318f
UB		 2982 }
2983
ac713874
UB		 2984 rc = proto_register(&smc_proto, 1);
2985 if (rc) {
aaa4d33f 2986 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
6dabd405 2987 goto out_core;
ac713874
UB		 2988 }
2989
aaa4d33f
KG		 2990 rc = proto_register(&smc_proto6, 1);
2991 if (rc) {
2992 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
2993 goto out_proto;
2994 }
2995
ac713874
UB		 2996 rc = sock_register(&smc_sock_family_ops);
2997 if (rc) {
2998 pr_err("%s: sock_register fails with %d\n", __func__, rc);
aaa4d33f 2999 goto out_proto6;
ac713874 3000 }
f16a7dd5 3001 INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
aaa4d33f 3002 INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
ac713874 3003
a4cf0443
UB		 3004 rc = smc_ib_register_client();
3005 if (rc) {
3006 pr_err("%s: ib_register fails with %d\n", __func__, rc);
3007 goto out_sock;
3008 }
3009
c5c1cc9c 3010 static_branch_enable(&tcp_have_smc);
ac713874
UB		 3011 return 0;
3012
a4cf0443
UB		 3013out_sock:
3014 sock_unregister(PF_SMC);
aaa4d33f
KG		 3015out_proto6:
3016 proto_unregister(&smc_proto6);
ac713874
UB		 3017out_proto:
3018 proto_unregister(&smc_proto);
6dabd405
UB		 3019out_core:
3020 smc_core_exit();
22ef473d
KG		 3021out_alloc_wqs:
3022 destroy_workqueue(smc_close_wq);
3023out_alloc_hs_wq:
3024 destroy_workqueue(smc_hs_wq);
6812baab
TR		 3025out_pnet:
3026 smc_pnet_exit();
e8372d9d
GG		 3027out_nl:
3028 smc_nl_exit();
8c33bf1b
Y		 3029out_pernet_subsys:
3030 unregister_pernet_subsys(&smc_net_ops);
3031
ac713874
UB		 3032 return rc;
3033}
3034
3035static void __exit smc_exit(void)
3036{
c5c1cc9c 3037 static_branch_disable(&tcp_have_smc);
ac713874 3038 sock_unregister(PF_SMC);
6dabd405
UB		 3039 smc_core_exit();
3040 smc_ib_unregister_client();
22ef473d
KG		 3041 destroy_workqueue(smc_close_wq);
3042 destroy_workqueue(smc_hs_wq);
aaa4d33f 3043 proto_unregister(&smc_proto6);
ac713874 3044 proto_unregister(&smc_proto);
6812baab 3045 smc_pnet_exit();
e8372d9d 3046 smc_nl_exit();
fefefe10 3047 smc_clc_exit();
194730a9 3048 unregister_pernet_subsys(&smc_net_stat_ops);
64e28b52 3049 unregister_pernet_subsys(&smc_net_ops);
4ead9c96 3050 rcu_barrier();
ac713874
UB		 3051}
3052
3053module_init(smc_init);
3054module_exit(smc_exit);
3055
3056MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
3057MODULE_DESCRIPTION("smc socket address family");
3058MODULE_LICENSE("GPL");
3059MODULE_ALIAS_NETPROTO(PF_SMC);
Ubuntu Jammy Linux kernel mirror