]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
9bf9abea UB |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE | |
4 | * | |
5 | * Link Layer Control (LLC) | |
6 | * | |
9bf9abea UB |
7 | * Copyright IBM Corp. 2016 |
8 | * | |
9 | * Author(s): Klaus Wacker <Klaus.Wacker@de.ibm.com> | |
10 | * Ursula Braun <ubraun@linux.vnet.ibm.com> | |
11 | */ | |
12 | ||
13 | #include <net/tcp.h> | |
14 | #include <rdma/ib_verbs.h> | |
15 | ||
16 | #include "smc.h" | |
17 | #include "smc_core.h" | |
18 | #include "smc_clc.h" | |
19 | #include "smc_llc.h" | |
336ba09f | 20 | #include "smc_pnet.h" |
9bf9abea | 21 | |
0f627126 SR |
22 | #define SMC_LLC_DATA_LEN 40 |
23 | ||
24 | struct smc_llc_hdr { | |
25 | struct smc_wr_rx_hdr common; | |
26 | u8 length; /* 44 */ | |
52bedf37 KG |
27 | #if defined(__BIG_ENDIAN_BITFIELD) |
28 | u8 reserved:4, | |
29 | add_link_rej_rsn:4; | |
30 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
31 | u8 add_link_rej_rsn:4, | |
32 | reserved:4; | |
33 | #endif | |
0f627126 SR |
34 | u8 flags; |
35 | }; | |
36 | ||
75d320d6 KG |
37 | #define SMC_LLC_FLAG_NO_RMBE_EYEC 0x03 |
38 | ||
0f627126 SR |
39 | struct smc_llc_msg_confirm_link { /* type 0x01 */ |
40 | struct smc_llc_hdr hd; | |
41 | u8 sender_mac[ETH_ALEN]; | |
42 | u8 sender_gid[SMC_GID_SIZE]; | |
43 | u8 sender_qp_num[3]; | |
44 | u8 link_num; | |
45 | u8 link_uid[SMC_LGR_ID_SIZE]; | |
46 | u8 max_links; | |
47 | u8 reserved[9]; | |
48 | }; | |
49 | ||
52bedf37 KG |
50 | #define SMC_LLC_FLAG_ADD_LNK_REJ 0x40 |
51 | #define SMC_LLC_REJ_RSN_NO_ALT_PATH 1 | |
52 | ||
53 | #define SMC_LLC_ADD_LNK_MAX_LINKS 2 | |
54 | ||
55 | struct smc_llc_msg_add_link { /* type 0x02 */ | |
56 | struct smc_llc_hdr hd; | |
57 | u8 sender_mac[ETH_ALEN]; | |
58 | u8 reserved2[2]; | |
59 | u8 sender_gid[SMC_GID_SIZE]; | |
60 | u8 sender_qp_num[3]; | |
61 | u8 link_num; | |
fbed3b37 KG |
62 | #if defined(__BIG_ENDIAN_BITFIELD) |
63 | u8 reserved3 : 4, | |
64 | qp_mtu : 4; | |
65 | #elif defined(__LITTLE_ENDIAN_BITFIELD) | |
66 | u8 qp_mtu : 4, | |
67 | reserved3 : 4; | |
68 | #endif | |
52bedf37 KG |
69 | u8 initial_psn[3]; |
70 | u8 reserved[8]; | |
71 | }; | |
72 | ||
87f88cda KG |
73 | struct smc_llc_msg_add_link_cont_rt { |
74 | __be32 rmb_key; | |
75 | __be32 rmb_key_new; | |
76 | __be64 rmb_vaddr_new; | |
77 | }; | |
78 | ||
79 | #define SMC_LLC_RKEYS_PER_CONT_MSG 2 | |
80 | ||
81 | struct smc_llc_msg_add_link_cont { /* type 0x03 */ | |
82 | struct smc_llc_hdr hd; | |
83 | u8 link_num; | |
84 | u8 num_rkeys; | |
85 | u8 reserved2[2]; | |
86 | struct smc_llc_msg_add_link_cont_rt rt[SMC_LLC_RKEYS_PER_CONT_MSG]; | |
87 | u8 reserved[4]; | |
88 | } __packed; /* format defined in RFC7609 */ | |
89 | ||
52bedf37 KG |
90 | #define SMC_LLC_FLAG_DEL_LINK_ALL 0x40 |
91 | #define SMC_LLC_FLAG_DEL_LINK_ORDERLY 0x20 | |
92 | ||
93 | struct smc_llc_msg_del_link { /* type 0x04 */ | |
94 | struct smc_llc_hdr hd; | |
95 | u8 link_num; | |
96 | __be32 reason; | |
97 | u8 reserved[35]; | |
98 | } __packed; /* format defined in RFC7609 */ | |
99 | ||
313164da KG |
100 | struct smc_llc_msg_test_link { /* type 0x07 */ |
101 | struct smc_llc_hdr hd; | |
102 | u8 user_data[16]; | |
103 | u8 reserved[24]; | |
104 | }; | |
105 | ||
4ed75de5 KG |
106 | struct smc_rmb_rtoken { |
107 | union { | |
108 | u8 num_rkeys; /* first rtoken byte of CONFIRM LINK msg */ | |
109 | /* is actually the num of rtokens, first */ | |
110 | /* rtoken is always for the current link */ | |
111 | u8 link_id; /* link id of the rtoken */ | |
112 | }; | |
113 | __be32 rmb_key; | |
114 | __be64 rmb_vaddr; | |
115 | } __packed; /* format defined in RFC7609 */ | |
116 | ||
117 | #define SMC_LLC_RKEYS_PER_MSG 3 | |
118 | ||
119 | struct smc_llc_msg_confirm_rkey { /* type 0x06 */ | |
120 | struct smc_llc_hdr hd; | |
121 | struct smc_rmb_rtoken rtoken[SMC_LLC_RKEYS_PER_MSG]; | |
122 | u8 reserved; | |
123 | }; | |
124 | ||
4ed75de5 | 125 | #define SMC_LLC_DEL_RKEY_MAX 8 |
3bc67e09 | 126 | #define SMC_LLC_FLAG_RKEY_RETRY 0x10 |
4ed75de5 KG |
127 | #define SMC_LLC_FLAG_RKEY_NEG 0x20 |
128 | ||
129 | struct smc_llc_msg_delete_rkey { /* type 0x09 */ | |
130 | struct smc_llc_hdr hd; | |
131 | u8 num_rkeys; | |
132 | u8 err_mask; | |
133 | u8 reserved[2]; | |
134 | __be32 rkey[8]; | |
135 | u8 reserved2[4]; | |
136 | }; | |
137 | ||
0f627126 SR |
138 | union smc_llc_msg { |
139 | struct smc_llc_msg_confirm_link confirm_link; | |
52bedf37 | 140 | struct smc_llc_msg_add_link add_link; |
87f88cda | 141 | struct smc_llc_msg_add_link_cont add_link_cont; |
52bedf37 | 142 | struct smc_llc_msg_del_link delete_link; |
4ed75de5 KG |
143 | |
144 | struct smc_llc_msg_confirm_rkey confirm_rkey; | |
4ed75de5 KG |
145 | struct smc_llc_msg_delete_rkey delete_rkey; |
146 | ||
313164da | 147 | struct smc_llc_msg_test_link test_link; |
0f627126 SR |
148 | struct { |
149 | struct smc_llc_hdr hdr; | |
150 | u8 data[SMC_LLC_DATA_LEN]; | |
151 | } raw; | |
152 | }; | |
153 | ||
154 | #define SMC_LLC_FLAG_RESP 0x80 | |
155 | ||
6c8968c4 KG |
156 | struct smc_llc_qentry { |
157 | struct list_head list; | |
158 | struct smc_link *link; | |
159 | union smc_llc_msg msg; | |
160 | }; | |
161 | ||
4dadd151 KG |
162 | static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc); |
163 | ||
555da9af KG |
164 | struct smc_llc_qentry *smc_llc_flow_qentry_clr(struct smc_llc_flow *flow) |
165 | { | |
166 | struct smc_llc_qentry *qentry = flow->qentry; | |
167 | ||
168 | flow->qentry = NULL; | |
169 | return qentry; | |
170 | } | |
171 | ||
172 | void smc_llc_flow_qentry_del(struct smc_llc_flow *flow) | |
173 | { | |
174 | struct smc_llc_qentry *qentry; | |
175 | ||
176 | if (flow->qentry) { | |
177 | qentry = flow->qentry; | |
178 | flow->qentry = NULL; | |
179 | kfree(qentry); | |
180 | } | |
181 | } | |
182 | ||
183 | static inline void smc_llc_flow_qentry_set(struct smc_llc_flow *flow, | |
184 | struct smc_llc_qentry *qentry) | |
185 | { | |
186 | flow->qentry = qentry; | |
187 | } | |
188 | ||
6778a6be KG |
189 | static void smc_llc_flow_parallel(struct smc_link_group *lgr, u8 flow_type, |
190 | struct smc_llc_qentry *qentry) | |
191 | { | |
192 | u8 msg_type = qentry->msg.raw.hdr.common.type; | |
193 | ||
194 | if ((msg_type == SMC_LLC_ADD_LINK || msg_type == SMC_LLC_DELETE_LINK) && | |
195 | flow_type != msg_type && !lgr->delayed_event) { | |
196 | lgr->delayed_event = qentry; | |
197 | return; | |
198 | } | |
199 | /* drop parallel or already-in-progress llc requests */ | |
200 | if (flow_type != msg_type) | |
201 | pr_warn_once("smc: SMC-R lg %*phN dropped parallel " | |
202 | "LLC msg: msg %d flow %d role %d\n", | |
203 | SMC_LGR_ID_SIZE, &lgr->id, | |
204 | qentry->msg.raw.hdr.common.type, | |
205 | flow_type, lgr->role); | |
206 | kfree(qentry); | |
207 | } | |
208 | ||
555da9af KG |
209 | /* try to start a new llc flow, initiated by an incoming llc msg */ |
210 | static bool smc_llc_flow_start(struct smc_llc_flow *flow, | |
211 | struct smc_llc_qentry *qentry) | |
212 | { | |
213 | struct smc_link_group *lgr = qentry->link->lgr; | |
214 | ||
215 | spin_lock_bh(&lgr->llc_flow_lock); | |
216 | if (flow->type) { | |
217 | /* a flow is already active */ | |
6778a6be | 218 | smc_llc_flow_parallel(lgr, flow->type, qentry); |
555da9af KG |
219 | spin_unlock_bh(&lgr->llc_flow_lock); |
220 | return false; | |
221 | } | |
222 | switch (qentry->msg.raw.hdr.common.type) { | |
223 | case SMC_LLC_ADD_LINK: | |
224 | flow->type = SMC_LLC_FLOW_ADD_LINK; | |
225 | break; | |
226 | case SMC_LLC_DELETE_LINK: | |
227 | flow->type = SMC_LLC_FLOW_DEL_LINK; | |
228 | break; | |
229 | case SMC_LLC_CONFIRM_RKEY: | |
230 | case SMC_LLC_DELETE_RKEY: | |
231 | flow->type = SMC_LLC_FLOW_RKEY; | |
232 | break; | |
233 | default: | |
234 | flow->type = SMC_LLC_FLOW_NONE; | |
235 | } | |
236 | if (qentry == lgr->delayed_event) | |
237 | lgr->delayed_event = NULL; | |
555da9af | 238 | smc_llc_flow_qentry_set(flow, qentry); |
6778a6be | 239 | spin_unlock_bh(&lgr->llc_flow_lock); |
555da9af KG |
240 | return true; |
241 | } | |
242 | ||
243 | /* start a new local llc flow, wait till current flow finished */ | |
244 | int smc_llc_flow_initiate(struct smc_link_group *lgr, | |
245 | enum smc_llc_flowtype type) | |
246 | { | |
247 | enum smc_llc_flowtype allowed_remote = SMC_LLC_FLOW_NONE; | |
248 | int rc; | |
249 | ||
250 | /* all flows except confirm_rkey and delete_rkey are exclusive, | |
251 | * confirm/delete rkey flows can run concurrently (local and remote) | |
252 | */ | |
253 | if (type == SMC_LLC_FLOW_RKEY) | |
254 | allowed_remote = SMC_LLC_FLOW_RKEY; | |
255 | again: | |
256 | if (list_empty(&lgr->list)) | |
257 | return -ENODEV; | |
258 | spin_lock_bh(&lgr->llc_flow_lock); | |
259 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && | |
260 | (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || | |
261 | lgr->llc_flow_rmt.type == allowed_remote)) { | |
262 | lgr->llc_flow_lcl.type = type; | |
263 | spin_unlock_bh(&lgr->llc_flow_lock); | |
264 | return 0; | |
265 | } | |
266 | spin_unlock_bh(&lgr->llc_flow_lock); | |
6778a6be KG |
267 | rc = wait_event_timeout(lgr->llc_flow_waiter, (list_empty(&lgr->list) || |
268 | (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE && | |
269 | (lgr->llc_flow_rmt.type == SMC_LLC_FLOW_NONE || | |
270 | lgr->llc_flow_rmt.type == allowed_remote))), | |
271 | SMC_LLC_WAIT_TIME * 10); | |
555da9af KG |
272 | if (!rc) |
273 | return -ETIMEDOUT; | |
274 | goto again; | |
275 | } | |
276 | ||
277 | /* finish the current llc flow */ | |
278 | void smc_llc_flow_stop(struct smc_link_group *lgr, struct smc_llc_flow *flow) | |
279 | { | |
280 | spin_lock_bh(&lgr->llc_flow_lock); | |
281 | memset(flow, 0, sizeof(*flow)); | |
282 | flow->type = SMC_LLC_FLOW_NONE; | |
283 | spin_unlock_bh(&lgr->llc_flow_lock); | |
284 | if (!list_empty(&lgr->list) && lgr->delayed_event && | |
285 | flow == &lgr->llc_flow_lcl) | |
286 | schedule_work(&lgr->llc_event_work); | |
287 | else | |
6778a6be | 288 | wake_up(&lgr->llc_flow_waiter); |
555da9af KG |
289 | } |
290 | ||
291 | /* lnk is optional and used for early wakeup when link goes down, useful in | |
292 | * cases where we wait for a response on the link after we sent a request | |
293 | */ | |
294 | struct smc_llc_qentry *smc_llc_wait(struct smc_link_group *lgr, | |
295 | struct smc_link *lnk, | |
296 | int time_out, u8 exp_msg) | |
297 | { | |
298 | struct smc_llc_flow *flow = &lgr->llc_flow_lcl; | |
6778a6be | 299 | u8 rcv_msg; |
555da9af | 300 | |
6778a6be KG |
301 | wait_event_timeout(lgr->llc_msg_waiter, |
302 | (flow->qentry || | |
303 | (lnk && !smc_link_usable(lnk)) || | |
304 | list_empty(&lgr->list)), | |
305 | time_out); | |
555da9af KG |
306 | if (!flow->qentry || |
307 | (lnk && !smc_link_usable(lnk)) || list_empty(&lgr->list)) { | |
308 | smc_llc_flow_qentry_del(flow); | |
309 | goto out; | |
310 | } | |
6778a6be KG |
311 | rcv_msg = flow->qentry->msg.raw.hdr.common.type; |
312 | if (exp_msg && rcv_msg != exp_msg) { | |
555da9af | 313 | if (exp_msg == SMC_LLC_ADD_LINK && |
6778a6be | 314 | rcv_msg == SMC_LLC_DELETE_LINK) { |
555da9af KG |
315 | /* flow_start will delay the unexpected msg */ |
316 | smc_llc_flow_start(&lgr->llc_flow_lcl, | |
317 | smc_llc_flow_qentry_clr(flow)); | |
318 | return NULL; | |
319 | } | |
6778a6be KG |
320 | pr_warn_once("smc: SMC-R lg %*phN dropped unexpected LLC msg: " |
321 | "msg %d exp %d flow %d role %d flags %x\n", | |
322 | SMC_LGR_ID_SIZE, &lgr->id, rcv_msg, exp_msg, | |
323 | flow->type, lgr->role, | |
324 | flow->qentry->msg.raw.hdr.flags); | |
555da9af KG |
325 | smc_llc_flow_qentry_del(flow); |
326 | } | |
327 | out: | |
328 | return flow->qentry; | |
329 | } | |
330 | ||
9bf9abea UB |
331 | /********************************** send *************************************/ |
332 | ||
333 | struct smc_llc_tx_pend { | |
334 | }; | |
335 | ||
336 | /* handler for send/transmission completion of an LLC msg */ | |
337 | static void smc_llc_tx_handler(struct smc_wr_tx_pend_priv *pend, | |
338 | struct smc_link *link, | |
339 | enum ib_wc_status wc_status) | |
340 | { | |
341 | /* future work: handle wc_status error for recovery and failover */ | |
342 | } | |
343 | ||
344 | /** | |
345 | * smc_llc_add_pending_send() - add LLC control message to pending WQE transmits | |
346 | * @link: Pointer to SMC link used for sending LLC control message. | |
347 | * @wr_buf: Out variable returning pointer to work request payload buffer. | |
348 | * @pend: Out variable returning pointer to private pending WR tracking. | |
349 | * It's the context the transmit complete handler will get. | |
350 | * | |
351 | * Reserves and pre-fills an entry for a pending work request send/tx. | |
352 | * Used by mid-level smc_llc_send_msg() to prepare for later actual send/tx. | |
353 | * Can sleep due to smc_get_ctrl_buf (if not in softirq context). | |
354 | * | |
355 | * Return: 0 on success, otherwise an error value. | |
356 | */ | |
357 | static int smc_llc_add_pending_send(struct smc_link *link, | |
358 | struct smc_wr_buf **wr_buf, | |
359 | struct smc_wr_tx_pend_priv **pend) | |
360 | { | |
361 | int rc; | |
362 | ||
ad6f317f UB |
363 | rc = smc_wr_tx_get_free_slot(link, smc_llc_tx_handler, wr_buf, NULL, |
364 | pend); | |
9bf9abea UB |
365 | if (rc < 0) |
366 | return rc; | |
367 | BUILD_BUG_ON_MSG( | |
368 | sizeof(union smc_llc_msg) > SMC_WR_BUF_SIZE, | |
369 | "must increase SMC_WR_BUF_SIZE to at least sizeof(struct smc_llc_msg)"); | |
370 | BUILD_BUG_ON_MSG( | |
371 | sizeof(union smc_llc_msg) != SMC_WR_TX_SIZE, | |
372 | "must adapt SMC_WR_TX_SIZE to sizeof(struct smc_llc_msg); if not all smc_wr upper layer protocols use the same message size any more, must start to set link->wr_tx_sges[i].length on each individual smc_wr_tx_send()"); | |
373 | BUILD_BUG_ON_MSG( | |
374 | sizeof(struct smc_llc_tx_pend) > SMC_WR_TX_PEND_PRIV_SIZE, | |
375 | "must increase SMC_WR_TX_PEND_PRIV_SIZE to at least sizeof(struct smc_llc_tx_pend)"); | |
376 | return 0; | |
377 | } | |
378 | ||
379 | /* high-level API to send LLC confirm link */ | |
947541f3 | 380 | int smc_llc_send_confirm_link(struct smc_link *link, |
9bf9abea UB |
381 | enum smc_llc_reqresp reqresp) |
382 | { | |
9bf9abea UB |
383 | struct smc_llc_msg_confirm_link *confllc; |
384 | struct smc_wr_tx_pend_priv *pend; | |
385 | struct smc_wr_buf *wr_buf; | |
386 | int rc; | |
387 | ||
388 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
389 | if (rc) | |
390 | return rc; | |
391 | confllc = (struct smc_llc_msg_confirm_link *)wr_buf; | |
392 | memset(confllc, 0, sizeof(*confllc)); | |
393 | confllc->hd.common.type = SMC_LLC_CONFIRM_LINK; | |
394 | confllc->hd.length = sizeof(struct smc_llc_msg_confirm_link); | |
75d320d6 | 395 | confllc->hd.flags |= SMC_LLC_FLAG_NO_RMBE_EYEC; |
9bf9abea UB |
396 | if (reqresp == SMC_LLC_RESP) |
397 | confllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
947541f3 UB |
398 | memcpy(confllc->sender_mac, link->smcibdev->mac[link->ibport - 1], |
399 | ETH_ALEN); | |
7005ada6 | 400 | memcpy(confllc->sender_gid, link->gid, SMC_GID_SIZE); |
9bf9abea | 401 | hton24(confllc->sender_qp_num, link->roce_qp->qp_num); |
2be922f3 | 402 | confllc->link_num = link->link_id; |
45fa8da0 | 403 | memcpy(confllc->link_uid, link->link_uid, SMC_LGR_ID_SIZE); |
b1570a87 | 404 | confllc->max_links = SMC_LLC_ADD_LNK_MAX_LINKS; |
52bedf37 KG |
405 | /* send llc message */ |
406 | rc = smc_wr_tx_send(link, pend); | |
407 | return rc; | |
408 | } | |
409 | ||
44aa81ce | 410 | /* send LLC confirm rkey request */ |
3d88a21b | 411 | static int smc_llc_send_confirm_rkey(struct smc_link *send_link, |
44aa81ce KG |
412 | struct smc_buf_desc *rmb_desc) |
413 | { | |
414 | struct smc_llc_msg_confirm_rkey *rkeyllc; | |
415 | struct smc_wr_tx_pend_priv *pend; | |
416 | struct smc_wr_buf *wr_buf; | |
3d88a21b KG |
417 | struct smc_link *link; |
418 | int i, rc, rtok_ix; | |
44aa81ce | 419 | |
3d88a21b | 420 | rc = smc_llc_add_pending_send(send_link, &wr_buf, &pend); |
44aa81ce KG |
421 | if (rc) |
422 | return rc; | |
423 | rkeyllc = (struct smc_llc_msg_confirm_rkey *)wr_buf; | |
424 | memset(rkeyllc, 0, sizeof(*rkeyllc)); | |
425 | rkeyllc->hd.common.type = SMC_LLC_CONFIRM_RKEY; | |
426 | rkeyllc->hd.length = sizeof(struct smc_llc_msg_confirm_rkey); | |
3d88a21b KG |
427 | |
428 | rtok_ix = 1; | |
429 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
430 | link = &send_link->lgr->lnk[i]; | |
431 | if (link->state == SMC_LNK_ACTIVE && link != send_link) { | |
432 | rkeyllc->rtoken[rtok_ix].link_id = link->link_id; | |
433 | rkeyllc->rtoken[rtok_ix].rmb_key = | |
434 | htonl(rmb_desc->mr_rx[link->link_idx]->rkey); | |
435 | rkeyllc->rtoken[rtok_ix].rmb_vaddr = cpu_to_be64( | |
436 | (u64)sg_dma_address( | |
437 | rmb_desc->sgt[link->link_idx].sgl)); | |
438 | rtok_ix++; | |
439 | } | |
440 | } | |
441 | /* rkey of send_link is in rtoken[0] */ | |
442 | rkeyllc->rtoken[0].num_rkeys = rtok_ix - 1; | |
44aa81ce | 443 | rkeyllc->rtoken[0].rmb_key = |
3d88a21b | 444 | htonl(rmb_desc->mr_rx[send_link->link_idx]->rkey); |
44aa81ce | 445 | rkeyllc->rtoken[0].rmb_vaddr = cpu_to_be64( |
3d88a21b | 446 | (u64)sg_dma_address(rmb_desc->sgt[send_link->link_idx].sgl)); |
44aa81ce | 447 | /* send llc message */ |
3d88a21b | 448 | rc = smc_wr_tx_send(send_link, pend); |
44aa81ce KG |
449 | return rc; |
450 | } | |
451 | ||
60e03c62 KG |
452 | /* send LLC delete rkey request */ |
453 | static int smc_llc_send_delete_rkey(struct smc_link *link, | |
454 | struct smc_buf_desc *rmb_desc) | |
455 | { | |
456 | struct smc_llc_msg_delete_rkey *rkeyllc; | |
457 | struct smc_wr_tx_pend_priv *pend; | |
458 | struct smc_wr_buf *wr_buf; | |
459 | int rc; | |
460 | ||
461 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
462 | if (rc) | |
463 | return rc; | |
464 | rkeyllc = (struct smc_llc_msg_delete_rkey *)wr_buf; | |
465 | memset(rkeyllc, 0, sizeof(*rkeyllc)); | |
466 | rkeyllc->hd.common.type = SMC_LLC_DELETE_RKEY; | |
467 | rkeyllc->hd.length = sizeof(struct smc_llc_msg_delete_rkey); | |
468 | rkeyllc->num_rkeys = 1; | |
387707fd | 469 | rkeyllc->rkey[0] = htonl(rmb_desc->mr_rx[link->link_idx]->rkey); |
60e03c62 KG |
470 | /* send llc message */ |
471 | rc = smc_wr_tx_send(link, pend); | |
472 | return rc; | |
473 | } | |
474 | ||
52bedf37 | 475 | /* send ADD LINK request or response */ |
7005ada6 | 476 | int smc_llc_send_add_link(struct smc_link *link, u8 mac[], u8 gid[], |
fbed3b37 | 477 | struct smc_link *link_new, |
52bedf37 KG |
478 | enum smc_llc_reqresp reqresp) |
479 | { | |
480 | struct smc_llc_msg_add_link *addllc; | |
481 | struct smc_wr_tx_pend_priv *pend; | |
482 | struct smc_wr_buf *wr_buf; | |
483 | int rc; | |
484 | ||
485 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
486 | if (rc) | |
487 | return rc; | |
488 | addllc = (struct smc_llc_msg_add_link *)wr_buf; | |
fbed3b37 KG |
489 | |
490 | memset(addllc, 0, sizeof(*addllc)); | |
491 | addllc->hd.common.type = SMC_LLC_ADD_LINK; | |
492 | addllc->hd.length = sizeof(struct smc_llc_msg_add_link); | |
493 | if (reqresp == SMC_LLC_RESP) | |
494 | addllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
495 | memcpy(addllc->sender_mac, mac, ETH_ALEN); | |
496 | memcpy(addllc->sender_gid, gid, SMC_GID_SIZE); | |
497 | if (link_new) { | |
498 | addllc->link_num = link_new->link_id; | |
499 | hton24(addllc->sender_qp_num, link_new->roce_qp->qp_num); | |
500 | hton24(addllc->initial_psn, link_new->psn_initial); | |
501 | if (reqresp == SMC_LLC_REQ) | |
502 | addllc->qp_mtu = link_new->path_mtu; | |
503 | else | |
504 | addllc->qp_mtu = min(link_new->path_mtu, | |
505 | link_new->peer_mtu); | |
506 | } | |
52bedf37 KG |
507 | /* send llc message */ |
508 | rc = smc_wr_tx_send(link, pend); | |
509 | return rc; | |
510 | } | |
511 | ||
512 | /* send DELETE LINK request or response */ | |
fbed3b37 KG |
513 | int smc_llc_send_delete_link(struct smc_link *link, u8 link_del_id, |
514 | enum smc_llc_reqresp reqresp, bool orderly, | |
515 | u32 reason) | |
52bedf37 KG |
516 | { |
517 | struct smc_llc_msg_del_link *delllc; | |
518 | struct smc_wr_tx_pend_priv *pend; | |
519 | struct smc_wr_buf *wr_buf; | |
520 | int rc; | |
521 | ||
522 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
523 | if (rc) | |
524 | return rc; | |
525 | delllc = (struct smc_llc_msg_del_link *)wr_buf; | |
fbed3b37 KG |
526 | |
527 | memset(delllc, 0, sizeof(*delllc)); | |
528 | delllc->hd.common.type = SMC_LLC_DELETE_LINK; | |
529 | delllc->hd.length = sizeof(struct smc_llc_msg_del_link); | |
530 | if (reqresp == SMC_LLC_RESP) | |
531 | delllc->hd.flags |= SMC_LLC_FLAG_RESP; | |
532 | if (orderly) | |
533 | delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; | |
534 | if (link_del_id) | |
535 | delllc->link_num = link_del_id; | |
536 | else | |
537 | delllc->hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; | |
538 | delllc->reason = htonl(reason); | |
9bf9abea UB |
539 | /* send llc message */ |
540 | rc = smc_wr_tx_send(link, pend); | |
541 | return rc; | |
542 | } | |
543 | ||
d97935fa KG |
544 | /* send LLC test link request */ |
545 | static int smc_llc_send_test_link(struct smc_link *link, u8 user_data[16]) | |
313164da KG |
546 | { |
547 | struct smc_llc_msg_test_link *testllc; | |
548 | struct smc_wr_tx_pend_priv *pend; | |
549 | struct smc_wr_buf *wr_buf; | |
550 | int rc; | |
551 | ||
552 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
553 | if (rc) | |
554 | return rc; | |
555 | testllc = (struct smc_llc_msg_test_link *)wr_buf; | |
556 | memset(testllc, 0, sizeof(*testllc)); | |
557 | testllc->hd.common.type = SMC_LLC_TEST_LINK; | |
558 | testllc->hd.length = sizeof(struct smc_llc_msg_test_link); | |
313164da KG |
559 | memcpy(testllc->user_data, user_data, sizeof(testllc->user_data)); |
560 | /* send llc message */ | |
561 | rc = smc_wr_tx_send(link, pend); | |
562 | return rc; | |
563 | } | |
564 | ||
6c8968c4 KG |
565 | /* schedule an llc send on link, may wait for buffers */ |
566 | static int smc_llc_send_message(struct smc_link *link, void *llcbuf) | |
4ed75de5 KG |
567 | { |
568 | struct smc_wr_tx_pend_priv *pend; | |
569 | struct smc_wr_buf *wr_buf; | |
570 | int rc; | |
571 | ||
6c8968c4 KG |
572 | if (!smc_link_usable(link)) |
573 | return -ENOLINK; | |
574 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
4ed75de5 | 575 | if (rc) |
6c8968c4 KG |
576 | return rc; |
577 | memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); | |
578 | return smc_wr_tx_send(link, pend); | |
4ed75de5 KG |
579 | } |
580 | ||
f3811fd7 KG |
581 | /* schedule an llc send on link, may wait for buffers, |
582 | * and wait for send completion notification. | |
583 | * @return 0 on success | |
584 | */ | |
585 | static int smc_llc_send_message_wait(struct smc_link *link, void *llcbuf) | |
586 | { | |
587 | struct smc_wr_tx_pend_priv *pend; | |
588 | struct smc_wr_buf *wr_buf; | |
589 | int rc; | |
590 | ||
591 | if (!smc_link_usable(link)) | |
592 | return -ENOLINK; | |
593 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
594 | if (rc) | |
595 | return rc; | |
596 | memcpy(wr_buf, llcbuf, sizeof(union smc_llc_msg)); | |
597 | return smc_wr_tx_send_wait(link, pend, SMC_LLC_WAIT_TIME); | |
598 | } | |
599 | ||
9bf9abea UB |
600 | /********************************* receive ***********************************/ |
601 | ||
336ba09f KG |
602 | static int smc_llc_alloc_alt_link(struct smc_link_group *lgr, |
603 | enum smc_lgr_type lgr_new_t) | |
604 | { | |
605 | int i; | |
606 | ||
607 | if (lgr->type == SMC_LGR_SYMMETRIC || | |
608 | (lgr->type != SMC_LGR_SINGLE && | |
609 | (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || | |
610 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER))) | |
611 | return -EMLINK; | |
612 | ||
613 | if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || | |
614 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) { | |
615 | for (i = SMC_LINKS_PER_LGR_MAX - 1; i >= 0; i--) | |
616 | if (lgr->lnk[i].state == SMC_LNK_UNUSED) | |
617 | return i; | |
618 | } else { | |
619 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) | |
620 | if (lgr->lnk[i].state == SMC_LNK_UNUSED) | |
621 | return i; | |
622 | } | |
623 | return -EMLINK; | |
624 | } | |
625 | ||
87f88cda KG |
626 | /* return first buffer from any of the next buf lists */ |
627 | static struct smc_buf_desc *_smc_llc_get_next_rmb(struct smc_link_group *lgr, | |
628 | int *buf_lst) | |
629 | { | |
630 | struct smc_buf_desc *buf_pos; | |
631 | ||
632 | while (*buf_lst < SMC_RMBE_SIZES) { | |
633 | buf_pos = list_first_entry_or_null(&lgr->rmbs[*buf_lst], | |
634 | struct smc_buf_desc, list); | |
635 | if (buf_pos) | |
636 | return buf_pos; | |
637 | (*buf_lst)++; | |
638 | } | |
639 | return NULL; | |
640 | } | |
641 | ||
642 | /* return next rmb from buffer lists */ | |
643 | static struct smc_buf_desc *smc_llc_get_next_rmb(struct smc_link_group *lgr, | |
644 | int *buf_lst, | |
645 | struct smc_buf_desc *buf_pos) | |
646 | { | |
647 | struct smc_buf_desc *buf_next; | |
648 | ||
649 | if (!buf_pos || list_is_last(&buf_pos->list, &lgr->rmbs[*buf_lst])) { | |
650 | (*buf_lst)++; | |
651 | return _smc_llc_get_next_rmb(lgr, buf_lst); | |
652 | } | |
653 | buf_next = list_next_entry(buf_pos, list); | |
654 | return buf_next; | |
655 | } | |
656 | ||
657 | static struct smc_buf_desc *smc_llc_get_first_rmb(struct smc_link_group *lgr, | |
658 | int *buf_lst) | |
659 | { | |
660 | *buf_lst = 0; | |
661 | return smc_llc_get_next_rmb(lgr, buf_lst, NULL); | |
662 | } | |
663 | ||
664 | /* send one add_link_continue msg */ | |
665 | static int smc_llc_add_link_cont(struct smc_link *link, | |
666 | struct smc_link *link_new, u8 *num_rkeys_todo, | |
667 | int *buf_lst, struct smc_buf_desc **buf_pos) | |
668 | { | |
669 | struct smc_llc_msg_add_link_cont *addc_llc; | |
670 | struct smc_link_group *lgr = link->lgr; | |
671 | int prim_lnk_idx, lnk_idx, i, rc; | |
672 | struct smc_wr_tx_pend_priv *pend; | |
673 | struct smc_wr_buf *wr_buf; | |
674 | struct smc_buf_desc *rmb; | |
675 | u8 n; | |
676 | ||
677 | rc = smc_llc_add_pending_send(link, &wr_buf, &pend); | |
678 | if (rc) | |
679 | return rc; | |
680 | addc_llc = (struct smc_llc_msg_add_link_cont *)wr_buf; | |
681 | memset(addc_llc, 0, sizeof(*addc_llc)); | |
682 | ||
683 | prim_lnk_idx = link->link_idx; | |
684 | lnk_idx = link_new->link_idx; | |
685 | addc_llc->link_num = link_new->link_id; | |
686 | addc_llc->num_rkeys = *num_rkeys_todo; | |
687 | n = *num_rkeys_todo; | |
688 | for (i = 0; i < min_t(u8, n, SMC_LLC_RKEYS_PER_CONT_MSG); i++) { | |
689 | if (!*buf_pos) { | |
690 | addc_llc->num_rkeys = addc_llc->num_rkeys - | |
691 | *num_rkeys_todo; | |
692 | *num_rkeys_todo = 0; | |
693 | break; | |
694 | } | |
695 | rmb = *buf_pos; | |
696 | ||
697 | addc_llc->rt[i].rmb_key = htonl(rmb->mr_rx[prim_lnk_idx]->rkey); | |
698 | addc_llc->rt[i].rmb_key_new = htonl(rmb->mr_rx[lnk_idx]->rkey); | |
699 | addc_llc->rt[i].rmb_vaddr_new = | |
700 | cpu_to_be64((u64)sg_dma_address(rmb->sgt[lnk_idx].sgl)); | |
701 | ||
702 | (*num_rkeys_todo)--; | |
703 | *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); | |
704 | while (*buf_pos && !(*buf_pos)->used) | |
705 | *buf_pos = smc_llc_get_next_rmb(lgr, buf_lst, *buf_pos); | |
706 | } | |
707 | addc_llc->hd.common.type = SMC_LLC_ADD_LINK_CONT; | |
708 | addc_llc->hd.length = sizeof(struct smc_llc_msg_add_link_cont); | |
709 | if (lgr->role == SMC_CLNT) | |
710 | addc_llc->hd.flags |= SMC_LLC_FLAG_RESP; | |
711 | return smc_wr_tx_send(link, pend); | |
712 | } | |
713 | ||
714 | static int smc_llc_cli_rkey_exchange(struct smc_link *link, | |
715 | struct smc_link *link_new) | |
716 | { | |
717 | struct smc_llc_msg_add_link_cont *addc_llc; | |
718 | struct smc_link_group *lgr = link->lgr; | |
719 | u8 max, num_rkeys_send, num_rkeys_recv; | |
720 | struct smc_llc_qentry *qentry; | |
721 | struct smc_buf_desc *buf_pos; | |
722 | int buf_lst; | |
723 | int rc = 0; | |
724 | int i; | |
725 | ||
726 | mutex_lock(&lgr->rmbs_lock); | |
727 | num_rkeys_send = lgr->conns_num; | |
728 | buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); | |
729 | do { | |
730 | qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_TIME, | |
731 | SMC_LLC_ADD_LINK_CONT); | |
732 | if (!qentry) { | |
733 | rc = -ETIMEDOUT; | |
734 | break; | |
735 | } | |
736 | addc_llc = &qentry->msg.add_link_cont; | |
737 | num_rkeys_recv = addc_llc->num_rkeys; | |
738 | max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); | |
739 | for (i = 0; i < max; i++) { | |
740 | smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, | |
741 | addc_llc->rt[i].rmb_key, | |
742 | addc_llc->rt[i].rmb_vaddr_new, | |
743 | addc_llc->rt[i].rmb_key_new); | |
744 | num_rkeys_recv--; | |
745 | } | |
746 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
747 | rc = smc_llc_add_link_cont(link, link_new, &num_rkeys_send, | |
748 | &buf_lst, &buf_pos); | |
749 | if (rc) | |
750 | break; | |
751 | } while (num_rkeys_send || num_rkeys_recv); | |
752 | ||
753 | mutex_unlock(&lgr->rmbs_lock); | |
754 | return rc; | |
755 | } | |
756 | ||
336ba09f KG |
757 | /* prepare and send an add link reject response */ |
758 | static int smc_llc_cli_add_link_reject(struct smc_llc_qentry *qentry) | |
759 | { | |
760 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; | |
761 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_ADD_LNK_REJ; | |
762 | qentry->msg.raw.hdr.add_link_rej_rsn = SMC_LLC_REJ_RSN_NO_ALT_PATH; | |
763 | return smc_llc_send_message(qentry->link, &qentry->msg); | |
764 | } | |
765 | ||
b1570a87 KG |
766 | static int smc_llc_cli_conf_link(struct smc_link *link, |
767 | struct smc_init_info *ini, | |
768 | struct smc_link *link_new, | |
769 | enum smc_lgr_type lgr_new_t) | |
770 | { | |
771 | struct smc_link_group *lgr = link->lgr; | |
b1570a87 KG |
772 | struct smc_llc_qentry *qentry = NULL; |
773 | int rc = 0; | |
774 | ||
775 | /* receive CONFIRM LINK request over RoCE fabric */ | |
776 | qentry = smc_llc_wait(lgr, NULL, SMC_LLC_WAIT_FIRST_TIME, 0); | |
777 | if (!qentry) { | |
778 | rc = smc_llc_send_delete_link(link, link_new->link_id, | |
779 | SMC_LLC_REQ, false, | |
780 | SMC_LLC_DEL_LOST_PATH); | |
781 | return -ENOLINK; | |
782 | } | |
783 | if (qentry->msg.raw.hdr.common.type != SMC_LLC_CONFIRM_LINK) { | |
784 | /* received DELETE_LINK instead */ | |
b1570a87 KG |
785 | qentry->msg.raw.hdr.flags |= SMC_LLC_FLAG_RESP; |
786 | smc_llc_send_message(link, &qentry->msg); | |
787 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
788 | return -ENOLINK; | |
789 | } | |
649758ff | 790 | smc_llc_save_peer_uid(qentry); |
b1570a87 KG |
791 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); |
792 | ||
793 | rc = smc_ib_modify_qp_rts(link_new); | |
794 | if (rc) { | |
795 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
796 | false, SMC_LLC_DEL_LOST_PATH); | |
797 | return -ENOLINK; | |
798 | } | |
799 | smc_wr_remember_qp_attr(link_new); | |
800 | ||
801 | rc = smcr_buf_reg_lgr(link_new); | |
802 | if (rc) { | |
803 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
804 | false, SMC_LLC_DEL_LOST_PATH); | |
805 | return -ENOLINK; | |
806 | } | |
807 | ||
808 | /* send CONFIRM LINK response over RoCE fabric */ | |
809 | rc = smc_llc_send_confirm_link(link_new, SMC_LLC_RESP); | |
810 | if (rc) { | |
811 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
812 | false, SMC_LLC_DEL_LOST_PATH); | |
813 | return -ENOLINK; | |
814 | } | |
815 | smc_llc_link_active(link_new); | |
ad6c111b KG |
816 | if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || |
817 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) | |
818 | smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); | |
819 | else | |
820 | smcr_lgr_set_type(lgr, lgr_new_t); | |
b1570a87 KG |
821 | return 0; |
822 | } | |
823 | ||
336ba09f KG |
824 | static void smc_llc_save_add_link_info(struct smc_link *link, |
825 | struct smc_llc_msg_add_link *add_llc) | |
826 | { | |
827 | link->peer_qpn = ntoh24(add_llc->sender_qp_num); | |
828 | memcpy(link->peer_gid, add_llc->sender_gid, SMC_GID_SIZE); | |
829 | memcpy(link->peer_mac, add_llc->sender_mac, ETH_ALEN); | |
830 | link->peer_psn = ntoh24(add_llc->initial_psn); | |
831 | link->peer_mtu = add_llc->qp_mtu; | |
832 | } | |
833 | ||
834 | /* as an SMC client, process an add link request */ | |
835 | int smc_llc_cli_add_link(struct smc_link *link, struct smc_llc_qentry *qentry) | |
836 | { | |
837 | struct smc_llc_msg_add_link *llc = &qentry->msg.add_link; | |
838 | enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; | |
839 | struct smc_link_group *lgr = smc_get_lgr(link); | |
840 | struct smc_link *lnk_new = NULL; | |
841 | struct smc_init_info ini; | |
842 | int lnk_idx, rc = 0; | |
843 | ||
844 | ini.vlan_id = lgr->vlan_id; | |
845 | smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); | |
846 | if (!memcmp(llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && | |
847 | !memcmp(llc->sender_mac, link->peer_mac, ETH_ALEN)) { | |
848 | if (!ini.ib_dev) | |
849 | goto out_reject; | |
850 | lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; | |
851 | } | |
852 | if (!ini.ib_dev) { | |
853 | lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; | |
854 | ini.ib_dev = link->smcibdev; | |
855 | ini.ib_port = link->ibport; | |
856 | } | |
857 | lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); | |
858 | if (lnk_idx < 0) | |
859 | goto out_reject; | |
860 | lnk_new = &lgr->lnk[lnk_idx]; | |
861 | rc = smcr_link_init(lgr, lnk_new, lnk_idx, &ini); | |
862 | if (rc) | |
863 | goto out_reject; | |
864 | smc_llc_save_add_link_info(lnk_new, llc); | |
45fa8da0 KG |
865 | lnk_new->link_id = llc->link_num; /* SMC server assigns link id */ |
866 | smc_llc_link_set_uid(lnk_new); | |
336ba09f KG |
867 | |
868 | rc = smc_ib_ready_link(lnk_new); | |
869 | if (rc) | |
870 | goto out_clear_lnk; | |
871 | ||
872 | rc = smcr_buf_map_lgr(lnk_new); | |
873 | if (rc) | |
874 | goto out_clear_lnk; | |
875 | ||
876 | rc = smc_llc_send_add_link(link, | |
877 | lnk_new->smcibdev->mac[ini.ib_port - 1], | |
878 | lnk_new->gid, lnk_new, SMC_LLC_RESP); | |
879 | if (rc) | |
880 | goto out_clear_lnk; | |
87f88cda | 881 | rc = smc_llc_cli_rkey_exchange(link, lnk_new); |
336ba09f KG |
882 | if (rc) { |
883 | rc = 0; | |
884 | goto out_clear_lnk; | |
885 | } | |
b1570a87 | 886 | rc = smc_llc_cli_conf_link(link, &ini, lnk_new, lgr_new_t); |
336ba09f KG |
887 | if (!rc) |
888 | goto out; | |
889 | out_clear_lnk: | |
0a99be43 | 890 | smcr_link_clear(lnk_new, false); |
336ba09f KG |
891 | out_reject: |
892 | smc_llc_cli_add_link_reject(qentry); | |
893 | out: | |
894 | kfree(qentry); | |
895 | return rc; | |
896 | } | |
897 | ||
b1570a87 KG |
898 | static void smc_llc_process_cli_add_link(struct smc_link_group *lgr) |
899 | { | |
900 | struct smc_llc_qentry *qentry; | |
901 | ||
902 | qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); | |
903 | ||
904 | mutex_lock(&lgr->llc_conf_mutex); | |
905 | smc_llc_cli_add_link(qentry->link, qentry); | |
906 | mutex_unlock(&lgr->llc_conf_mutex); | |
907 | } | |
908 | ||
9c416878 KG |
909 | static int smc_llc_active_link_count(struct smc_link_group *lgr) |
910 | { | |
911 | int i, link_count = 0; | |
912 | ||
913 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
914 | if (!smc_link_usable(&lgr->lnk[i])) | |
915 | continue; | |
916 | link_count++; | |
917 | } | |
918 | return link_count; | |
919 | } | |
920 | ||
c9a5d243 KG |
921 | /* find the asymmetric link when 3 links are established */ |
922 | static struct smc_link *smc_llc_find_asym_link(struct smc_link_group *lgr) | |
923 | { | |
924 | int asym_idx = -ENOENT; | |
925 | int i, j, k; | |
926 | bool found; | |
927 | ||
928 | /* determine asymmetric link */ | |
929 | found = false; | |
930 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
931 | for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { | |
932 | if (!smc_link_usable(&lgr->lnk[i]) || | |
933 | !smc_link_usable(&lgr->lnk[j])) | |
934 | continue; | |
935 | if (!memcmp(lgr->lnk[i].gid, lgr->lnk[j].gid, | |
936 | SMC_GID_SIZE)) { | |
937 | found = true; /* asym_lnk is i or j */ | |
938 | break; | |
939 | } | |
940 | } | |
941 | if (found) | |
942 | break; | |
943 | } | |
944 | if (!found) | |
945 | goto out; /* no asymmetric link */ | |
946 | for (k = 0; k < SMC_LINKS_PER_LGR_MAX; k++) { | |
947 | if (!smc_link_usable(&lgr->lnk[k])) | |
948 | continue; | |
949 | if (k != i && | |
950 | !memcmp(lgr->lnk[i].peer_gid, lgr->lnk[k].peer_gid, | |
951 | SMC_GID_SIZE)) { | |
952 | asym_idx = i; | |
953 | break; | |
954 | } | |
955 | if (k != j && | |
956 | !memcmp(lgr->lnk[j].peer_gid, lgr->lnk[k].peer_gid, | |
957 | SMC_GID_SIZE)) { | |
958 | asym_idx = j; | |
959 | break; | |
960 | } | |
961 | } | |
962 | out: | |
963 | return (asym_idx < 0) ? NULL : &lgr->lnk[asym_idx]; | |
964 | } | |
965 | ||
966 | static void smc_llc_delete_asym_link(struct smc_link_group *lgr) | |
967 | { | |
968 | struct smc_link *lnk_new = NULL, *lnk_asym; | |
969 | struct smc_llc_qentry *qentry; | |
970 | int rc; | |
971 | ||
972 | lnk_asym = smc_llc_find_asym_link(lgr); | |
973 | if (!lnk_asym) | |
974 | return; /* no asymmetric link */ | |
975 | if (!smc_link_downing(&lnk_asym->state)) | |
976 | return; | |
c6f02ebe | 977 | lnk_new = smc_switch_conns(lgr, lnk_asym, false); |
c9a5d243 KG |
978 | smc_wr_tx_wait_no_pending_sends(lnk_asym); |
979 | if (!lnk_new) | |
980 | goto out_free; | |
981 | /* change flow type from ADD_LINK into DEL_LINK */ | |
982 | lgr->llc_flow_lcl.type = SMC_LLC_FLOW_DEL_LINK; | |
983 | rc = smc_llc_send_delete_link(lnk_new, lnk_asym->link_id, SMC_LLC_REQ, | |
984 | true, SMC_LLC_DEL_NO_ASYM_NEEDED); | |
985 | if (rc) { | |
986 | smcr_link_down_cond(lnk_new); | |
987 | goto out_free; | |
988 | } | |
989 | qentry = smc_llc_wait(lgr, lnk_new, SMC_LLC_WAIT_TIME, | |
990 | SMC_LLC_DELETE_LINK); | |
991 | if (!qentry) { | |
992 | smcr_link_down_cond(lnk_new); | |
993 | goto out_free; | |
994 | } | |
995 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
996 | out_free: | |
0a99be43 | 997 | smcr_link_clear(lnk_asym, true); |
c9a5d243 KG |
998 | } |
999 | ||
57b49924 KG |
1000 | static int smc_llc_srv_rkey_exchange(struct smc_link *link, |
1001 | struct smc_link *link_new) | |
1002 | { | |
1003 | struct smc_llc_msg_add_link_cont *addc_llc; | |
1004 | struct smc_link_group *lgr = link->lgr; | |
1005 | u8 max, num_rkeys_send, num_rkeys_recv; | |
1006 | struct smc_llc_qentry *qentry = NULL; | |
1007 | struct smc_buf_desc *buf_pos; | |
1008 | int buf_lst; | |
1009 | int rc = 0; | |
1010 | int i; | |
1011 | ||
1012 | mutex_lock(&lgr->rmbs_lock); | |
1013 | num_rkeys_send = lgr->conns_num; | |
1014 | buf_pos = smc_llc_get_first_rmb(lgr, &buf_lst); | |
1015 | do { | |
1016 | smc_llc_add_link_cont(link, link_new, &num_rkeys_send, | |
1017 | &buf_lst, &buf_pos); | |
1018 | qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, | |
1019 | SMC_LLC_ADD_LINK_CONT); | |
1020 | if (!qentry) { | |
1021 | rc = -ETIMEDOUT; | |
1022 | goto out; | |
1023 | } | |
1024 | addc_llc = &qentry->msg.add_link_cont; | |
1025 | num_rkeys_recv = addc_llc->num_rkeys; | |
1026 | max = min_t(u8, num_rkeys_recv, SMC_LLC_RKEYS_PER_CONT_MSG); | |
1027 | for (i = 0; i < max; i++) { | |
1028 | smc_rtoken_set(lgr, link->link_idx, link_new->link_idx, | |
1029 | addc_llc->rt[i].rmb_key, | |
1030 | addc_llc->rt[i].rmb_vaddr_new, | |
1031 | addc_llc->rt[i].rmb_key_new); | |
1032 | num_rkeys_recv--; | |
1033 | } | |
1034 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1035 | } while (num_rkeys_send || num_rkeys_recv); | |
1036 | out: | |
1037 | mutex_unlock(&lgr->rmbs_lock); | |
1038 | return rc; | |
1039 | } | |
1040 | ||
1551c95b KG |
1041 | static int smc_llc_srv_conf_link(struct smc_link *link, |
1042 | struct smc_link *link_new, | |
1043 | enum smc_lgr_type lgr_new_t) | |
1044 | { | |
1045 | struct smc_link_group *lgr = link->lgr; | |
1046 | struct smc_llc_qentry *qentry = NULL; | |
1047 | int rc; | |
1048 | ||
1049 | /* send CONFIRM LINK request over the RoCE fabric */ | |
1050 | rc = smc_llc_send_confirm_link(link_new, SMC_LLC_REQ); | |
1051 | if (rc) | |
1052 | return -ENOLINK; | |
1053 | /* receive CONFIRM LINK response over the RoCE fabric */ | |
1054 | qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_FIRST_TIME, | |
1055 | SMC_LLC_CONFIRM_LINK); | |
1056 | if (!qentry) { | |
1057 | /* send DELETE LINK */ | |
1058 | smc_llc_send_delete_link(link, link_new->link_id, SMC_LLC_REQ, | |
1059 | false, SMC_LLC_DEL_LOST_PATH); | |
1060 | return -ENOLINK; | |
1061 | } | |
649758ff | 1062 | smc_llc_save_peer_uid(qentry); |
1551c95b | 1063 | smc_llc_link_active(link_new); |
ad6c111b KG |
1064 | if (lgr_new_t == SMC_LGR_ASYMMETRIC_LOCAL || |
1065 | lgr_new_t == SMC_LGR_ASYMMETRIC_PEER) | |
1066 | smcr_lgr_set_type_asym(lgr, lgr_new_t, link_new->link_idx); | |
1067 | else | |
1068 | smcr_lgr_set_type(lgr, lgr_new_t); | |
1551c95b KG |
1069 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); |
1070 | return 0; | |
1071 | } | |
1072 | ||
2d2209f2 KG |
1073 | int smc_llc_srv_add_link(struct smc_link *link) |
1074 | { | |
1075 | enum smc_lgr_type lgr_new_t = SMC_LGR_SYMMETRIC; | |
1076 | struct smc_link_group *lgr = link->lgr; | |
1077 | struct smc_llc_msg_add_link *add_llc; | |
1078 | struct smc_llc_qentry *qentry = NULL; | |
1079 | struct smc_link *link_new; | |
1080 | struct smc_init_info ini; | |
1081 | int lnk_idx, rc = 0; | |
1082 | ||
1083 | /* ignore client add link recommendation, start new flow */ | |
1084 | ini.vlan_id = lgr->vlan_id; | |
1085 | smc_pnet_find_alt_roce(lgr, &ini, link->smcibdev); | |
1086 | if (!ini.ib_dev) { | |
1087 | lgr_new_t = SMC_LGR_ASYMMETRIC_LOCAL; | |
1088 | ini.ib_dev = link->smcibdev; | |
1089 | ini.ib_port = link->ibport; | |
1090 | } | |
1091 | lnk_idx = smc_llc_alloc_alt_link(lgr, lgr_new_t); | |
1092 | if (lnk_idx < 0) | |
1093 | return 0; | |
1094 | ||
1095 | rc = smcr_link_init(lgr, &lgr->lnk[lnk_idx], lnk_idx, &ini); | |
1096 | if (rc) | |
1097 | return rc; | |
1098 | link_new = &lgr->lnk[lnk_idx]; | |
1099 | rc = smc_llc_send_add_link(link, | |
1100 | link_new->smcibdev->mac[ini.ib_port - 1], | |
1101 | link_new->gid, link_new, SMC_LLC_REQ); | |
1102 | if (rc) | |
1103 | goto out_err; | |
1104 | /* receive ADD LINK response over the RoCE fabric */ | |
1105 | qentry = smc_llc_wait(lgr, link, SMC_LLC_WAIT_TIME, SMC_LLC_ADD_LINK); | |
1106 | if (!qentry) { | |
1107 | rc = -ETIMEDOUT; | |
1108 | goto out_err; | |
1109 | } | |
1110 | add_llc = &qentry->msg.add_link; | |
1111 | if (add_llc->hd.flags & SMC_LLC_FLAG_ADD_LNK_REJ) { | |
1112 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1113 | rc = -ENOLINK; | |
1114 | goto out_err; | |
1115 | } | |
1116 | if (lgr->type == SMC_LGR_SINGLE && | |
1117 | (!memcmp(add_llc->sender_gid, link->peer_gid, SMC_GID_SIZE) && | |
1118 | !memcmp(add_llc->sender_mac, link->peer_mac, ETH_ALEN))) { | |
1119 | lgr_new_t = SMC_LGR_ASYMMETRIC_PEER; | |
1120 | } | |
1121 | smc_llc_save_add_link_info(link_new, add_llc); | |
1122 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1123 | ||
1124 | rc = smc_ib_ready_link(link_new); | |
1125 | if (rc) | |
1126 | goto out_err; | |
1127 | rc = smcr_buf_map_lgr(link_new); | |
1128 | if (rc) | |
1129 | goto out_err; | |
1130 | rc = smcr_buf_reg_lgr(link_new); | |
1131 | if (rc) | |
1132 | goto out_err; | |
57b49924 | 1133 | rc = smc_llc_srv_rkey_exchange(link, link_new); |
2d2209f2 KG |
1134 | if (rc) |
1135 | goto out_err; | |
1551c95b | 1136 | rc = smc_llc_srv_conf_link(link, link_new, lgr_new_t); |
2d2209f2 KG |
1137 | if (rc) |
1138 | goto out_err; | |
1139 | return 0; | |
1140 | out_err: | |
0a99be43 | 1141 | smcr_link_clear(link_new, false); |
2d2209f2 KG |
1142 | return rc; |
1143 | } | |
1144 | ||
1145 | static void smc_llc_process_srv_add_link(struct smc_link_group *lgr) | |
1146 | { | |
1147 | struct smc_link *link = lgr->llc_flow_lcl.qentry->link; | |
1148 | int rc; | |
1149 | ||
1150 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1151 | ||
1152 | mutex_lock(&lgr->llc_conf_mutex); | |
1153 | rc = smc_llc_srv_add_link(link); | |
1154 | if (!rc && lgr->type == SMC_LGR_SYMMETRIC) { | |
1155 | /* delete any asymmetric link */ | |
c9a5d243 | 1156 | smc_llc_delete_asym_link(lgr); |
2d2209f2 KG |
1157 | } |
1158 | mutex_unlock(&lgr->llc_conf_mutex); | |
1159 | } | |
1160 | ||
4dadd151 KG |
1161 | /* enqueue a local add_link req to trigger a new add_link flow, only as SERV */ |
1162 | void smc_llc_srv_add_link_local(struct smc_link *link) | |
1163 | { | |
1164 | struct smc_llc_msg_add_link add_llc = {0}; | |
1165 | ||
1166 | add_llc.hd.length = sizeof(add_llc); | |
1167 | add_llc.hd.common.type = SMC_LLC_ADD_LINK; | |
1168 | /* no dev and port needed, we as server ignore client data anyway */ | |
1169 | smc_llc_enqueue(link, (union smc_llc_msg *)&add_llc); | |
1170 | } | |
1171 | ||
b45e7f98 KG |
1172 | /* worker to process an add link message */ |
1173 | static void smc_llc_add_link_work(struct work_struct *work) | |
1174 | { | |
1175 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, | |
1176 | llc_add_link_work); | |
1177 | ||
1178 | if (list_empty(&lgr->list)) { | |
1179 | /* link group is terminating */ | |
1180 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1181 | goto out; | |
1182 | } | |
1183 | ||
b1570a87 KG |
1184 | if (lgr->role == SMC_CLNT) |
1185 | smc_llc_process_cli_add_link(lgr); | |
2d2209f2 KG |
1186 | else |
1187 | smc_llc_process_srv_add_link(lgr); | |
b45e7f98 KG |
1188 | out: |
1189 | smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); | |
1190 | } | |
1191 | ||
4dadd151 KG |
1192 | /* enqueue a local del_link msg to trigger a new del_link flow, |
1193 | * called only for role SMC_SERV | |
1194 | */ | |
1195 | void smc_llc_srv_delete_link_local(struct smc_link *link, u8 del_link_id) | |
1196 | { | |
1197 | struct smc_llc_msg_del_link del_llc = {0}; | |
1198 | ||
1199 | del_llc.hd.length = sizeof(del_llc); | |
1200 | del_llc.hd.common.type = SMC_LLC_DELETE_LINK; | |
1201 | del_llc.link_num = del_link_id; | |
1202 | del_llc.reason = htonl(SMC_LLC_DEL_LOST_PATH); | |
1203 | del_llc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; | |
1204 | smc_llc_enqueue(link, (union smc_llc_msg *)&del_llc); | |
1205 | } | |
1206 | ||
9c416878 KG |
1207 | static void smc_llc_process_cli_delete_link(struct smc_link_group *lgr) |
1208 | { | |
1209 | struct smc_link *lnk_del = NULL, *lnk_asym, *lnk; | |
1210 | struct smc_llc_msg_del_link *del_llc; | |
1211 | struct smc_llc_qentry *qentry; | |
1212 | int active_links; | |
1213 | int lnk_idx; | |
1214 | ||
1215 | qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); | |
1216 | lnk = qentry->link; | |
1217 | del_llc = &qentry->msg.delete_link; | |
1218 | ||
1219 | if (del_llc->hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { | |
1220 | smc_lgr_terminate_sched(lgr); | |
1221 | goto out; | |
1222 | } | |
1223 | mutex_lock(&lgr->llc_conf_mutex); | |
1224 | /* delete single link */ | |
1225 | for (lnk_idx = 0; lnk_idx < SMC_LINKS_PER_LGR_MAX; lnk_idx++) { | |
1226 | if (lgr->lnk[lnk_idx].link_id != del_llc->link_num) | |
1227 | continue; | |
1228 | lnk_del = &lgr->lnk[lnk_idx]; | |
1229 | break; | |
1230 | } | |
1231 | del_llc->hd.flags |= SMC_LLC_FLAG_RESP; | |
1232 | if (!lnk_del) { | |
1233 | /* link was not found */ | |
1234 | del_llc->reason = htonl(SMC_LLC_DEL_NOLNK); | |
1235 | smc_llc_send_message(lnk, &qentry->msg); | |
1236 | goto out_unlock; | |
1237 | } | |
1238 | lnk_asym = smc_llc_find_asym_link(lgr); | |
1239 | ||
1240 | del_llc->reason = 0; | |
1241 | smc_llc_send_message(lnk, &qentry->msg); /* response */ | |
1242 | ||
1243 | if (smc_link_downing(&lnk_del->state)) { | |
c6f02ebe | 1244 | smc_switch_conns(lgr, lnk_del, false); |
9c416878 KG |
1245 | smc_wr_tx_wait_no_pending_sends(lnk_del); |
1246 | } | |
0a99be43 | 1247 | smcr_link_clear(lnk_del, true); |
9c416878 KG |
1248 | |
1249 | active_links = smc_llc_active_link_count(lgr); | |
1250 | if (lnk_del == lnk_asym) { | |
1251 | /* expected deletion of asym link, don't change lgr state */ | |
1252 | } else if (active_links == 1) { | |
ad6c111b | 1253 | smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); |
9c416878 | 1254 | } else if (!active_links) { |
ad6c111b | 1255 | smcr_lgr_set_type(lgr, SMC_LGR_NONE); |
9c416878 KG |
1256 | smc_lgr_terminate_sched(lgr); |
1257 | } | |
1258 | out_unlock: | |
1259 | mutex_unlock(&lgr->llc_conf_mutex); | |
1260 | out: | |
1261 | kfree(qentry); | |
1262 | } | |
1263 | ||
f3811fd7 KG |
1264 | /* try to send a DELETE LINK ALL request on any active link, |
1265 | * waiting for send completion | |
1266 | */ | |
1267 | void smc_llc_send_link_delete_all(struct smc_link_group *lgr, bool ord, u32 rsn) | |
1268 | { | |
1269 | struct smc_llc_msg_del_link delllc = {0}; | |
1270 | int i; | |
1271 | ||
1272 | delllc.hd.common.type = SMC_LLC_DELETE_LINK; | |
1273 | delllc.hd.length = sizeof(delllc); | |
1274 | if (ord) | |
1275 | delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ORDERLY; | |
1276 | delllc.hd.flags |= SMC_LLC_FLAG_DEL_LINK_ALL; | |
1277 | delllc.reason = htonl(rsn); | |
1278 | ||
1279 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
1280 | if (!smc_link_usable(&lgr->lnk[i])) | |
1281 | continue; | |
1282 | if (!smc_llc_send_message_wait(&lgr->lnk[i], &delllc)) | |
1283 | break; | |
1284 | } | |
1285 | } | |
1286 | ||
08ae27dd KG |
1287 | static void smc_llc_process_srv_delete_link(struct smc_link_group *lgr) |
1288 | { | |
1289 | struct smc_llc_msg_del_link *del_llc; | |
1290 | struct smc_link *lnk, *lnk_del; | |
1291 | struct smc_llc_qentry *qentry; | |
1292 | int active_links; | |
1293 | int i; | |
1294 | ||
1295 | mutex_lock(&lgr->llc_conf_mutex); | |
1296 | qentry = smc_llc_flow_qentry_clr(&lgr->llc_flow_lcl); | |
1297 | lnk = qentry->link; | |
1298 | del_llc = &qentry->msg.delete_link; | |
1299 | ||
1300 | if (qentry->msg.delete_link.hd.flags & SMC_LLC_FLAG_DEL_LINK_ALL) { | |
1301 | /* delete entire lgr */ | |
f3811fd7 KG |
1302 | smc_llc_send_link_delete_all(lgr, true, ntohl( |
1303 | qentry->msg.delete_link.reason)); | |
08ae27dd KG |
1304 | smc_lgr_terminate_sched(lgr); |
1305 | goto out; | |
1306 | } | |
1307 | /* delete single link */ | |
1308 | lnk_del = NULL; | |
1309 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
1310 | if (lgr->lnk[i].link_id == del_llc->link_num) { | |
1311 | lnk_del = &lgr->lnk[i]; | |
1312 | break; | |
1313 | } | |
1314 | } | |
1315 | if (!lnk_del) | |
1316 | goto out; /* asymmetric link already deleted */ | |
1317 | ||
1318 | if (smc_link_downing(&lnk_del->state)) { | |
c6f02ebe | 1319 | smc_switch_conns(lgr, lnk_del, false); |
08ae27dd KG |
1320 | smc_wr_tx_wait_no_pending_sends(lnk_del); |
1321 | } | |
1322 | if (!list_empty(&lgr->list)) { | |
1323 | /* qentry is either a request from peer (send it back to | |
1324 | * initiate the DELETE_LINK processing), or a locally | |
1325 | * enqueued DELETE_LINK request (forward it) | |
1326 | */ | |
1327 | if (!smc_llc_send_message(lnk, &qentry->msg)) { | |
08ae27dd KG |
1328 | struct smc_llc_qentry *qentry2; |
1329 | ||
1330 | qentry2 = smc_llc_wait(lgr, lnk, SMC_LLC_WAIT_TIME, | |
1331 | SMC_LLC_DELETE_LINK); | |
ca7e3edc | 1332 | if (qentry2) |
08ae27dd | 1333 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); |
08ae27dd KG |
1334 | } |
1335 | } | |
0a99be43 | 1336 | smcr_link_clear(lnk_del, true); |
08ae27dd KG |
1337 | |
1338 | active_links = smc_llc_active_link_count(lgr); | |
1339 | if (active_links == 1) { | |
ad6c111b | 1340 | smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); |
08ae27dd | 1341 | } else if (!active_links) { |
ad6c111b | 1342 | smcr_lgr_set_type(lgr, SMC_LGR_NONE); |
08ae27dd KG |
1343 | smc_lgr_terminate_sched(lgr); |
1344 | } | |
1345 | ||
1346 | if (lgr->type == SMC_LGR_SINGLE && !list_empty(&lgr->list)) { | |
1347 | /* trigger setup of asymm alt link */ | |
4dadd151 | 1348 | smc_llc_srv_add_link_local(lnk); |
08ae27dd KG |
1349 | } |
1350 | out: | |
1351 | mutex_unlock(&lgr->llc_conf_mutex); | |
1352 | kfree(qentry); | |
1353 | } | |
1354 | ||
9ec6bf19 | 1355 | static void smc_llc_delete_link_work(struct work_struct *work) |
52bedf37 | 1356 | { |
9ec6bf19 KG |
1357 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, |
1358 | llc_del_link_work); | |
52bedf37 | 1359 | |
9ec6bf19 KG |
1360 | if (list_empty(&lgr->list)) { |
1361 | /* link group is terminating */ | |
1362 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
1363 | goto out; | |
52bedf37 | 1364 | } |
9c416878 KG |
1365 | |
1366 | if (lgr->role == SMC_CLNT) | |
1367 | smc_llc_process_cli_delete_link(lgr); | |
08ae27dd KG |
1368 | else |
1369 | smc_llc_process_srv_delete_link(lgr); | |
9ec6bf19 KG |
1370 | out: |
1371 | smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); | |
52bedf37 KG |
1372 | } |
1373 | ||
3bc67e09 KG |
1374 | /* process a confirm_rkey request from peer, remote flow */ |
1375 | static void smc_llc_rmt_conf_rkey(struct smc_link_group *lgr) | |
4ed75de5 | 1376 | { |
3bc67e09 KG |
1377 | struct smc_llc_msg_confirm_rkey *llc; |
1378 | struct smc_llc_qentry *qentry; | |
1379 | struct smc_link *link; | |
1380 | int num_entries; | |
1381 | int rk_idx; | |
1382 | int i; | |
1383 | ||
1384 | qentry = lgr->llc_flow_rmt.qentry; | |
1385 | llc = &qentry->msg.confirm_rkey; | |
1386 | link = qentry->link; | |
1387 | ||
1388 | num_entries = llc->rtoken[0].num_rkeys; | |
1389 | /* first rkey entry is for receiving link */ | |
1390 | rk_idx = smc_rtoken_add(link, | |
1391 | llc->rtoken[0].rmb_vaddr, | |
1392 | llc->rtoken[0].rmb_key); | |
1393 | if (rk_idx < 0) | |
1394 | goto out_err; | |
1395 | ||
1396 | for (i = 1; i <= min_t(u8, num_entries, SMC_LLC_RKEYS_PER_MSG - 1); i++) | |
1397 | smc_rtoken_set2(lgr, rk_idx, llc->rtoken[i].link_id, | |
1398 | llc->rtoken[i].rmb_vaddr, | |
1399 | llc->rtoken[i].rmb_key); | |
1400 | /* max links is 3 so there is no need to support conf_rkey_cont msgs */ | |
1401 | goto out; | |
1402 | out_err: | |
1403 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; | |
1404 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_RETRY; | |
1405 | out: | |
ef79d439 | 1406 | llc->hd.flags |= SMC_LLC_FLAG_RESP; |
3bc67e09 KG |
1407 | smc_llc_send_message(link, &qentry->msg); |
1408 | smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); | |
4ed75de5 KG |
1409 | } |
1410 | ||
218b24fe KG |
1411 | /* process a delete_rkey request from peer, remote flow */ |
1412 | static void smc_llc_rmt_delete_rkey(struct smc_link_group *lgr) | |
4ed75de5 | 1413 | { |
218b24fe KG |
1414 | struct smc_llc_msg_delete_rkey *llc; |
1415 | struct smc_llc_qentry *qentry; | |
1416 | struct smc_link *link; | |
4ed75de5 KG |
1417 | u8 err_mask = 0; |
1418 | int i, max; | |
1419 | ||
218b24fe KG |
1420 | qentry = lgr->llc_flow_rmt.qentry; |
1421 | llc = &qentry->msg.delete_rkey; | |
1422 | link = qentry->link; | |
1423 | ||
ef79d439 KG |
1424 | max = min_t(u8, llc->num_rkeys, SMC_LLC_DEL_RKEY_MAX); |
1425 | for (i = 0; i < max; i++) { | |
1426 | if (smc_rtoken_delete(link, llc->rkey[i])) | |
1427 | err_mask |= 1 << (SMC_LLC_DEL_RKEY_MAX - 1 - i); | |
1428 | } | |
ef79d439 KG |
1429 | if (err_mask) { |
1430 | llc->hd.flags |= SMC_LLC_FLAG_RKEY_NEG; | |
1431 | llc->err_mask = err_mask; | |
4ed75de5 | 1432 | } |
218b24fe KG |
1433 | llc->hd.flags |= SMC_LLC_FLAG_RESP; |
1434 | smc_llc_send_message(link, &qentry->msg); | |
1435 | smc_llc_flow_qentry_del(&lgr->llc_flow_rmt); | |
1436 | } | |
ef79d439 | 1437 | |
3e0c40af KG |
1438 | static void smc_llc_protocol_violation(struct smc_link_group *lgr, u8 type) |
1439 | { | |
1440 | pr_warn_ratelimited("smc: SMC-R lg %*phN LLC protocol violation: " | |
1441 | "llc_type %d\n", SMC_LGR_ID_SIZE, &lgr->id, type); | |
1442 | smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_PROT_VIOL); | |
1443 | smc_lgr_terminate_sched(lgr); | |
1444 | } | |
1445 | ||
6c8968c4 | 1446 | /* flush the llc event queue */ |
00a049cf | 1447 | static void smc_llc_event_flush(struct smc_link_group *lgr) |
9bf9abea | 1448 | { |
6c8968c4 KG |
1449 | struct smc_llc_qentry *qentry, *q; |
1450 | ||
1451 | spin_lock_bh(&lgr->llc_event_q_lock); | |
1452 | list_for_each_entry_safe(qentry, q, &lgr->llc_event_q, list) { | |
1453 | list_del_init(&qentry->list); | |
1454 | kfree(qentry); | |
1455 | } | |
1456 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1457 | } | |
1458 | ||
1459 | static void smc_llc_event_handler(struct smc_llc_qentry *qentry) | |
1460 | { | |
1461 | union smc_llc_msg *llc = &qentry->msg; | |
1462 | struct smc_link *link = qentry->link; | |
0fb0b02b | 1463 | struct smc_link_group *lgr = link->lgr; |
9bf9abea | 1464 | |
d854fcbf | 1465 | if (!smc_link_usable(link)) |
6c8968c4 | 1466 | goto out; |
313164da KG |
1467 | |
1468 | switch (llc->raw.hdr.common.type) { | |
1469 | case SMC_LLC_TEST_LINK: | |
56e8091c KG |
1470 | llc->test_link.hd.flags |= SMC_LLC_FLAG_RESP; |
1471 | smc_llc_send_message(link, llc); | |
313164da | 1472 | break; |
52bedf37 | 1473 | case SMC_LLC_ADD_LINK: |
0fb0b02b KG |
1474 | if (list_empty(&lgr->list)) |
1475 | goto out; /* lgr is terminating */ | |
1476 | if (lgr->role == SMC_CLNT) { | |
1477 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK) { | |
1478 | /* a flow is waiting for this message */ | |
1479 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, | |
1480 | qentry); | |
6778a6be | 1481 | wake_up(&lgr->llc_msg_waiter); |
0fb0b02b KG |
1482 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, |
1483 | qentry)) { | |
b45e7f98 | 1484 | schedule_work(&lgr->llc_add_link_work); |
0fb0b02b KG |
1485 | } |
1486 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, qentry)) { | |
1487 | /* as smc server, handle client suggestion */ | |
b45e7f98 | 1488 | schedule_work(&lgr->llc_add_link_work); |
0fb0b02b KG |
1489 | } |
1490 | return; | |
1491 | case SMC_LLC_CONFIRM_LINK: | |
87f88cda | 1492 | case SMC_LLC_ADD_LINK_CONT: |
0fb0b02b KG |
1493 | if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { |
1494 | /* a flow is waiting for this message */ | |
1495 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, qentry); | |
6778a6be | 1496 | wake_up(&lgr->llc_msg_waiter); |
0fb0b02b KG |
1497 | return; |
1498 | } | |
52bedf37 KG |
1499 | break; |
1500 | case SMC_LLC_DELETE_LINK: | |
9ec6bf19 KG |
1501 | if (lgr->role == SMC_CLNT) { |
1502 | /* server requests to delete this link, send response */ | |
1503 | if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { | |
1504 | /* DEL LINK REQ during ADD LINK SEQ */ | |
1505 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, | |
1506 | qentry); | |
6778a6be | 1507 | wake_up(&lgr->llc_msg_waiter); |
9ec6bf19 KG |
1508 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, |
1509 | qentry)) { | |
1510 | schedule_work(&lgr->llc_del_link_work); | |
1511 | } | |
1512 | } else { | |
1513 | if (lgr->llc_flow_lcl.type == SMC_LLC_FLOW_ADD_LINK && | |
1514 | !lgr->llc_flow_lcl.qentry) { | |
1515 | /* DEL LINK REQ during ADD LINK SEQ */ | |
1516 | smc_llc_flow_qentry_set(&lgr->llc_flow_lcl, | |
1517 | qentry); | |
6778a6be | 1518 | wake_up(&lgr->llc_msg_waiter); |
9ec6bf19 KG |
1519 | } else if (smc_llc_flow_start(&lgr->llc_flow_lcl, |
1520 | qentry)) { | |
1521 | schedule_work(&lgr->llc_del_link_work); | |
1522 | } | |
1523 | } | |
1524 | return; | |
4ed75de5 | 1525 | case SMC_LLC_CONFIRM_RKEY: |
3bc67e09 KG |
1526 | /* new request from remote, assign to remote flow */ |
1527 | if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { | |
1528 | /* process here, does not wait for more llc msgs */ | |
1529 | smc_llc_rmt_conf_rkey(lgr); | |
1530 | smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); | |
1531 | } | |
1532 | return; | |
4ed75de5 | 1533 | case SMC_LLC_CONFIRM_RKEY_CONT: |
42d18acc KG |
1534 | /* not used because max links is 3, and 3 rkeys fit into |
1535 | * one CONFIRM_RKEY message | |
1536 | */ | |
4ed75de5 KG |
1537 | break; |
1538 | case SMC_LLC_DELETE_RKEY: | |
218b24fe KG |
1539 | /* new request from remote, assign to remote flow */ |
1540 | if (smc_llc_flow_start(&lgr->llc_flow_rmt, qentry)) { | |
1541 | /* process here, does not wait for more llc msgs */ | |
1542 | smc_llc_rmt_delete_rkey(lgr); | |
1543 | smc_llc_flow_stop(lgr, &lgr->llc_flow_rmt); | |
1544 | } | |
1545 | return; | |
3e0c40af KG |
1546 | default: |
1547 | smc_llc_protocol_violation(lgr, llc->raw.hdr.common.type); | |
1548 | break; | |
313164da | 1549 | } |
6c8968c4 KG |
1550 | out: |
1551 | kfree(qentry); | |
1552 | } | |
1553 | ||
1554 | /* worker to process llc messages on the event queue */ | |
1555 | static void smc_llc_event_work(struct work_struct *work) | |
1556 | { | |
1557 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, | |
1558 | llc_event_work); | |
1559 | struct smc_llc_qentry *qentry; | |
1560 | ||
555da9af KG |
1561 | if (!lgr->llc_flow_lcl.type && lgr->delayed_event) { |
1562 | if (smc_link_usable(lgr->delayed_event->link)) { | |
1563 | smc_llc_event_handler(lgr->delayed_event); | |
1564 | } else { | |
1565 | qentry = lgr->delayed_event; | |
1566 | lgr->delayed_event = NULL; | |
1567 | kfree(qentry); | |
1568 | } | |
1569 | } | |
1570 | ||
6c8968c4 KG |
1571 | again: |
1572 | spin_lock_bh(&lgr->llc_event_q_lock); | |
1573 | if (!list_empty(&lgr->llc_event_q)) { | |
1574 | qentry = list_first_entry(&lgr->llc_event_q, | |
1575 | struct smc_llc_qentry, list); | |
1576 | list_del_init(&qentry->list); | |
1577 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1578 | smc_llc_event_handler(qentry); | |
1579 | goto again; | |
1580 | } | |
1581 | spin_unlock_bh(&lgr->llc_event_q_lock); | |
1582 | } | |
1583 | ||
ef79d439 | 1584 | /* process llc responses in tasklet context */ |
a6688d91 KG |
1585 | static void smc_llc_rx_response(struct smc_link *link, |
1586 | struct smc_llc_qentry *qentry) | |
ef79d439 | 1587 | { |
a6688d91 | 1588 | u8 llc_type = qentry->msg.raw.hdr.common.type; |
ef79d439 | 1589 | |
a6688d91 | 1590 | switch (llc_type) { |
ef79d439 KG |
1591 | case SMC_LLC_TEST_LINK: |
1592 | if (link->state == SMC_LNK_ACTIVE) | |
1593 | complete(&link->llc_testlink_resp); | |
1594 | break; | |
ef79d439 | 1595 | case SMC_LLC_ADD_LINK: |
9ec6bf19 | 1596 | case SMC_LLC_DELETE_LINK: |
4667bb4a | 1597 | case SMC_LLC_CONFIRM_LINK: |
87f88cda | 1598 | case SMC_LLC_ADD_LINK_CONT: |
3d88a21b | 1599 | case SMC_LLC_CONFIRM_RKEY: |
6d74c3a8 | 1600 | case SMC_LLC_DELETE_RKEY: |
4667bb4a KG |
1601 | /* assign responses to the local flow, we requested them */ |
1602 | smc_llc_flow_qentry_set(&link->lgr->llc_flow_lcl, qentry); | |
6778a6be | 1603 | wake_up(&link->lgr->llc_msg_waiter); |
4667bb4a | 1604 | return; |
ef79d439 | 1605 | case SMC_LLC_CONFIRM_RKEY_CONT: |
42d18acc | 1606 | /* not used because max links is 3 */ |
ef79d439 | 1607 | break; |
3e0c40af KG |
1608 | default: |
1609 | smc_llc_protocol_violation(link->lgr, llc_type); | |
1610 | break; | |
ef79d439 | 1611 | } |
a6688d91 | 1612 | kfree(qentry); |
ef79d439 KG |
1613 | } |
1614 | ||
a6688d91 | 1615 | static void smc_llc_enqueue(struct smc_link *link, union smc_llc_msg *llc) |
6c8968c4 | 1616 | { |
6c8968c4 KG |
1617 | struct smc_link_group *lgr = link->lgr; |
1618 | struct smc_llc_qentry *qentry; | |
6c8968c4 KG |
1619 | unsigned long flags; |
1620 | ||
6c8968c4 KG |
1621 | qentry = kmalloc(sizeof(*qentry), GFP_ATOMIC); |
1622 | if (!qentry) | |
1623 | return; | |
1624 | qentry->link = link; | |
1625 | INIT_LIST_HEAD(&qentry->list); | |
1626 | memcpy(&qentry->msg, llc, sizeof(union smc_llc_msg)); | |
a6688d91 KG |
1627 | |
1628 | /* process responses immediately */ | |
1629 | if (llc->raw.hdr.flags & SMC_LLC_FLAG_RESP) { | |
1630 | smc_llc_rx_response(link, qentry); | |
1631 | return; | |
1632 | } | |
1633 | ||
1634 | /* add requests to event queue */ | |
6c8968c4 KG |
1635 | spin_lock_irqsave(&lgr->llc_event_q_lock, flags); |
1636 | list_add_tail(&qentry->list, &lgr->llc_event_q); | |
1637 | spin_unlock_irqrestore(&lgr->llc_event_q_lock, flags); | |
6778a6be | 1638 | schedule_work(&lgr->llc_event_work); |
9bf9abea UB |
1639 | } |
1640 | ||
a6688d91 KG |
1641 | /* copy received msg and add it to the event queue */ |
1642 | static void smc_llc_rx_handler(struct ib_wc *wc, void *buf) | |
1643 | { | |
1644 | struct smc_link *link = (struct smc_link *)wc->qp->qp_context; | |
1645 | union smc_llc_msg *llc = buf; | |
1646 | ||
1647 | if (wc->byte_len < sizeof(*llc)) | |
1648 | return; /* short message */ | |
1649 | if (llc->raw.hdr.length != sizeof(*llc)) | |
1650 | return; /* invalid message */ | |
1651 | ||
1652 | smc_llc_enqueue(link, llc); | |
1653 | } | |
1654 | ||
44aa81ce | 1655 | /***************************** worker, utils *********************************/ |
877ae5be KG |
1656 | |
1657 | static void smc_llc_testlink_work(struct work_struct *work) | |
1658 | { | |
1659 | struct smc_link *link = container_of(to_delayed_work(work), | |
1660 | struct smc_link, llc_testlink_wrk); | |
1661 | unsigned long next_interval; | |
877ae5be KG |
1662 | unsigned long expire_time; |
1663 | u8 user_data[16] = { 0 }; | |
1664 | int rc; | |
1665 | ||
877ae5be KG |
1666 | if (link->state != SMC_LNK_ACTIVE) |
1667 | return; /* don't reschedule worker */ | |
1668 | expire_time = link->wr_rx_tstamp + link->llc_testlink_time; | |
1669 | if (time_is_after_jiffies(expire_time)) { | |
1670 | next_interval = expire_time - jiffies; | |
1671 | goto out; | |
1672 | } | |
1673 | reinit_completion(&link->llc_testlink_resp); | |
d97935fa | 1674 | smc_llc_send_test_link(link, user_data); |
877ae5be KG |
1675 | /* receive TEST LINK response over RoCE fabric */ |
1676 | rc = wait_for_completion_interruptible_timeout(&link->llc_testlink_resp, | |
1677 | SMC_LLC_WAIT_TIME); | |
1020e1ef KG |
1678 | if (link->state != SMC_LNK_ACTIVE) |
1679 | return; /* link state changed */ | |
877ae5be | 1680 | if (rc <= 0) { |
87523930 | 1681 | smcr_link_down_cond_sched(link); |
877ae5be KG |
1682 | return; |
1683 | } | |
1684 | next_interval = link->llc_testlink_time; | |
1685 | out: | |
1020e1ef | 1686 | schedule_delayed_work(&link->llc_testlink_wrk, next_interval); |
877ae5be KG |
1687 | } |
1688 | ||
00a049cf KG |
1689 | void smc_llc_lgr_init(struct smc_link_group *lgr, struct smc_sock *smc) |
1690 | { | |
1691 | struct net *net = sock_net(smc->clcsock->sk); | |
1692 | ||
1693 | INIT_WORK(&lgr->llc_event_work, smc_llc_event_work); | |
b45e7f98 | 1694 | INIT_WORK(&lgr->llc_add_link_work, smc_llc_add_link_work); |
9ec6bf19 | 1695 | INIT_WORK(&lgr->llc_del_link_work, smc_llc_delete_link_work); |
00a049cf KG |
1696 | INIT_LIST_HEAD(&lgr->llc_event_q); |
1697 | spin_lock_init(&lgr->llc_event_q_lock); | |
555da9af | 1698 | spin_lock_init(&lgr->llc_flow_lock); |
6778a6be KG |
1699 | init_waitqueue_head(&lgr->llc_flow_waiter); |
1700 | init_waitqueue_head(&lgr->llc_msg_waiter); | |
d5500667 | 1701 | mutex_init(&lgr->llc_conf_mutex); |
00a049cf KG |
1702 | lgr->llc_testlink_time = net->ipv4.sysctl_tcp_keepalive_time; |
1703 | } | |
1704 | ||
1705 | /* called after lgr was removed from lgr_list */ | |
1706 | void smc_llc_lgr_clear(struct smc_link_group *lgr) | |
1707 | { | |
1708 | smc_llc_event_flush(lgr); | |
6778a6be KG |
1709 | wake_up_all(&lgr->llc_flow_waiter); |
1710 | wake_up_all(&lgr->llc_msg_waiter); | |
00a049cf | 1711 | cancel_work_sync(&lgr->llc_event_work); |
b45e7f98 | 1712 | cancel_work_sync(&lgr->llc_add_link_work); |
9ec6bf19 | 1713 | cancel_work_sync(&lgr->llc_del_link_work); |
555da9af KG |
1714 | if (lgr->delayed_event) { |
1715 | kfree(lgr->delayed_event); | |
1716 | lgr->delayed_event = NULL; | |
1717 | } | |
00a049cf KG |
1718 | } |
1719 | ||
2a4c57a9 | 1720 | int smc_llc_link_init(struct smc_link *link) |
877ae5be KG |
1721 | { |
1722 | init_completion(&link->llc_testlink_resp); | |
1723 | INIT_DELAYED_WORK(&link->llc_testlink_wrk, smc_llc_testlink_work); | |
2a4c57a9 | 1724 | return 0; |
b32cf4ab KG |
1725 | } |
1726 | ||
00a049cf | 1727 | void smc_llc_link_active(struct smc_link *link) |
b32cf4ab | 1728 | { |
0a99be43 KG |
1729 | pr_warn_ratelimited("smc: SMC-R lg %*phN link added: id %*phN, " |
1730 | "peerid %*phN, ibdev %s, ibport %d\n", | |
1731 | SMC_LGR_ID_SIZE, &link->lgr->id, | |
1732 | SMC_LGR_ID_SIZE, &link->link_uid, | |
1733 | SMC_LGR_ID_SIZE, &link->peer_link_uid, | |
1734 | link->smcibdev->ibdev->name, link->ibport); | |
877ae5be | 1735 | link->state = SMC_LNK_ACTIVE; |
00a049cf KG |
1736 | if (link->lgr->llc_testlink_time) { |
1737 | link->llc_testlink_time = link->lgr->llc_testlink_time * HZ; | |
1020e1ef KG |
1738 | schedule_delayed_work(&link->llc_testlink_wrk, |
1739 | link->llc_testlink_time); | |
877ae5be KG |
1740 | } |
1741 | } | |
1742 | ||
877ae5be | 1743 | /* called in worker context */ |
0a99be43 | 1744 | void smc_llc_link_clear(struct smc_link *link, bool log) |
877ae5be | 1745 | { |
0a99be43 KG |
1746 | if (log) |
1747 | pr_warn_ratelimited("smc: SMC-R lg %*phN link removed: id %*phN" | |
1748 | ", peerid %*phN, ibdev %s, ibport %d\n", | |
1749 | SMC_LGR_ID_SIZE, &link->lgr->id, | |
1750 | SMC_LGR_ID_SIZE, &link->link_uid, | |
1751 | SMC_LGR_ID_SIZE, &link->peer_link_uid, | |
1752 | link->smcibdev->ibdev->name, link->ibport); | |
2140ac26 KG |
1753 | complete(&link->llc_testlink_resp); |
1754 | cancel_delayed_work_sync(&link->llc_testlink_wrk); | |
1755 | smc_wr_wakeup_reg_wait(link); | |
1756 | smc_wr_wakeup_tx_wait(link); | |
877ae5be KG |
1757 | } |
1758 | ||
3d88a21b KG |
1759 | /* register a new rtoken at the remote peer (for all links) */ |
1760 | int smc_llc_do_confirm_rkey(struct smc_link *send_link, | |
44aa81ce KG |
1761 | struct smc_buf_desc *rmb_desc) |
1762 | { | |
3d88a21b KG |
1763 | struct smc_link_group *lgr = send_link->lgr; |
1764 | struct smc_llc_qentry *qentry = NULL; | |
1765 | int rc = 0; | |
44aa81ce | 1766 | |
3d88a21b KG |
1767 | rc = smc_llc_send_confirm_rkey(send_link, rmb_desc); |
1768 | if (rc) | |
1769 | goto out; | |
44aa81ce | 1770 | /* receive CONFIRM RKEY response from server over RoCE fabric */ |
3d88a21b KG |
1771 | qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, |
1772 | SMC_LLC_CONFIRM_RKEY); | |
1773 | if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) | |
1774 | rc = -EFAULT; | |
1775 | out: | |
1776 | if (qentry) | |
1777 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
3d88a21b | 1778 | return rc; |
44aa81ce KG |
1779 | } |
1780 | ||
60e03c62 | 1781 | /* unregister an rtoken at the remote peer */ |
6d74c3a8 | 1782 | int smc_llc_do_delete_rkey(struct smc_link_group *lgr, |
60e03c62 KG |
1783 | struct smc_buf_desc *rmb_desc) |
1784 | { | |
6d74c3a8 KG |
1785 | struct smc_llc_qentry *qentry = NULL; |
1786 | struct smc_link *send_link; | |
0b29ec64 | 1787 | int rc = 0; |
60e03c62 | 1788 | |
6d74c3a8 KG |
1789 | send_link = smc_llc_usable_link(lgr); |
1790 | if (!send_link) | |
1791 | return -ENOLINK; | |
1792 | ||
6d74c3a8 KG |
1793 | /* protected by llc_flow control */ |
1794 | rc = smc_llc_send_delete_rkey(send_link, rmb_desc); | |
60e03c62 KG |
1795 | if (rc) |
1796 | goto out; | |
1797 | /* receive DELETE RKEY response from server over RoCE fabric */ | |
6d74c3a8 KG |
1798 | qentry = smc_llc_wait(lgr, send_link, SMC_LLC_WAIT_TIME, |
1799 | SMC_LLC_DELETE_RKEY); | |
1800 | if (!qentry || (qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_RKEY_NEG)) | |
60e03c62 | 1801 | rc = -EFAULT; |
60e03c62 | 1802 | out: |
6d74c3a8 KG |
1803 | if (qentry) |
1804 | smc_llc_flow_qentry_del(&lgr->llc_flow_lcl); | |
60e03c62 KG |
1805 | return rc; |
1806 | } | |
1807 | ||
45fa8da0 KG |
1808 | void smc_llc_link_set_uid(struct smc_link *link) |
1809 | { | |
1810 | __be32 link_uid; | |
1811 | ||
1812 | link_uid = htonl(*((u32 *)link->lgr->id) + link->link_id); | |
1813 | memcpy(link->link_uid, &link_uid, SMC_LGR_ID_SIZE); | |
1814 | } | |
1815 | ||
649758ff KG |
1816 | /* save peers link user id, used for debug purposes */ |
1817 | void smc_llc_save_peer_uid(struct smc_llc_qentry *qentry) | |
1818 | { | |
1819 | memcpy(qentry->link->peer_link_uid, qentry->msg.confirm_link.link_uid, | |
1820 | SMC_LGR_ID_SIZE); | |
1821 | } | |
1822 | ||
92334cfc KG |
1823 | /* evaluate confirm link request or response */ |
1824 | int smc_llc_eval_conf_link(struct smc_llc_qentry *qentry, | |
1825 | enum smc_llc_reqresp type) | |
1826 | { | |
45fa8da0 | 1827 | if (type == SMC_LLC_REQ) { /* SMC server assigns link_id */ |
92334cfc | 1828 | qentry->link->link_id = qentry->msg.confirm_link.link_num; |
45fa8da0 KG |
1829 | smc_llc_link_set_uid(qentry->link); |
1830 | } | |
92334cfc KG |
1831 | if (!(qentry->msg.raw.hdr.flags & SMC_LLC_FLAG_NO_RMBE_EYEC)) |
1832 | return -ENOTSUPP; | |
1833 | return 0; | |
1834 | } | |
1835 | ||
9bf9abea UB |
1836 | /***************************** init, exit, misc ******************************/ |
1837 | ||
1838 | static struct smc_wr_rx_handler smc_llc_rx_handlers[] = { | |
1839 | { | |
1840 | .handler = smc_llc_rx_handler, | |
1841 | .type = SMC_LLC_CONFIRM_LINK | |
1842 | }, | |
313164da KG |
1843 | { |
1844 | .handler = smc_llc_rx_handler, | |
1845 | .type = SMC_LLC_TEST_LINK | |
1846 | }, | |
52bedf37 KG |
1847 | { |
1848 | .handler = smc_llc_rx_handler, | |
1849 | .type = SMC_LLC_ADD_LINK | |
1850 | }, | |
87f88cda KG |
1851 | { |
1852 | .handler = smc_llc_rx_handler, | |
1853 | .type = SMC_LLC_ADD_LINK_CONT | |
1854 | }, | |
52bedf37 KG |
1855 | { |
1856 | .handler = smc_llc_rx_handler, | |
1857 | .type = SMC_LLC_DELETE_LINK | |
1858 | }, | |
4ed75de5 KG |
1859 | { |
1860 | .handler = smc_llc_rx_handler, | |
1861 | .type = SMC_LLC_CONFIRM_RKEY | |
1862 | }, | |
1863 | { | |
1864 | .handler = smc_llc_rx_handler, | |
1865 | .type = SMC_LLC_CONFIRM_RKEY_CONT | |
1866 | }, | |
1867 | { | |
1868 | .handler = smc_llc_rx_handler, | |
1869 | .type = SMC_LLC_DELETE_RKEY | |
1870 | }, | |
9bf9abea UB |
1871 | { |
1872 | .handler = NULL, | |
1873 | } | |
1874 | }; | |
1875 | ||
1876 | int __init smc_llc_init(void) | |
1877 | { | |
1878 | struct smc_wr_rx_handler *handler; | |
1879 | int rc = 0; | |
1880 | ||
1881 | for (handler = smc_llc_rx_handlers; handler->handler; handler++) { | |
1882 | INIT_HLIST_NODE(&handler->list); | |
1883 | rc = smc_wr_rx_register_handler(handler); | |
1884 | if (rc) | |
1885 | break; | |
1886 | } | |
1887 | return rc; | |
1888 | } |