]>
Commit | Line | Data |
---|---|---|
a2268cfb | 1 | // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause |
f58851e6 | 2 | /* |
62b56a67 | 3 | * Copyright (c) 2014-2017 Oracle. All rights reserved. |
f58851e6 TT |
4 | * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved. |
5 | * | |
6 | * This software is available to you under a choice of one of two | |
7 | * licenses. You may choose to be licensed under the terms of the GNU | |
8 | * General Public License (GPL) Version 2, available from the file | |
9 | * COPYING in the main directory of this source tree, or the BSD-type | |
10 | * license below: | |
11 | * | |
12 | * Redistribution and use in source and binary forms, with or without | |
13 | * modification, are permitted provided that the following conditions | |
14 | * are met: | |
15 | * | |
16 | * Redistributions of source code must retain the above copyright | |
17 | * notice, this list of conditions and the following disclaimer. | |
18 | * | |
19 | * Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials provided | |
22 | * with the distribution. | |
23 | * | |
24 | * Neither the name of the Network Appliance, Inc. nor the names of | |
25 | * its contributors may be used to endorse or promote products | |
26 | * derived from this software without specific prior written | |
27 | * permission. | |
28 | * | |
29 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
30 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
31 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
32 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
33 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
34 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
35 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
36 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
37 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
38 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
39 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
40 | */ | |
41 | ||
42 | /* | |
43 | * transport.c | |
44 | * | |
45 | * This file contains the top-level implementation of an RPC RDMA | |
46 | * transport. | |
47 | * | |
48 | * Naming convention: functions beginning with xprt_ are part of the | |
49 | * transport switch. All others are RPC RDMA internal. | |
50 | */ | |
51 | ||
52 | #include <linux/module.h> | |
5a0e3ad6 | 53 | #include <linux/slab.h> |
f58851e6 | 54 | #include <linux/seq_file.h> |
bd2abef3 CL |
55 | #include <linux/smp.h> |
56 | ||
5976687a | 57 | #include <linux/sunrpc/addr.h> |
bd2abef3 | 58 | #include <linux/sunrpc/svc_rdma.h> |
f58851e6 TT |
59 | |
60 | #include "xprt_rdma.h" | |
b6e717cb | 61 | #include <trace/events/rpcrdma.h> |
f58851e6 | 62 | |
f895b252 | 63 | #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
f58851e6 TT |
64 | # define RPCDBG_FACILITY RPCDBG_TRANS |
65 | #endif | |
66 | ||
f58851e6 TT |
67 | /* |
68 | * tunables | |
69 | */ | |
70 | ||
71 | static unsigned int xprt_rdma_slot_table_entries = RPCRDMA_DEF_SLOT_TABLE; | |
5d252f90 | 72 | unsigned int xprt_rdma_max_inline_read = RPCRDMA_DEF_INLINE; |
f58851e6 | 73 | static unsigned int xprt_rdma_max_inline_write = RPCRDMA_DEF_INLINE; |
ce5b3717 | 74 | unsigned int xprt_rdma_memreg_strategy = RPCRDMA_FRWR; |
fff09594 | 75 | int xprt_rdma_pad_optimize; |
f58851e6 | 76 | |
f895b252 | 77 | #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
f58851e6 TT |
78 | |
79 | static unsigned int min_slot_table_size = RPCRDMA_MIN_SLOT_TABLE; | |
80 | static unsigned int max_slot_table_size = RPCRDMA_MAX_SLOT_TABLE; | |
29c55422 CL |
81 | static unsigned int min_inline_size = RPCRDMA_MIN_INLINE; |
82 | static unsigned int max_inline_size = RPCRDMA_MAX_INLINE; | |
f58851e6 TT |
83 | static unsigned int zero; |
84 | static unsigned int max_padding = PAGE_SIZE; | |
85 | static unsigned int min_memreg = RPCRDMA_BOUNCEBUFFERS; | |
86 | static unsigned int max_memreg = RPCRDMA_LAST - 1; | |
10492704 | 87 | static unsigned int dummy; |
f58851e6 TT |
88 | |
89 | static struct ctl_table_header *sunrpc_table_header; | |
90 | ||
fe2c6338 | 91 | static struct ctl_table xr_tunables_table[] = { |
f58851e6 | 92 | { |
f58851e6 TT |
93 | .procname = "rdma_slot_table_entries", |
94 | .data = &xprt_rdma_slot_table_entries, | |
95 | .maxlen = sizeof(unsigned int), | |
96 | .mode = 0644, | |
6d456111 | 97 | .proc_handler = proc_dointvec_minmax, |
f58851e6 TT |
98 | .extra1 = &min_slot_table_size, |
99 | .extra2 = &max_slot_table_size | |
100 | }, | |
101 | { | |
f58851e6 TT |
102 | .procname = "rdma_max_inline_read", |
103 | .data = &xprt_rdma_max_inline_read, | |
104 | .maxlen = sizeof(unsigned int), | |
105 | .mode = 0644, | |
44829d02 | 106 | .proc_handler = proc_dointvec_minmax, |
29c55422 CL |
107 | .extra1 = &min_inline_size, |
108 | .extra2 = &max_inline_size, | |
f58851e6 TT |
109 | }, |
110 | { | |
f58851e6 TT |
111 | .procname = "rdma_max_inline_write", |
112 | .data = &xprt_rdma_max_inline_write, | |
113 | .maxlen = sizeof(unsigned int), | |
114 | .mode = 0644, | |
44829d02 | 115 | .proc_handler = proc_dointvec_minmax, |
29c55422 CL |
116 | .extra1 = &min_inline_size, |
117 | .extra2 = &max_inline_size, | |
f58851e6 TT |
118 | }, |
119 | { | |
f58851e6 | 120 | .procname = "rdma_inline_write_padding", |
10492704 | 121 | .data = &dummy, |
f58851e6 TT |
122 | .maxlen = sizeof(unsigned int), |
123 | .mode = 0644, | |
6d456111 | 124 | .proc_handler = proc_dointvec_minmax, |
f58851e6 TT |
125 | .extra1 = &zero, |
126 | .extra2 = &max_padding, | |
127 | }, | |
128 | { | |
f58851e6 TT |
129 | .procname = "rdma_memreg_strategy", |
130 | .data = &xprt_rdma_memreg_strategy, | |
131 | .maxlen = sizeof(unsigned int), | |
132 | .mode = 0644, | |
6d456111 | 133 | .proc_handler = proc_dointvec_minmax, |
f58851e6 TT |
134 | .extra1 = &min_memreg, |
135 | .extra2 = &max_memreg, | |
136 | }, | |
9191ca3b | 137 | { |
9191ca3b TT |
138 | .procname = "rdma_pad_optimize", |
139 | .data = &xprt_rdma_pad_optimize, | |
140 | .maxlen = sizeof(unsigned int), | |
141 | .mode = 0644, | |
6d456111 | 142 | .proc_handler = proc_dointvec, |
9191ca3b | 143 | }, |
f8572d8f | 144 | { }, |
f58851e6 TT |
145 | }; |
146 | ||
fe2c6338 | 147 | static struct ctl_table sunrpc_table[] = { |
f58851e6 | 148 | { |
f58851e6 TT |
149 | .procname = "sunrpc", |
150 | .mode = 0555, | |
151 | .child = xr_tunables_table | |
152 | }, | |
f8572d8f | 153 | { }, |
f58851e6 TT |
154 | }; |
155 | ||
156 | #endif | |
157 | ||
d31ae254 | 158 | static const struct rpc_xprt_ops xprt_rdma_procs; |
f58851e6 | 159 | |
0dd39cae CL |
160 | static void |
161 | xprt_rdma_format_addresses4(struct rpc_xprt *xprt, struct sockaddr *sap) | |
162 | { | |
163 | struct sockaddr_in *sin = (struct sockaddr_in *)sap; | |
164 | char buf[20]; | |
165 | ||
166 | snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr)); | |
167 | xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); | |
168 | ||
169 | xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA; | |
170 | } | |
171 | ||
172 | static void | |
173 | xprt_rdma_format_addresses6(struct rpc_xprt *xprt, struct sockaddr *sap) | |
174 | { | |
175 | struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap; | |
176 | char buf[40]; | |
177 | ||
178 | snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr); | |
179 | xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); | |
180 | ||
181 | xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA6; | |
182 | } | |
183 | ||
5d252f90 | 184 | void |
5231eb97 | 185 | xprt_rdma_format_addresses(struct rpc_xprt *xprt, struct sockaddr *sap) |
f58851e6 | 186 | { |
0dd39cae CL |
187 | char buf[128]; |
188 | ||
189 | switch (sap->sa_family) { | |
190 | case AF_INET: | |
191 | xprt_rdma_format_addresses4(xprt, sap); | |
192 | break; | |
193 | case AF_INET6: | |
194 | xprt_rdma_format_addresses6(xprt, sap); | |
195 | break; | |
196 | default: | |
197 | pr_err("rpcrdma: Unrecognized address family\n"); | |
198 | return; | |
199 | } | |
f58851e6 | 200 | |
c877b849 CL |
201 | (void)rpc_ntop(sap, buf, sizeof(buf)); |
202 | xprt->address_strings[RPC_DISPLAY_ADDR] = kstrdup(buf, GFP_KERNEL); | |
f58851e6 | 203 | |
81160e66 | 204 | snprintf(buf, sizeof(buf), "%u", rpc_get_port(sap)); |
c877b849 | 205 | xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); |
f58851e6 | 206 | |
81160e66 | 207 | snprintf(buf, sizeof(buf), "%4hx", rpc_get_port(sap)); |
c877b849 | 208 | xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); |
f58851e6 | 209 | |
0dd39cae | 210 | xprt->address_strings[RPC_DISPLAY_PROTO] = "rdma"; |
f58851e6 TT |
211 | } |
212 | ||
5d252f90 | 213 | void |
f58851e6 TT |
214 | xprt_rdma_free_addresses(struct rpc_xprt *xprt) |
215 | { | |
33e01dc7 CL |
216 | unsigned int i; |
217 | ||
218 | for (i = 0; i < RPC_DISPLAY_MAX; i++) | |
219 | switch (i) { | |
220 | case RPC_DISPLAY_PROTO: | |
221 | case RPC_DISPLAY_NETID: | |
222 | continue; | |
223 | default: | |
224 | kfree(xprt->address_strings[i]); | |
225 | } | |
f58851e6 TT |
226 | } |
227 | ||
31e62d25 CL |
228 | /** |
229 | * xprt_rdma_connect_worker - establish connection in the background | |
230 | * @work: worker thread context | |
231 | * | |
232 | * Requester holds the xprt's send lock to prevent activity on this | |
233 | * transport while a fresh connection is being established. RPC tasks | |
234 | * sleep on the xprt's pending queue waiting for connect to complete. | |
235 | */ | |
f58851e6 TT |
236 | static void |
237 | xprt_rdma_connect_worker(struct work_struct *work) | |
238 | { | |
5abefb86 CL |
239 | struct rpcrdma_xprt *r_xprt = container_of(work, struct rpcrdma_xprt, |
240 | rx_connect_worker.work); | |
241 | struct rpc_xprt *xprt = &r_xprt->rx_xprt; | |
31e62d25 | 242 | int rc; |
d19751e7 | 243 | |
d19751e7 | 244 | rc = rpcrdma_ep_connect(&r_xprt->rx_ep, &r_xprt->rx_ia); |
f58851e6 | 245 | xprt_clear_connecting(xprt); |
31e62d25 | 246 | if (r_xprt->rx_ep.rep_connected > 0) { |
3968a8a5 CL |
247 | if (!xprt_test_and_set_connected(xprt)) { |
248 | xprt->stat.connect_count++; | |
249 | xprt->stat.connect_time += (long)jiffies - | |
250 | xprt->stat.connect_start; | |
31e62d25 | 251 | xprt_wake_pending_tasks(xprt, -EAGAIN); |
3968a8a5 | 252 | } |
3a72dc77 CL |
253 | } else { |
254 | if (xprt_test_and_clear_connected(xprt)) | |
31e62d25 | 255 | xprt_wake_pending_tasks(xprt, rc); |
3a72dc77 | 256 | } |
f58851e6 TT |
257 | } |
258 | ||
f26c32fa CL |
259 | /** |
260 | * xprt_rdma_inject_disconnect - inject a connection fault | |
261 | * @xprt: transport context | |
262 | * | |
263 | * If @xprt is connected, disconnect it to simulate spurious connection | |
264 | * loss. | |
265 | */ | |
4a068258 CL |
266 | static void |
267 | xprt_rdma_inject_disconnect(struct rpc_xprt *xprt) | |
268 | { | |
ad091180 | 269 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); |
4a068258 | 270 | |
b4744e00 | 271 | trace_xprtrdma_inject_dsc(r_xprt); |
4a068258 CL |
272 | rdma_disconnect(r_xprt->rx_ia.ri_id); |
273 | } | |
274 | ||
f26c32fa CL |
275 | /** |
276 | * xprt_rdma_destroy - Full tear down of transport | |
277 | * @xprt: doomed transport context | |
f58851e6 | 278 | * |
f26c32fa CL |
279 | * Caller guarantees there will be no more calls to us with |
280 | * this @xprt. | |
f58851e6 TT |
281 | */ |
282 | static void | |
283 | xprt_rdma_destroy(struct rpc_xprt *xprt) | |
284 | { | |
285 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); | |
f58851e6 | 286 | |
b4744e00 | 287 | trace_xprtrdma_destroy(r_xprt); |
f58851e6 | 288 | |
5abefb86 | 289 | cancel_delayed_work_sync(&r_xprt->rx_connect_worker); |
f58851e6 | 290 | |
7f1d5419 | 291 | rpcrdma_ep_destroy(&r_xprt->rx_ep, &r_xprt->rx_ia); |
72c02173 | 292 | rpcrdma_buffer_destroy(&r_xprt->rx_buf); |
f58851e6 TT |
293 | rpcrdma_ia_close(&r_xprt->rx_ia); |
294 | ||
295 | xprt_rdma_free_addresses(xprt); | |
e204e621 | 296 | xprt_free(xprt); |
f58851e6 | 297 | |
f58851e6 TT |
298 | module_put(THIS_MODULE); |
299 | } | |
300 | ||
2881ae74 TM |
301 | static const struct rpc_timeout xprt_rdma_default_timeout = { |
302 | .to_initval = 60 * HZ, | |
303 | .to_maxval = 60 * HZ, | |
304 | }; | |
305 | ||
f58851e6 TT |
306 | /** |
307 | * xprt_setup_rdma - Set up transport to use RDMA | |
308 | * | |
309 | * @args: rpc transport arguments | |
310 | */ | |
311 | static struct rpc_xprt * | |
312 | xprt_setup_rdma(struct xprt_create *args) | |
313 | { | |
314 | struct rpcrdma_create_data_internal cdata; | |
315 | struct rpc_xprt *xprt; | |
316 | struct rpcrdma_xprt *new_xprt; | |
317 | struct rpcrdma_ep *new_ep; | |
5231eb97 | 318 | struct sockaddr *sap; |
f58851e6 TT |
319 | int rc; |
320 | ||
ddbb347f | 321 | if (args->addrlen > sizeof(xprt->addr)) |
f58851e6 | 322 | return ERR_PTR(-EBADF); |
f58851e6 | 323 | |
edb41e61 | 324 | xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt), 0, 0); |
ddbb347f | 325 | if (!xprt) |
f58851e6 | 326 | return ERR_PTR(-ENOMEM); |
f58851e6 | 327 | |
f58851e6 | 328 | /* 60 second timeout, no retries */ |
ba7392bb | 329 | xprt->timeout = &xprt_rdma_default_timeout; |
bfaee096 CL |
330 | xprt->bind_timeout = RPCRDMA_BIND_TO; |
331 | xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO; | |
332 | xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO; | |
f58851e6 TT |
333 | |
334 | xprt->resvport = 0; /* privileged port not needed */ | |
335 | xprt->tsh_size = 0; /* RPC-RDMA handles framing */ | |
f58851e6 TT |
336 | xprt->ops = &xprt_rdma_procs; |
337 | ||
338 | /* | |
339 | * Set up RDMA-specific connect data. | |
340 | */ | |
dd229cee | 341 | sap = args->dstaddr; |
f58851e6 TT |
342 | |
343 | /* Ensure xprt->addr holds valid server TCP (not RDMA) | |
344 | * address, for any side protocols which peek at it */ | |
345 | xprt->prot = IPPROTO_TCP; | |
346 | xprt->addrlen = args->addrlen; | |
5231eb97 | 347 | memcpy(&xprt->addr, sap, xprt->addrlen); |
f58851e6 | 348 | |
5231eb97 | 349 | if (rpc_get_port(sap)) |
f58851e6 | 350 | xprt_set_bound(xprt); |
d461f1f2 | 351 | xprt_rdma_format_addresses(xprt, sap); |
f58851e6 | 352 | |
edb41e61 | 353 | cdata.max_requests = xprt_rdma_slot_table_entries; |
f58851e6 | 354 | |
f58851e6 TT |
355 | cdata.rsize = RPCRDMA_MAX_SEGS * PAGE_SIZE; /* RDMA write max */ |
356 | cdata.wsize = RPCRDMA_MAX_SEGS * PAGE_SIZE; /* RDMA read max */ | |
357 | ||
358 | cdata.inline_wsize = xprt_rdma_max_inline_write; | |
359 | if (cdata.inline_wsize > cdata.wsize) | |
360 | cdata.inline_wsize = cdata.wsize; | |
361 | ||
362 | cdata.inline_rsize = xprt_rdma_max_inline_read; | |
363 | if (cdata.inline_rsize > cdata.rsize) | |
364 | cdata.inline_rsize = cdata.rsize; | |
365 | ||
f58851e6 TT |
366 | /* |
367 | * Create new transport instance, which includes initialized | |
368 | * o ia | |
369 | * o endpoint | |
370 | * o buffers | |
371 | */ | |
372 | ||
373 | new_xprt = rpcx_to_rdmax(xprt); | |
374 | ||
dd229cee | 375 | rc = rpcrdma_ia_open(new_xprt); |
f58851e6 TT |
376 | if (rc) |
377 | goto out1; | |
378 | ||
379 | /* | |
380 | * initialize and create ep | |
381 | */ | |
382 | new_xprt->rx_data = cdata; | |
383 | new_ep = &new_xprt->rx_ep; | |
f58851e6 TT |
384 | |
385 | rc = rpcrdma_ep_create(&new_xprt->rx_ep, | |
386 | &new_xprt->rx_ia, &new_xprt->rx_data); | |
387 | if (rc) | |
388 | goto out2; | |
389 | ||
ac920d04 | 390 | rc = rpcrdma_buffer_create(new_xprt); |
f58851e6 TT |
391 | if (rc) |
392 | goto out3; | |
393 | ||
5abefb86 CL |
394 | INIT_DELAYED_WORK(&new_xprt->rx_connect_worker, |
395 | xprt_rdma_connect_worker); | |
f58851e6 | 396 | |
5f62412b | 397 | xprt->max_payload = frwr_maxpages(new_xprt); |
1c9351ee CL |
398 | if (xprt->max_payload == 0) |
399 | goto out4; | |
400 | xprt->max_payload <<= PAGE_SHIFT; | |
43e95988 CL |
401 | dprintk("RPC: %s: transport data payload maximum: %zu bytes\n", |
402 | __func__, xprt->max_payload); | |
f58851e6 TT |
403 | |
404 | if (!try_module_get(THIS_MODULE)) | |
405 | goto out4; | |
406 | ||
5231eb97 CL |
407 | dprintk("RPC: %s: %s:%s\n", __func__, |
408 | xprt->address_strings[RPC_DISPLAY_ADDR], | |
409 | xprt->address_strings[RPC_DISPLAY_PORT]); | |
b4744e00 | 410 | trace_xprtrdma_create(new_xprt); |
f58851e6 TT |
411 | return xprt; |
412 | ||
413 | out4: | |
03ac1a76 | 414 | rpcrdma_buffer_destroy(&new_xprt->rx_buf); |
03ac1a76 | 415 | rc = -ENODEV; |
f58851e6 | 416 | out3: |
7f1d5419 | 417 | rpcrdma_ep_destroy(new_ep, &new_xprt->rx_ia); |
f58851e6 TT |
418 | out2: |
419 | rpcrdma_ia_close(&new_xprt->rx_ia); | |
420 | out1: | |
b4744e00 | 421 | trace_xprtrdma_destroy(new_xprt); |
d461f1f2 | 422 | xprt_rdma_free_addresses(xprt); |
e204e621 | 423 | xprt_free(xprt); |
f58851e6 TT |
424 | return ERR_PTR(rc); |
425 | } | |
426 | ||
bebd0318 | 427 | /** |
f26c32fa CL |
428 | * xprt_rdma_close - close a transport connection |
429 | * @xprt: transport context | |
bebd0318 | 430 | * |
f26c32fa CL |
431 | * Called during transport shutdown, reconnect, or device removal. |
432 | * Caller holds @xprt's send lock to prevent activity on this | |
433 | * transport while the connection is torn down. | |
f58851e6 | 434 | */ |
0c0829bc | 435 | void xprt_rdma_close(struct rpc_xprt *xprt) |
f58851e6 TT |
436 | { |
437 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); | |
bebd0318 CL |
438 | struct rpcrdma_ep *ep = &r_xprt->rx_ep; |
439 | struct rpcrdma_ia *ia = &r_xprt->rx_ia; | |
440 | ||
6d2d0ee2 CL |
441 | might_sleep(); |
442 | ||
6d2d0ee2 CL |
443 | /* Prevent marshaling and sending of new requests */ |
444 | xprt_clear_connected(xprt); | |
445 | ||
bebd0318 | 446 | if (test_and_clear_bit(RPCRDMA_IAF_REMOVING, &ia->ri_flags)) { |
bebd0318 | 447 | rpcrdma_ia_remove(ia); |
0c0829bc | 448 | goto out; |
bebd0318 | 449 | } |
0c0829bc | 450 | |
bebd0318 CL |
451 | if (ep->rep_connected == -ENODEV) |
452 | return; | |
453 | if (ep->rep_connected > 0) | |
08ca0dce | 454 | xprt->reestablish_timeout = 0; |
bebd0318 | 455 | rpcrdma_ep_disconnect(ep, ia); |
ef739b21 CL |
456 | |
457 | /* Prepare @xprt for the next connection by reinitializing | |
458 | * its credit grant to one (see RFC 8166, Section 3.3.3). | |
459 | */ | |
460 | r_xprt->rx_buf.rb_credits = 1; | |
461 | xprt->cwnd = RPC_CWNDSHIFT; | |
0c0829bc CL |
462 | |
463 | out: | |
464 | ++xprt->connect_cookie; | |
465 | xprt_disconnect_done(xprt); | |
f58851e6 TT |
466 | } |
467 | ||
20035edf CL |
468 | /** |
469 | * xprt_rdma_set_port - update server port with rpcbind result | |
470 | * @xprt: controlling RPC transport | |
471 | * @port: new port value | |
472 | * | |
473 | * Transport connect status is unchanged. | |
474 | */ | |
f58851e6 TT |
475 | static void |
476 | xprt_rdma_set_port(struct rpc_xprt *xprt, u16 port) | |
477 | { | |
20035edf CL |
478 | struct sockaddr *sap = (struct sockaddr *)&xprt->addr; |
479 | char buf[8]; | |
480 | ||
481 | dprintk("RPC: %s: setting port for xprt %p (%s:%s) to %u\n", | |
482 | __func__, xprt, | |
483 | xprt->address_strings[RPC_DISPLAY_ADDR], | |
484 | xprt->address_strings[RPC_DISPLAY_PORT], | |
485 | port); | |
486 | ||
487 | rpc_set_port(sap, port); | |
f58851e6 | 488 | |
20035edf CL |
489 | kfree(xprt->address_strings[RPC_DISPLAY_PORT]); |
490 | snprintf(buf, sizeof(buf), "%u", port); | |
491 | xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); | |
492 | ||
493 | kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]); | |
494 | snprintf(buf, sizeof(buf), "%4hx", port); | |
495 | xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); | |
f58851e6 TT |
496 | } |
497 | ||
33849792 CL |
498 | /** |
499 | * xprt_rdma_timer - invoked when an RPC times out | |
500 | * @xprt: controlling RPC transport | |
501 | * @task: RPC task that timed out | |
502 | * | |
503 | * Invoked when the transport is still connected, but an RPC | |
504 | * retransmit timeout occurs. | |
505 | * | |
506 | * Since RDMA connections don't have a keep-alive, forcibly | |
507 | * disconnect and retry to connect. This drives full | |
508 | * detection of the network path, and retransmissions of | |
509 | * all pending RPCs. | |
510 | */ | |
511 | static void | |
512 | xprt_rdma_timer(struct rpc_xprt *xprt, struct rpc_task *task) | |
513 | { | |
33849792 CL |
514 | xprt_force_disconnect(xprt); |
515 | } | |
516 | ||
f26c32fa CL |
517 | /** |
518 | * xprt_rdma_connect - try to establish a transport connection | |
519 | * @xprt: transport state | |
520 | * @task: RPC scheduler context | |
521 | * | |
522 | */ | |
f58851e6 | 523 | static void |
1b092092 | 524 | xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task) |
f58851e6 | 525 | { |
f58851e6 TT |
526 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); |
527 | ||
0b9e7943 TM |
528 | if (r_xprt->rx_ep.rep_connected != 0) { |
529 | /* Reconnect */ | |
5abefb86 CL |
530 | schedule_delayed_work(&r_xprt->rx_connect_worker, |
531 | xprt->reestablish_timeout); | |
0b9e7943 | 532 | xprt->reestablish_timeout <<= 1; |
bfaee096 CL |
533 | if (xprt->reestablish_timeout > RPCRDMA_MAX_REEST_TO) |
534 | xprt->reestablish_timeout = RPCRDMA_MAX_REEST_TO; | |
535 | else if (xprt->reestablish_timeout < RPCRDMA_INIT_REEST_TO) | |
536 | xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO; | |
0b9e7943 | 537 | } else { |
5abefb86 | 538 | schedule_delayed_work(&r_xprt->rx_connect_worker, 0); |
0b9e7943 | 539 | if (!RPC_IS_ASYNC(task)) |
5abefb86 | 540 | flush_delayed_work(&r_xprt->rx_connect_worker); |
f58851e6 TT |
541 | } |
542 | } | |
543 | ||
48be539d CL |
544 | /** |
545 | * xprt_rdma_alloc_slot - allocate an rpc_rqst | |
546 | * @xprt: controlling RPC transport | |
547 | * @task: RPC task requesting a fresh rpc_rqst | |
548 | * | |
549 | * tk_status values: | |
550 | * %0 if task->tk_rqstp points to a fresh rpc_rqst | |
551 | * %-EAGAIN if no rpc_rqst is available; queued on backlog | |
552 | */ | |
553 | static void | |
554 | xprt_rdma_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task) | |
555 | { | |
edb41e61 CL |
556 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); |
557 | struct rpcrdma_req *req; | |
48be539d | 558 | |
edb41e61 CL |
559 | req = rpcrdma_buffer_get(&r_xprt->rx_buf); |
560 | if (!req) | |
48be539d | 561 | goto out_sleep; |
edb41e61 | 562 | task->tk_rqstp = &req->rl_slot; |
48be539d CL |
563 | task->tk_status = 0; |
564 | return; | |
565 | ||
566 | out_sleep: | |
567 | rpc_sleep_on(&xprt->backlog, task, NULL); | |
48be539d CL |
568 | task->tk_status = -EAGAIN; |
569 | } | |
570 | ||
571 | /** | |
572 | * xprt_rdma_free_slot - release an rpc_rqst | |
573 | * @xprt: controlling RPC transport | |
574 | * @rqst: rpc_rqst to release | |
575 | * | |
576 | */ | |
577 | static void | |
578 | xprt_rdma_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *rqst) | |
579 | { | |
580 | memset(rqst, 0, sizeof(*rqst)); | |
edb41e61 | 581 | rpcrdma_buffer_put(rpcr_to_rdmar(rqst)); |
48be539d | 582 | rpc_wake_up_next(&xprt->backlog); |
48be539d CL |
583 | } |
584 | ||
9c40c49f CL |
585 | static bool |
586 | rpcrdma_get_sendbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, | |
587 | size_t size, gfp_t flags) | |
588 | { | |
589 | struct rpcrdma_regbuf *rb; | |
9c40c49f CL |
590 | |
591 | if (req->rl_sendbuf && rdmab_length(req->rl_sendbuf) >= size) | |
592 | return true; | |
593 | ||
655fec69 | 594 | rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, flags); |
9c40c49f CL |
595 | if (IS_ERR(rb)) |
596 | return false; | |
597 | ||
13650c23 | 598 | rpcrdma_free_regbuf(req->rl_sendbuf); |
655fec69 | 599 | r_xprt->rx_stats.hardway_register_count += size; |
9c40c49f CL |
600 | req->rl_sendbuf = rb; |
601 | return true; | |
602 | } | |
603 | ||
604 | /* The rq_rcv_buf is used only if a Reply chunk is necessary. | |
605 | * The decision to use a Reply chunk is made later in | |
606 | * rpcrdma_marshal_req. This buffer is registered at that time. | |
607 | * | |
608 | * Otherwise, the associated RPC Reply arrives in a separate | |
609 | * Receive buffer, arbitrarily chosen by the HCA. The buffer | |
610 | * allocated here for the RPC Reply is not utilized in that | |
611 | * case. See rpcrdma_inline_fixup. | |
612 | * | |
613 | * A regbuf is used here to remember the buffer size. | |
614 | */ | |
615 | static bool | |
616 | rpcrdma_get_recvbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req, | |
617 | size_t size, gfp_t flags) | |
618 | { | |
619 | struct rpcrdma_regbuf *rb; | |
620 | ||
621 | if (req->rl_recvbuf && rdmab_length(req->rl_recvbuf) >= size) | |
622 | return true; | |
623 | ||
13650c23 | 624 | rb = rpcrdma_alloc_regbuf(size, DMA_NONE, flags); |
9c40c49f CL |
625 | if (IS_ERR(rb)) |
626 | return false; | |
627 | ||
13650c23 | 628 | rpcrdma_free_regbuf(req->rl_recvbuf); |
9c40c49f CL |
629 | r_xprt->rx_stats.hardway_register_count += size; |
630 | req->rl_recvbuf = rb; | |
631 | return true; | |
632 | } | |
633 | ||
5fe6eaa1 CL |
634 | /** |
635 | * xprt_rdma_allocate - allocate transport resources for an RPC | |
636 | * @task: RPC task | |
637 | * | |
638 | * Return values: | |
639 | * 0: Success; rq_buffer points to RPC buffer to use | |
640 | * ENOMEM: Out of memory, call again later | |
641 | * EIO: A permanent error occurred, do not retry | |
f58851e6 | 642 | */ |
5fe6eaa1 CL |
643 | static int |
644 | xprt_rdma_allocate(struct rpc_task *task) | |
f58851e6 | 645 | { |
5fe6eaa1 | 646 | struct rpc_rqst *rqst = task->tk_rqstp; |
5fe6eaa1 | 647 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt); |
edb41e61 | 648 | struct rpcrdma_req *req = rpcr_to_rdmar(rqst); |
a0a1d50c | 649 | gfp_t flags; |
f58851e6 | 650 | |
5d252f90 | 651 | flags = RPCRDMA_DEF_GFP; |
a0a1d50c CL |
652 | if (RPC_IS_SWAPPER(task)) |
653 | flags = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN; | |
654 | ||
9c40c49f CL |
655 | if (!rpcrdma_get_sendbuf(r_xprt, req, rqst->rq_callsize, flags)) |
656 | goto out_fail; | |
657 | if (!rpcrdma_get_recvbuf(r_xprt, req, rqst->rq_rcvsize, flags)) | |
658 | goto out_fail; | |
659 | ||
5fe6eaa1 | 660 | rqst->rq_buffer = req->rl_sendbuf->rg_base; |
9c40c49f | 661 | rqst->rq_rbuffer = req->rl_recvbuf->rg_base; |
ae724676 | 662 | trace_xprtrdma_allocate(task, req); |
5fe6eaa1 | 663 | return 0; |
0ca77dc3 | 664 | |
0ca77dc3 | 665 | out_fail: |
ae724676 | 666 | trace_xprtrdma_allocate(task, NULL); |
5fe6eaa1 | 667 | return -ENOMEM; |
f58851e6 TT |
668 | } |
669 | ||
3435c74a CL |
670 | /** |
671 | * xprt_rdma_free - release resources allocated by xprt_rdma_allocate | |
672 | * @task: RPC task | |
673 | * | |
674 | * Caller guarantees rqst->rq_buffer is non-NULL. | |
f58851e6 TT |
675 | */ |
676 | static void | |
3435c74a | 677 | xprt_rdma_free(struct rpc_task *task) |
f58851e6 | 678 | { |
3435c74a CL |
679 | struct rpc_rqst *rqst = task->tk_rqstp; |
680 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt); | |
681 | struct rpcrdma_req *req = rpcr_to_rdmar(rqst); | |
f58851e6 | 682 | |
0ba6f370 CL |
683 | if (test_bit(RPCRDMA_REQ_F_PENDING, &req->rl_flags)) |
684 | rpcrdma_release_rqst(r_xprt, req); | |
ae724676 | 685 | trace_xprtrdma_rpc_done(task, req); |
f58851e6 TT |
686 | } |
687 | ||
7a89f9c6 CL |
688 | /** |
689 | * xprt_rdma_send_request - marshal and send an RPC request | |
adfa7144 | 690 | * @rqst: RPC message in rq_snd_buf |
7a89f9c6 | 691 | * |
bebd0318 CL |
692 | * Caller holds the transport's write lock. |
693 | * | |
cf73daf5 CL |
694 | * Returns: |
695 | * %0 if the RPC message has been sent | |
696 | * %-ENOTCONN if the caller should reconnect and call again | |
9e679d5e CL |
697 | * %-EAGAIN if the caller should call again |
698 | * %-ENOBUFS if the caller should call again after a delay | |
6946f823 CL |
699 | * %-EMSGSIZE if encoding ran out of buffer space. The request |
700 | * was not sent. Do not try to send this message again. | |
701 | * %-EIO if an I/O error occurred. The request was not sent. | |
702 | * Do not try to send this message again. | |
f58851e6 | 703 | */ |
f58851e6 | 704 | static int |
adfa7144 | 705 | xprt_rdma_send_request(struct rpc_rqst *rqst) |
f58851e6 | 706 | { |
a4f0835c | 707 | struct rpc_xprt *xprt = rqst->rq_xprt; |
f58851e6 TT |
708 | struct rpcrdma_req *req = rpcr_to_rdmar(rqst); |
709 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); | |
6ab59945 | 710 | int rc = 0; |
f58851e6 | 711 | |
cf73daf5 CL |
712 | #if defined(CONFIG_SUNRPC_BACKCHANNEL) |
713 | if (unlikely(!rqst->rq_buffer)) | |
714 | return xprt_rdma_bc_send_reply(rqst); | |
715 | #endif /* CONFIG_SUNRPC_BACKCHANNEL */ | |
716 | ||
bebd0318 | 717 | if (!xprt_connected(xprt)) |
0c0829bc | 718 | return -ENOTCONN; |
bebd0318 | 719 | |
75891f50 TM |
720 | if (!xprt_request_get_cong(xprt, rqst)) |
721 | return -EBADSLT; | |
722 | ||
09e60641 | 723 | rc = rpcrdma_marshal_req(r_xprt, rqst); |
6ab59945 CL |
724 | if (rc < 0) |
725 | goto failed_marshal; | |
f58851e6 | 726 | |
575448bd | 727 | /* Must suppress retransmit to maintain credits */ |
8a14793e | 728 | if (rqst->rq_connect_cookie == xprt->connect_cookie) |
575448bd | 729 | goto drop_connection; |
78215759 | 730 | rqst->rq_xtime = ktime_get(); |
575448bd | 731 | |
42b9f5c5 | 732 | __set_bit(RPCRDMA_REQ_F_PENDING, &req->rl_flags); |
575448bd TT |
733 | if (rpcrdma_ep_post(&r_xprt->rx_ia, &r_xprt->rx_ep, req)) |
734 | goto drop_connection; | |
f58851e6 | 735 | |
d60dbb20 | 736 | rqst->rq_xmit_bytes_sent += rqst->rq_snd_buf.len; |
f58851e6 | 737 | rqst->rq_bytes_sent = 0; |
fb14ae88 CL |
738 | |
739 | /* An RPC with no reply will throw off credit accounting, | |
740 | * so drop the connection to reset the credit grant. | |
741 | */ | |
50f484e2 | 742 | if (!rpc_reply_expected(rqst->rq_task)) |
fb14ae88 | 743 | goto drop_connection; |
f58851e6 | 744 | return 0; |
575448bd | 745 | |
c93c6223 | 746 | failed_marshal: |
7a89f9c6 CL |
747 | if (rc != -ENOTCONN) |
748 | return rc; | |
575448bd | 749 | drop_connection: |
0c0829bc CL |
750 | xprt_rdma_close(xprt); |
751 | return -ENOTCONN; | |
f58851e6 TT |
752 | } |
753 | ||
5d252f90 | 754 | void xprt_rdma_print_stats(struct rpc_xprt *xprt, struct seq_file *seq) |
f58851e6 TT |
755 | { |
756 | struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); | |
757 | long idle_time = 0; | |
758 | ||
759 | if (xprt_connected(xprt)) | |
760 | idle_time = (long)(jiffies - xprt->last_used) / HZ; | |
761 | ||
763f7e4e CL |
762 | seq_puts(seq, "\txprt:\trdma "); |
763 | seq_printf(seq, "%u %lu %lu %lu %ld %lu %lu %lu %llu %llu ", | |
764 | 0, /* need a local port? */ | |
765 | xprt->stat.bind_count, | |
766 | xprt->stat.connect_count, | |
8440a886 | 767 | xprt->stat.connect_time / HZ, |
763f7e4e CL |
768 | idle_time, |
769 | xprt->stat.sends, | |
770 | xprt->stat.recvs, | |
771 | xprt->stat.bad_xids, | |
772 | xprt->stat.req_u, | |
773 | xprt->stat.bklog_u); | |
505bbe64 | 774 | seq_printf(seq, "%lu %lu %lu %llu %llu %llu %llu %lu %lu %lu %lu ", |
763f7e4e CL |
775 | r_xprt->rx_stats.read_chunk_count, |
776 | r_xprt->rx_stats.write_chunk_count, | |
777 | r_xprt->rx_stats.reply_chunk_count, | |
778 | r_xprt->rx_stats.total_rdma_request, | |
779 | r_xprt->rx_stats.total_rdma_reply, | |
780 | r_xprt->rx_stats.pullup_copy_count, | |
781 | r_xprt->rx_stats.fixup_copy_count, | |
782 | r_xprt->rx_stats.hardway_register_count, | |
783 | r_xprt->rx_stats.failed_marshal_count, | |
860477d1 CL |
784 | r_xprt->rx_stats.bad_reply_count, |
785 | r_xprt->rx_stats.nomsg_call_count); | |
01bb35c8 | 786 | seq_printf(seq, "%lu %lu %lu %lu %lu %lu\n", |
61da886b | 787 | r_xprt->rx_stats.mrs_recycled, |
e2ac236c | 788 | r_xprt->rx_stats.mrs_orphaned, |
c8b920bb | 789 | r_xprt->rx_stats.mrs_allocated, |
ae72950a | 790 | r_xprt->rx_stats.local_inv_needed, |
01bb35c8 CL |
791 | r_xprt->rx_stats.empty_sendctx_q, |
792 | r_xprt->rx_stats.reply_waits_for_send); | |
f58851e6 TT |
793 | } |
794 | ||
d67fa4d8 JL |
795 | static int |
796 | xprt_rdma_enable_swap(struct rpc_xprt *xprt) | |
797 | { | |
a0451788 | 798 | return 0; |
d67fa4d8 JL |
799 | } |
800 | ||
801 | static void | |
802 | xprt_rdma_disable_swap(struct rpc_xprt *xprt) | |
803 | { | |
804 | } | |
805 | ||
f58851e6 TT |
806 | /* |
807 | * Plumbing for rpc transport switch and kernel module | |
808 | */ | |
809 | ||
d31ae254 | 810 | static const struct rpc_xprt_ops xprt_rdma_procs = { |
e7ce710a | 811 | .reserve_xprt = xprt_reserve_xprt_cong, |
f58851e6 | 812 | .release_xprt = xprt_release_xprt_cong, /* sunrpc/xprt.c */ |
48be539d CL |
813 | .alloc_slot = xprt_rdma_alloc_slot, |
814 | .free_slot = xprt_rdma_free_slot, | |
f58851e6 TT |
815 | .release_request = xprt_release_rqst_cong, /* ditto */ |
816 | .set_retrans_timeout = xprt_set_retrans_timeout_def, /* ditto */ | |
33849792 | 817 | .timer = xprt_rdma_timer, |
f58851e6 TT |
818 | .rpcbind = rpcb_getport_async, /* sunrpc/rpcb_clnt.c */ |
819 | .set_port = xprt_rdma_set_port, | |
820 | .connect = xprt_rdma_connect, | |
821 | .buf_alloc = xprt_rdma_allocate, | |
822 | .buf_free = xprt_rdma_free, | |
823 | .send_request = xprt_rdma_send_request, | |
824 | .close = xprt_rdma_close, | |
825 | .destroy = xprt_rdma_destroy, | |
d67fa4d8 JL |
826 | .print_stats = xprt_rdma_print_stats, |
827 | .enable_swap = xprt_rdma_enable_swap, | |
828 | .disable_swap = xprt_rdma_disable_swap, | |
f531a5db CL |
829 | .inject_disconnect = xprt_rdma_inject_disconnect, |
830 | #if defined(CONFIG_SUNRPC_BACKCHANNEL) | |
831 | .bc_setup = xprt_rdma_bc_setup, | |
76566773 | 832 | .bc_up = xprt_rdma_bc_up, |
6b26cc8c | 833 | .bc_maxpayload = xprt_rdma_bc_maxpayload, |
f531a5db CL |
834 | .bc_free_rqst = xprt_rdma_bc_free_rqst, |
835 | .bc_destroy = xprt_rdma_bc_destroy, | |
836 | #endif | |
f58851e6 TT |
837 | }; |
838 | ||
839 | static struct xprt_class xprt_rdma = { | |
840 | .list = LIST_HEAD_INIT(xprt_rdma.list), | |
841 | .name = "rdma", | |
842 | .owner = THIS_MODULE, | |
843 | .ident = XPRT_TRANSPORT_RDMA, | |
844 | .setup = xprt_setup_rdma, | |
845 | }; | |
846 | ||
ffe1f0df | 847 | void xprt_rdma_cleanup(void) |
f58851e6 | 848 | { |
f895b252 | 849 | #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
f58851e6 TT |
850 | if (sunrpc_table_header) { |
851 | unregister_sysctl_table(sunrpc_table_header); | |
852 | sunrpc_table_header = NULL; | |
853 | } | |
854 | #endif | |
951e721c | 855 | |
ddbb347f CL |
856 | xprt_unregister_transport(&xprt_rdma); |
857 | xprt_unregister_transport(&xprt_rdma_bc); | |
f58851e6 TT |
858 | } |
859 | ||
ffe1f0df | 860 | int xprt_rdma_init(void) |
f58851e6 TT |
861 | { |
862 | int rc; | |
863 | ||
951e721c | 864 | rc = xprt_register_transport(&xprt_rdma); |
6d2d0ee2 | 865 | if (rc) |
951e721c | 866 | return rc; |
951e721c | 867 | |
5d252f90 CL |
868 | rc = xprt_register_transport(&xprt_rdma_bc); |
869 | if (rc) { | |
870 | xprt_unregister_transport(&xprt_rdma); | |
5d252f90 CL |
871 | return rc; |
872 | } | |
873 | ||
f895b252 | 874 | #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) |
f58851e6 TT |
875 | if (!sunrpc_table_header) |
876 | sunrpc_table_header = register_sysctl_table(sunrpc_table); | |
877 | #endif | |
878 | return 0; | |
879 | } |