]>
Commit | Line | Data |
---|---|---|
80b14d5e TM |
1 | /* |
2 | * Multipath support for RPC | |
3 | * | |
4 | * Copyright (c) 2015, 2016, Primary Data, Inc. All rights reserved. | |
5 | * | |
6 | * Trond Myklebust <trond.myklebust@primarydata.com> | |
7 | * | |
8 | */ | |
9 | #include <linux/types.h> | |
10 | #include <linux/kref.h> | |
11 | #include <linux/list.h> | |
12 | #include <linux/rcupdate.h> | |
13 | #include <linux/rculist.h> | |
14 | #include <linux/slab.h> | |
15 | #include <asm/cmpxchg.h> | |
16 | #include <linux/spinlock.h> | |
17 | #include <linux/sunrpc/xprt.h> | |
18 | #include <linux/sunrpc/xprtmultipath.h> | |
19 | ||
20 | typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head, | |
21 | const struct rpc_xprt *cur); | |
22 | ||
23 | static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular; | |
24 | static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin; | |
25 | static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall; | |
26 | ||
27 | static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps, | |
28 | struct rpc_xprt *xprt) | |
29 | { | |
30 | if (unlikely(xprt_get(xprt) == NULL)) | |
31 | return; | |
32 | list_add_tail_rcu(&xprt->xprt_switch, &xps->xps_xprt_list); | |
33 | smp_wmb(); | |
34 | if (xps->xps_nxprts == 0) | |
35 | xps->xps_net = xprt->xprt_net; | |
36 | xps->xps_nxprts++; | |
37 | } | |
38 | ||
39 | /** | |
40 | * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch | |
41 | * @xps: pointer to struct rpc_xprt_switch | |
42 | * @xprt: pointer to struct rpc_xprt | |
43 | * | |
44 | * Adds xprt to the end of the list of struct rpc_xprt in xps. | |
45 | */ | |
46 | void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch *xps, | |
47 | struct rpc_xprt *xprt) | |
48 | { | |
49 | if (xprt == NULL) | |
50 | return; | |
51 | spin_lock(&xps->xps_lock); | |
52 | if (xps->xps_net == xprt->xprt_net || xps->xps_net == NULL) | |
53 | xprt_switch_add_xprt_locked(xps, xprt); | |
54 | spin_unlock(&xps->xps_lock); | |
55 | } | |
56 | ||
57 | static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps, | |
58 | struct rpc_xprt *xprt) | |
59 | { | |
60 | if (unlikely(xprt == NULL)) | |
61 | return; | |
62 | xps->xps_nxprts--; | |
63 | if (xps->xps_nxprts == 0) | |
64 | xps->xps_net = NULL; | |
65 | smp_wmb(); | |
66 | list_del_rcu(&xprt->xprt_switch); | |
67 | } | |
68 | ||
69 | /** | |
70 | * rpc_xprt_switch_remove_xprt - Removes an rpc_xprt from a rpc_xprt_switch | |
71 | * @xps: pointer to struct rpc_xprt_switch | |
72 | * @xprt: pointer to struct rpc_xprt | |
73 | * | |
74 | * Removes xprt from the list of struct rpc_xprt in xps. | |
75 | */ | |
76 | void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps, | |
77 | struct rpc_xprt *xprt) | |
78 | { | |
79 | spin_lock(&xps->xps_lock); | |
80 | xprt_switch_remove_xprt_locked(xps, xprt); | |
81 | spin_unlock(&xps->xps_lock); | |
82 | xprt_put(xprt); | |
83 | } | |
84 | ||
85 | /** | |
86 | * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch | |
87 | * @xprt: pointer to struct rpc_xprt | |
88 | * @gfp_flags: allocation flags | |
89 | * | |
90 | * On success, returns an initialised struct rpc_xprt_switch, containing | |
91 | * the entry xprt. Returns NULL on failure. | |
92 | */ | |
93 | struct rpc_xprt_switch *xprt_switch_alloc(struct rpc_xprt *xprt, | |
94 | gfp_t gfp_flags) | |
95 | { | |
96 | struct rpc_xprt_switch *xps; | |
97 | ||
98 | xps = kmalloc(sizeof(*xps), gfp_flags); | |
99 | if (xps != NULL) { | |
100 | spin_lock_init(&xps->xps_lock); | |
101 | kref_init(&xps->xps_kref); | |
102 | xps->xps_nxprts = 0; | |
103 | INIT_LIST_HEAD(&xps->xps_xprt_list); | |
104 | xps->xps_iter_ops = &rpc_xprt_iter_singular; | |
105 | xprt_switch_add_xprt_locked(xps, xprt); | |
106 | } | |
107 | ||
108 | return xps; | |
109 | } | |
110 | ||
111 | static void xprt_switch_free_entries(struct rpc_xprt_switch *xps) | |
112 | { | |
113 | spin_lock(&xps->xps_lock); | |
114 | while (!list_empty(&xps->xps_xprt_list)) { | |
115 | struct rpc_xprt *xprt; | |
116 | ||
117 | xprt = list_first_entry(&xps->xps_xprt_list, | |
118 | struct rpc_xprt, xprt_switch); | |
119 | xprt_switch_remove_xprt_locked(xps, xprt); | |
120 | spin_unlock(&xps->xps_lock); | |
121 | xprt_put(xprt); | |
122 | spin_lock(&xps->xps_lock); | |
123 | } | |
124 | spin_unlock(&xps->xps_lock); | |
125 | } | |
126 | ||
127 | static void xprt_switch_free(struct kref *kref) | |
128 | { | |
129 | struct rpc_xprt_switch *xps = container_of(kref, | |
130 | struct rpc_xprt_switch, xps_kref); | |
131 | ||
132 | xprt_switch_free_entries(xps); | |
133 | kfree_rcu(xps, xps_rcu); | |
134 | } | |
135 | ||
136 | /** | |
137 | * xprt_switch_get - Return a reference to a rpc_xprt_switch | |
138 | * @xps: pointer to struct rpc_xprt_switch | |
139 | * | |
140 | * Returns a reference to xps unless the refcount is already zero. | |
141 | */ | |
142 | struct rpc_xprt_switch *xprt_switch_get(struct rpc_xprt_switch *xps) | |
143 | { | |
144 | if (xps != NULL && kref_get_unless_zero(&xps->xps_kref)) | |
145 | return xps; | |
146 | return NULL; | |
147 | } | |
148 | ||
149 | /** | |
150 | * xprt_switch_put - Release a reference to a rpc_xprt_switch | |
151 | * @xps: pointer to struct rpc_xprt_switch | |
152 | * | |
153 | * Release the reference to xps, and free it once the refcount is zero. | |
154 | */ | |
155 | void xprt_switch_put(struct rpc_xprt_switch *xps) | |
156 | { | |
157 | if (xps != NULL) | |
158 | kref_put(&xps->xps_kref, xprt_switch_free); | |
159 | } | |
160 | ||
161 | /** | |
162 | * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch | |
163 | * @xps: pointer to struct rpc_xprt_switch | |
164 | * | |
165 | * Sets a round-robin default policy for iterators acting on xps. | |
166 | */ | |
167 | void rpc_xprt_switch_set_roundrobin(struct rpc_xprt_switch *xps) | |
168 | { | |
169 | if (READ_ONCE(xps->xps_iter_ops) != &rpc_xprt_iter_roundrobin) | |
170 | WRITE_ONCE(xps->xps_iter_ops, &rpc_xprt_iter_roundrobin); | |
171 | } | |
172 | ||
173 | static | |
174 | const struct rpc_xprt_iter_ops *xprt_iter_ops(const struct rpc_xprt_iter *xpi) | |
175 | { | |
176 | if (xpi->xpi_ops != NULL) | |
177 | return xpi->xpi_ops; | |
178 | return rcu_dereference(xpi->xpi_xpswitch)->xps_iter_ops; | |
179 | } | |
180 | ||
181 | static | |
182 | void xprt_iter_no_rewind(struct rpc_xprt_iter *xpi) | |
183 | { | |
184 | } | |
185 | ||
186 | static | |
187 | void xprt_iter_default_rewind(struct rpc_xprt_iter *xpi) | |
188 | { | |
189 | WRITE_ONCE(xpi->xpi_cursor, NULL); | |
190 | } | |
191 | ||
192 | static | |
193 | struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head) | |
194 | { | |
195 | return list_first_or_null_rcu(head, struct rpc_xprt, xprt_switch); | |
196 | } | |
197 | ||
198 | static | |
199 | struct rpc_xprt *xprt_iter_first_entry(struct rpc_xprt_iter *xpi) | |
200 | { | |
201 | struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch); | |
202 | ||
203 | if (xps == NULL) | |
204 | return NULL; | |
205 | return xprt_switch_find_first_entry(&xps->xps_xprt_list); | |
206 | } | |
207 | ||
208 | static | |
209 | struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head, | |
210 | const struct rpc_xprt *cur) | |
211 | { | |
212 | struct rpc_xprt *pos; | |
213 | ||
214 | list_for_each_entry_rcu(pos, head, xprt_switch) { | |
215 | if (cur == pos) | |
216 | return pos; | |
217 | } | |
218 | return NULL; | |
219 | } | |
220 | ||
221 | static | |
222 | struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi) | |
223 | { | |
224 | struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch); | |
225 | struct list_head *head; | |
226 | ||
227 | if (xps == NULL) | |
228 | return NULL; | |
229 | head = &xps->xps_xprt_list; | |
230 | if (xpi->xpi_cursor == NULL || xps->xps_nxprts < 2) | |
231 | return xprt_switch_find_first_entry(head); | |
232 | return xprt_switch_find_current_entry(head, xpi->xpi_cursor); | |
233 | } | |
234 | ||
235 | static | |
236 | struct rpc_xprt *xprt_switch_find_next_entry(struct list_head *head, | |
237 | const struct rpc_xprt *cur) | |
238 | { | |
239 | struct rpc_xprt *pos, *prev = NULL; | |
240 | ||
241 | list_for_each_entry_rcu(pos, head, xprt_switch) { | |
242 | if (cur == prev) | |
243 | return pos; | |
244 | prev = pos; | |
245 | } | |
246 | return NULL; | |
247 | } | |
248 | ||
249 | static | |
250 | struct rpc_xprt *xprt_switch_set_next_cursor(struct list_head *head, | |
251 | struct rpc_xprt **cursor, | |
252 | xprt_switch_find_xprt_t find_next) | |
253 | { | |
254 | struct rpc_xprt *cur, *pos, *old; | |
255 | ||
256 | cur = READ_ONCE(*cursor); | |
257 | for (;;) { | |
258 | old = cur; | |
259 | pos = find_next(head, old); | |
260 | if (pos == NULL) | |
261 | break; | |
262 | cur = cmpxchg_relaxed(cursor, old, pos); | |
263 | if (cur == old) | |
264 | break; | |
265 | } | |
266 | return pos; | |
267 | } | |
268 | ||
269 | static | |
270 | struct rpc_xprt *xprt_iter_next_entry_multiple(struct rpc_xprt_iter *xpi, | |
271 | xprt_switch_find_xprt_t find_next) | |
272 | { | |
273 | struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch); | |
274 | struct list_head *head; | |
275 | ||
276 | if (xps == NULL) | |
277 | return NULL; | |
278 | head = &xps->xps_xprt_list; | |
279 | if (xps->xps_nxprts < 2) | |
280 | return xprt_switch_find_first_entry(head); | |
281 | return xprt_switch_set_next_cursor(head, &xpi->xpi_cursor, find_next); | |
282 | } | |
283 | ||
284 | static | |
285 | struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct list_head *head, | |
286 | const struct rpc_xprt *cur) | |
287 | { | |
288 | struct rpc_xprt *ret; | |
289 | ||
290 | ret = xprt_switch_find_next_entry(head, cur); | |
291 | if (ret != NULL) | |
292 | return ret; | |
293 | return xprt_switch_find_first_entry(head); | |
294 | } | |
295 | ||
296 | static | |
297 | struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi) | |
298 | { | |
299 | return xprt_iter_next_entry_multiple(xpi, | |
300 | xprt_switch_find_next_entry_roundrobin); | |
301 | } | |
302 | ||
303 | static | |
304 | struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi) | |
305 | { | |
306 | return xprt_iter_next_entry_multiple(xpi, xprt_switch_find_next_entry); | |
307 | } | |
308 | ||
309 | /* | |
310 | * xprt_iter_rewind - Resets the xprt iterator | |
311 | * @xpi: pointer to rpc_xprt_iter | |
312 | * | |
313 | * Resets xpi to ensure that it points to the first entry in the list | |
314 | * of transports. | |
315 | */ | |
316 | static | |
317 | void xprt_iter_rewind(struct rpc_xprt_iter *xpi) | |
318 | { | |
319 | rcu_read_lock(); | |
320 | xprt_iter_ops(xpi)->xpi_rewind(xpi); | |
321 | rcu_read_unlock(); | |
322 | } | |
323 | ||
324 | static void __xprt_iter_init(struct rpc_xprt_iter *xpi, | |
325 | struct rpc_xprt_switch *xps, | |
326 | const struct rpc_xprt_iter_ops *ops) | |
327 | { | |
328 | rcu_assign_pointer(xpi->xpi_xpswitch, xprt_switch_get(xps)); | |
329 | xpi->xpi_cursor = NULL; | |
330 | xpi->xpi_ops = ops; | |
331 | } | |
332 | ||
333 | /** | |
334 | * xprt_iter_init - Initialise an xprt iterator | |
335 | * @xpi: pointer to rpc_xprt_iter | |
336 | * @xps: pointer to rpc_xprt_switch | |
337 | * | |
338 | * Initialises the iterator to use the default iterator ops | |
339 | * as set in xps. This function is mainly intended for internal | |
340 | * use in the rpc_client. | |
341 | */ | |
342 | void xprt_iter_init(struct rpc_xprt_iter *xpi, | |
343 | struct rpc_xprt_switch *xps) | |
344 | { | |
345 | __xprt_iter_init(xpi, xps, NULL); | |
346 | } | |
347 | ||
348 | /** | |
349 | * xprt_iter_init_listall - Initialise an xprt iterator | |
350 | * @xpi: pointer to rpc_xprt_iter | |
351 | * @xps: pointer to rpc_xprt_switch | |
352 | * | |
353 | * Initialises the iterator to iterate once through the entire list | |
354 | * of entries in xps. | |
355 | */ | |
356 | void xprt_iter_init_listall(struct rpc_xprt_iter *xpi, | |
357 | struct rpc_xprt_switch *xps) | |
358 | { | |
359 | __xprt_iter_init(xpi, xps, &rpc_xprt_iter_listall); | |
360 | } | |
361 | ||
362 | /** | |
363 | * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch | |
364 | * @xpi: pointer to rpc_xprt_iter | |
365 | * @xps: pointer to a new rpc_xprt_switch or NULL | |
366 | * | |
367 | * Swaps out the existing xpi->xpi_xpswitch with a new value. | |
368 | */ | |
369 | struct rpc_xprt_switch *xprt_iter_xchg_switch(struct rpc_xprt_iter *xpi, | |
370 | struct rpc_xprt_switch *newswitch) | |
371 | { | |
372 | struct rpc_xprt_switch __rcu *oldswitch; | |
373 | ||
374 | /* Atomically swap out the old xpswitch */ | |
375 | oldswitch = xchg(&xpi->xpi_xpswitch, RCU_INITIALIZER(newswitch)); | |
376 | if (newswitch != NULL) | |
377 | xprt_iter_rewind(xpi); | |
378 | return rcu_dereference_protected(oldswitch, true); | |
379 | } | |
380 | ||
381 | /** | |
382 | * xprt_iter_destroy - Destroys the xprt iterator | |
383 | * @xpi pointer to rpc_xprt_iter | |
384 | */ | |
385 | void xprt_iter_destroy(struct rpc_xprt_iter *xpi) | |
386 | { | |
387 | xprt_switch_put(xprt_iter_xchg_switch(xpi, NULL)); | |
388 | } | |
389 | ||
390 | /** | |
391 | * xprt_iter_xprt - Returns the rpc_xprt pointed to by the cursor | |
392 | * @xpi: pointer to rpc_xprt_iter | |
393 | * | |
394 | * Returns a pointer to the struct rpc_xprt that is currently | |
395 | * pointed to by the cursor. | |
396 | * Caller must be holding rcu_read_lock(). | |
397 | */ | |
398 | struct rpc_xprt *xprt_iter_xprt(struct rpc_xprt_iter *xpi) | |
399 | { | |
400 | WARN_ON_ONCE(!rcu_read_lock_held()); | |
401 | return xprt_iter_ops(xpi)->xpi_xprt(xpi); | |
402 | } | |
403 | ||
404 | static | |
405 | struct rpc_xprt *xprt_iter_get_helper(struct rpc_xprt_iter *xpi, | |
406 | struct rpc_xprt *(*fn)(struct rpc_xprt_iter *)) | |
407 | { | |
408 | struct rpc_xprt *ret; | |
409 | ||
410 | do { | |
411 | ret = fn(xpi); | |
412 | if (ret == NULL) | |
413 | break; | |
414 | ret = xprt_get(ret); | |
415 | } while (ret == NULL); | |
416 | return ret; | |
417 | } | |
418 | ||
419 | /** | |
420 | * xprt_iter_get_xprt - Returns the rpc_xprt pointed to by the cursor | |
421 | * @xpi: pointer to rpc_xprt_iter | |
422 | * | |
423 | * Returns a reference to the struct rpc_xprt that is currently | |
424 | * pointed to by the cursor. | |
425 | */ | |
426 | struct rpc_xprt *xprt_iter_get_xprt(struct rpc_xprt_iter *xpi) | |
427 | { | |
428 | struct rpc_xprt *xprt; | |
429 | ||
430 | rcu_read_lock(); | |
431 | xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_xprt); | |
432 | rcu_read_unlock(); | |
433 | return xprt; | |
434 | } | |
435 | ||
436 | /** | |
437 | * xprt_iter_get_next - Returns the next rpc_xprt following the cursor | |
438 | * @xpi: pointer to rpc_xprt_iter | |
439 | * | |
440 | * Returns a reference to the struct rpc_xprt that immediately follows the | |
441 | * entry pointed to by the cursor. | |
442 | */ | |
443 | struct rpc_xprt *xprt_iter_get_next(struct rpc_xprt_iter *xpi) | |
444 | { | |
445 | struct rpc_xprt *xprt; | |
446 | ||
447 | rcu_read_lock(); | |
448 | xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_next); | |
449 | rcu_read_unlock(); | |
450 | return xprt; | |
451 | } | |
452 | ||
453 | /* Policy for always returning the first entry in the rpc_xprt_switch */ | |
454 | static | |
455 | const struct rpc_xprt_iter_ops rpc_xprt_iter_singular = { | |
456 | .xpi_rewind = xprt_iter_no_rewind, | |
457 | .xpi_xprt = xprt_iter_first_entry, | |
458 | .xpi_next = xprt_iter_first_entry, | |
459 | }; | |
460 | ||
461 | /* Policy for round-robin iteration of entries in the rpc_xprt_switch */ | |
462 | static | |
463 | const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin = { | |
464 | .xpi_rewind = xprt_iter_default_rewind, | |
465 | .xpi_xprt = xprt_iter_current_entry, | |
466 | .xpi_next = xprt_iter_next_entry_roundrobin, | |
467 | }; | |
468 | ||
469 | /* Policy for once-through iteration of entries in the rpc_xprt_switch */ | |
470 | static | |
471 | const struct rpc_xprt_iter_ops rpc_xprt_iter_listall = { | |
472 | .xpi_rewind = xprt_iter_default_rewind, | |
473 | .xpi_xprt = xprt_iter_current_entry, | |
474 | .xpi_next = xprt_iter_next_entry_all, | |
475 | }; |