2 * Multipath support for RPC
4 * Copyright (c) 2015, 2016, Primary Data, Inc. All rights reserved.
6 * Trond Myklebust <trond.myklebust@primarydata.com>
9 #include <linux/types.h>
10 #include <linux/kref.h>
11 #include <linux/list.h>
12 #include <linux/rcupdate.h>
13 #include <linux/rculist.h>
14 #include <linux/slab.h>
15 #include <asm/cmpxchg.h>
16 #include <linux/spinlock.h>
17 #include <linux/sunrpc/xprt.h>
18 #include <linux/sunrpc/xprtmultipath.h>
20 typedef struct rpc_xprt
*(*xprt_switch_find_xprt_t
)(struct list_head
*head
,
21 const struct rpc_xprt
*cur
);
23 static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular
;
24 static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin
;
25 static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall
;
27 static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch
*xps
,
28 struct rpc_xprt
*xprt
)
30 if (unlikely(xprt_get(xprt
) == NULL
))
32 list_add_tail_rcu(&xprt
->xprt_switch
, &xps
->xps_xprt_list
);
34 if (xps
->xps_nxprts
== 0)
35 xps
->xps_net
= xprt
->xprt_net
;
40 * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch
41 * @xps: pointer to struct rpc_xprt_switch
42 * @xprt: pointer to struct rpc_xprt
44 * Adds xprt to the end of the list of struct rpc_xprt in xps.
46 void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch
*xps
,
47 struct rpc_xprt
*xprt
)
51 spin_lock(&xps
->xps_lock
);
52 if (xps
->xps_net
== xprt
->xprt_net
|| xps
->xps_net
== NULL
)
53 xprt_switch_add_xprt_locked(xps
, xprt
);
54 spin_unlock(&xps
->xps_lock
);
57 static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch
*xps
,
58 struct rpc_xprt
*xprt
)
60 if (unlikely(xprt
== NULL
))
63 if (xps
->xps_nxprts
== 0)
66 list_del_rcu(&xprt
->xprt_switch
);
70 * rpc_xprt_switch_remove_xprt - Removes an rpc_xprt from a rpc_xprt_switch
71 * @xps: pointer to struct rpc_xprt_switch
72 * @xprt: pointer to struct rpc_xprt
74 * Removes xprt from the list of struct rpc_xprt in xps.
76 void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch
*xps
,
77 struct rpc_xprt
*xprt
)
79 spin_lock(&xps
->xps_lock
);
80 xprt_switch_remove_xprt_locked(xps
, xprt
);
81 spin_unlock(&xps
->xps_lock
);
86 * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch
87 * @xprt: pointer to struct rpc_xprt
88 * @gfp_flags: allocation flags
90 * On success, returns an initialised struct rpc_xprt_switch, containing
91 * the entry xprt. Returns NULL on failure.
93 struct rpc_xprt_switch
*xprt_switch_alloc(struct rpc_xprt
*xprt
,
96 struct rpc_xprt_switch
*xps
;
98 xps
= kmalloc(sizeof(*xps
), gfp_flags
);
100 spin_lock_init(&xps
->xps_lock
);
101 kref_init(&xps
->xps_kref
);
103 INIT_LIST_HEAD(&xps
->xps_xprt_list
);
104 xps
->xps_iter_ops
= &rpc_xprt_iter_singular
;
105 xprt_switch_add_xprt_locked(xps
, xprt
);
111 static void xprt_switch_free_entries(struct rpc_xprt_switch
*xps
)
113 spin_lock(&xps
->xps_lock
);
114 while (!list_empty(&xps
->xps_xprt_list
)) {
115 struct rpc_xprt
*xprt
;
117 xprt
= list_first_entry(&xps
->xps_xprt_list
,
118 struct rpc_xprt
, xprt_switch
);
119 xprt_switch_remove_xprt_locked(xps
, xprt
);
120 spin_unlock(&xps
->xps_lock
);
122 spin_lock(&xps
->xps_lock
);
124 spin_unlock(&xps
->xps_lock
);
127 static void xprt_switch_free(struct kref
*kref
)
129 struct rpc_xprt_switch
*xps
= container_of(kref
,
130 struct rpc_xprt_switch
, xps_kref
);
132 xprt_switch_free_entries(xps
);
133 kfree_rcu(xps
, xps_rcu
);
137 * xprt_switch_get - Return a reference to a rpc_xprt_switch
138 * @xps: pointer to struct rpc_xprt_switch
140 * Returns a reference to xps unless the refcount is already zero.
142 struct rpc_xprt_switch
*xprt_switch_get(struct rpc_xprt_switch
*xps
)
144 if (xps
!= NULL
&& kref_get_unless_zero(&xps
->xps_kref
))
150 * xprt_switch_put - Release a reference to a rpc_xprt_switch
151 * @xps: pointer to struct rpc_xprt_switch
153 * Release the reference to xps, and free it once the refcount is zero.
155 void xprt_switch_put(struct rpc_xprt_switch
*xps
)
158 kref_put(&xps
->xps_kref
, xprt_switch_free
);
162 * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch
163 * @xps: pointer to struct rpc_xprt_switch
165 * Sets a round-robin default policy for iterators acting on xps.
167 void rpc_xprt_switch_set_roundrobin(struct rpc_xprt_switch
*xps
)
169 if (READ_ONCE(xps
->xps_iter_ops
) != &rpc_xprt_iter_roundrobin
)
170 WRITE_ONCE(xps
->xps_iter_ops
, &rpc_xprt_iter_roundrobin
);
174 const struct rpc_xprt_iter_ops
*xprt_iter_ops(const struct rpc_xprt_iter
*xpi
)
176 if (xpi
->xpi_ops
!= NULL
)
178 return rcu_dereference(xpi
->xpi_xpswitch
)->xps_iter_ops
;
182 void xprt_iter_no_rewind(struct rpc_xprt_iter
*xpi
)
187 void xprt_iter_default_rewind(struct rpc_xprt_iter
*xpi
)
189 WRITE_ONCE(xpi
->xpi_cursor
, NULL
);
193 struct rpc_xprt
*xprt_switch_find_first_entry(struct list_head
*head
)
195 return list_first_or_null_rcu(head
, struct rpc_xprt
, xprt_switch
);
199 struct rpc_xprt
*xprt_iter_first_entry(struct rpc_xprt_iter
*xpi
)
201 struct rpc_xprt_switch
*xps
= rcu_dereference(xpi
->xpi_xpswitch
);
205 return xprt_switch_find_first_entry(&xps
->xps_xprt_list
);
209 struct rpc_xprt
*xprt_switch_find_current_entry(struct list_head
*head
,
210 const struct rpc_xprt
*cur
)
212 struct rpc_xprt
*pos
;
214 list_for_each_entry_rcu(pos
, head
, xprt_switch
) {
222 struct rpc_xprt
*xprt_iter_current_entry(struct rpc_xprt_iter
*xpi
)
224 struct rpc_xprt_switch
*xps
= rcu_dereference(xpi
->xpi_xpswitch
);
225 struct list_head
*head
;
229 head
= &xps
->xps_xprt_list
;
230 if (xpi
->xpi_cursor
== NULL
|| xps
->xps_nxprts
< 2)
231 return xprt_switch_find_first_entry(head
);
232 return xprt_switch_find_current_entry(head
, xpi
->xpi_cursor
);
236 struct rpc_xprt
*xprt_switch_find_next_entry(struct list_head
*head
,
237 const struct rpc_xprt
*cur
)
239 struct rpc_xprt
*pos
, *prev
= NULL
;
241 list_for_each_entry_rcu(pos
, head
, xprt_switch
) {
250 struct rpc_xprt
*xprt_switch_set_next_cursor(struct list_head
*head
,
251 struct rpc_xprt
**cursor
,
252 xprt_switch_find_xprt_t find_next
)
254 struct rpc_xprt
*cur
, *pos
, *old
;
256 cur
= READ_ONCE(*cursor
);
259 pos
= find_next(head
, old
);
262 cur
= cmpxchg_relaxed(cursor
, old
, pos
);
270 struct rpc_xprt
*xprt_iter_next_entry_multiple(struct rpc_xprt_iter
*xpi
,
271 xprt_switch_find_xprt_t find_next
)
273 struct rpc_xprt_switch
*xps
= rcu_dereference(xpi
->xpi_xpswitch
);
277 return xprt_switch_set_next_cursor(&xps
->xps_xprt_list
,
283 struct rpc_xprt
*xprt_switch_find_next_entry_roundrobin(struct list_head
*head
,
284 const struct rpc_xprt
*cur
)
286 struct rpc_xprt
*ret
;
288 ret
= xprt_switch_find_next_entry(head
, cur
);
291 return xprt_switch_find_first_entry(head
);
295 struct rpc_xprt
*xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter
*xpi
)
297 return xprt_iter_next_entry_multiple(xpi
,
298 xprt_switch_find_next_entry_roundrobin
);
302 struct rpc_xprt
*xprt_iter_next_entry_all(struct rpc_xprt_iter
*xpi
)
304 return xprt_iter_next_entry_multiple(xpi
, xprt_switch_find_next_entry
);
308 * xprt_iter_rewind - Resets the xprt iterator
309 * @xpi: pointer to rpc_xprt_iter
311 * Resets xpi to ensure that it points to the first entry in the list
315 void xprt_iter_rewind(struct rpc_xprt_iter
*xpi
)
318 xprt_iter_ops(xpi
)->xpi_rewind(xpi
);
322 static void __xprt_iter_init(struct rpc_xprt_iter
*xpi
,
323 struct rpc_xprt_switch
*xps
,
324 const struct rpc_xprt_iter_ops
*ops
)
326 rcu_assign_pointer(xpi
->xpi_xpswitch
, xprt_switch_get(xps
));
327 xpi
->xpi_cursor
= NULL
;
332 * xprt_iter_init - Initialise an xprt iterator
333 * @xpi: pointer to rpc_xprt_iter
334 * @xps: pointer to rpc_xprt_switch
336 * Initialises the iterator to use the default iterator ops
337 * as set in xps. This function is mainly intended for internal
338 * use in the rpc_client.
340 void xprt_iter_init(struct rpc_xprt_iter
*xpi
,
341 struct rpc_xprt_switch
*xps
)
343 __xprt_iter_init(xpi
, xps
, NULL
);
347 * xprt_iter_init_listall - Initialise an xprt iterator
348 * @xpi: pointer to rpc_xprt_iter
349 * @xps: pointer to rpc_xprt_switch
351 * Initialises the iterator to iterate once through the entire list
354 void xprt_iter_init_listall(struct rpc_xprt_iter
*xpi
,
355 struct rpc_xprt_switch
*xps
)
357 __xprt_iter_init(xpi
, xps
, &rpc_xprt_iter_listall
);
361 * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch
362 * @xpi: pointer to rpc_xprt_iter
363 * @xps: pointer to a new rpc_xprt_switch or NULL
365 * Swaps out the existing xpi->xpi_xpswitch with a new value.
367 struct rpc_xprt_switch
*xprt_iter_xchg_switch(struct rpc_xprt_iter
*xpi
,
368 struct rpc_xprt_switch
*newswitch
)
370 struct rpc_xprt_switch __rcu
*oldswitch
;
372 /* Atomically swap out the old xpswitch */
373 oldswitch
= xchg(&xpi
->xpi_xpswitch
, RCU_INITIALIZER(newswitch
));
374 if (newswitch
!= NULL
)
375 xprt_iter_rewind(xpi
);
376 return rcu_dereference_protected(oldswitch
, true);
380 * xprt_iter_destroy - Destroys the xprt iterator
381 * @xpi pointer to rpc_xprt_iter
383 void xprt_iter_destroy(struct rpc_xprt_iter
*xpi
)
385 xprt_switch_put(xprt_iter_xchg_switch(xpi
, NULL
));
389 * xprt_iter_xprt - Returns the rpc_xprt pointed to by the cursor
390 * @xpi: pointer to rpc_xprt_iter
392 * Returns a pointer to the struct rpc_xprt that is currently
393 * pointed to by the cursor.
394 * Caller must be holding rcu_read_lock().
396 struct rpc_xprt
*xprt_iter_xprt(struct rpc_xprt_iter
*xpi
)
398 WARN_ON_ONCE(!rcu_read_lock_held());
399 return xprt_iter_ops(xpi
)->xpi_xprt(xpi
);
403 struct rpc_xprt
*xprt_iter_get_helper(struct rpc_xprt_iter
*xpi
,
404 struct rpc_xprt
*(*fn
)(struct rpc_xprt_iter
*))
406 struct rpc_xprt
*ret
;
413 } while (ret
== NULL
);
418 * xprt_iter_get_xprt - Returns the rpc_xprt pointed to by the cursor
419 * @xpi: pointer to rpc_xprt_iter
421 * Returns a reference to the struct rpc_xprt that is currently
422 * pointed to by the cursor.
424 struct rpc_xprt
*xprt_iter_get_xprt(struct rpc_xprt_iter
*xpi
)
426 struct rpc_xprt
*xprt
;
429 xprt
= xprt_iter_get_helper(xpi
, xprt_iter_ops(xpi
)->xpi_xprt
);
435 * xprt_iter_get_next - Returns the next rpc_xprt following the cursor
436 * @xpi: pointer to rpc_xprt_iter
438 * Returns a reference to the struct rpc_xprt that immediately follows the
439 * entry pointed to by the cursor.
441 struct rpc_xprt
*xprt_iter_get_next(struct rpc_xprt_iter
*xpi
)
443 struct rpc_xprt
*xprt
;
446 xprt
= xprt_iter_get_helper(xpi
, xprt_iter_ops(xpi
)->xpi_next
);
451 /* Policy for always returning the first entry in the rpc_xprt_switch */
453 const struct rpc_xprt_iter_ops rpc_xprt_iter_singular
= {
454 .xpi_rewind
= xprt_iter_no_rewind
,
455 .xpi_xprt
= xprt_iter_first_entry
,
456 .xpi_next
= xprt_iter_first_entry
,
459 /* Policy for round-robin iteration of entries in the rpc_xprt_switch */
461 const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin
= {
462 .xpi_rewind
= xprt_iter_default_rewind
,
463 .xpi_xprt
= xprt_iter_current_entry
,
464 .xpi_next
= xprt_iter_next_entry_roundrobin
,
467 /* Policy for once-through iteration of entries in the rpc_xprt_switch */
469 const struct rpc_xprt_iter_ops rpc_xprt_iter_listall
= {
470 .xpi_rewind
= xprt_iter_default_rewind
,
471 .xpi_xprt
= xprt_iter_current_entry
,
472 .xpi_next
= xprt_iter_next_entry_all
,