]>
Commit | Line | Data |
---|---|---|
96b903f5 | 1 | // SPDX-License-Identifier: GPL-2.0+ |
cc44ca84 ON |
2 | /* |
3 | * RCU-based infrastructure for lightweight reader-writer locking | |
4 | * | |
cc44ca84 ON |
5 | * Copyright (c) 2015, Red Hat, Inc. |
6 | * | |
7 | * Author: Oleg Nesterov <oleg@redhat.com> | |
8 | */ | |
9 | ||
10 | #include <linux/rcu_sync.h> | |
11 | #include <linux/sched.h> | |
12 | ||
89da3b94 | 13 | enum { GP_IDLE = 0, GP_ENTER, GP_PASSED, GP_EXIT, GP_REPLAY }; |
cc44ca84 ON |
14 | |
15 | #define rss_lock gp_wait.lock | |
16 | ||
17 | /** | |
18 | * rcu_sync_init() - Initialize an rcu_sync structure | |
19 | * @rsp: Pointer to rcu_sync structure to be initialized | |
cc44ca84 | 20 | */ |
95bf33b5 | 21 | void rcu_sync_init(struct rcu_sync *rsp) |
cc44ca84 ON |
22 | { |
23 | memset(rsp, 0, sizeof(*rsp)); | |
24 | init_waitqueue_head(&rsp->gp_wait); | |
cc44ca84 ON |
25 | } |
26 | ||
3942a9bd | 27 | /** |
27fdb35f PM |
28 | * rcu_sync_enter_start - Force readers onto slow path for multiple updates |
29 | * @rsp: Pointer to rcu_sync structure to use for synchronization | |
30 | * | |
3942a9bd PZ |
31 | * Must be called after rcu_sync_init() and before first use. |
32 | * | |
33 | * Ensures rcu_sync_is_idle() returns false and rcu_sync_{enter,exit}() | |
34 | * pairs turn into NO-OPs. | |
35 | */ | |
36 | void rcu_sync_enter_start(struct rcu_sync *rsp) | |
37 | { | |
38 | rsp->gp_count++; | |
39 | rsp->gp_state = GP_PASSED; | |
40 | } | |
41 | ||
cc44ca84 | 42 | |
89da3b94 | 43 | static void rcu_sync_func(struct rcu_head *rhp); |
cc44ca84 | 44 | |
89da3b94 ON |
45 | static void rcu_sync_call(struct rcu_sync *rsp) |
46 | { | |
47 | call_rcu(&rsp->cb_head, rcu_sync_func); | |
cc44ca84 ON |
48 | } |
49 | ||
50 | /** | |
51 | * rcu_sync_func() - Callback function managing reader access to fastpath | |
27fdb35f | 52 | * @rhp: Pointer to rcu_head in rcu_sync structure to use for synchronization |
cc44ca84 | 53 | * |
89da3b94 | 54 | * This function is passed to call_rcu() function by rcu_sync_enter() and |
cc44ca84 | 55 | * rcu_sync_exit(), so that it is invoked after a grace period following the |
89da3b94 ON |
56 | * that invocation of enter/exit. |
57 | * | |
58 | * If it is called by rcu_sync_enter() it signals that all the readers were | |
59 | * switched onto slow path. | |
60 | * | |
61 | * If it is called by rcu_sync_exit() it takes action based on events that | |
cc44ca84 ON |
62 | * have taken place in the meantime, so that closely spaced rcu_sync_enter() |
63 | * and rcu_sync_exit() pairs need not wait for a grace period. | |
64 | * | |
65 | * If another rcu_sync_enter() is invoked before the grace period | |
66 | * ended, reset state to allow the next rcu_sync_exit() to let the | |
67 | * readers back onto their fastpaths (after a grace period). If both | |
68 | * another rcu_sync_enter() and its matching rcu_sync_exit() are invoked | |
69 | * before the grace period ended, re-invoke call_rcu() on behalf of that | |
70 | * rcu_sync_exit(). Otherwise, set all state back to idle so that readers | |
71 | * can again use their fastpaths. | |
72 | */ | |
27fdb35f | 73 | static void rcu_sync_func(struct rcu_head *rhp) |
cc44ca84 | 74 | { |
27fdb35f | 75 | struct rcu_sync *rsp = container_of(rhp, struct rcu_sync, cb_head); |
cc44ca84 ON |
76 | unsigned long flags; |
77 | ||
89da3b94 ON |
78 | WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE); |
79 | WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED); | |
cc44ca84 ON |
80 | |
81 | spin_lock_irqsave(&rsp->rss_lock, flags); | |
82 | if (rsp->gp_count) { | |
83 | /* | |
89da3b94 | 84 | * We're at least a GP after the GP_IDLE->GP_ENTER transition. |
cc44ca84 | 85 | */ |
89da3b94 ON |
86 | WRITE_ONCE(rsp->gp_state, GP_PASSED); |
87 | wake_up_locked(&rsp->gp_wait); | |
88 | } else if (rsp->gp_state == GP_REPLAY) { | |
cc44ca84 | 89 | /* |
89da3b94 ON |
90 | * A new rcu_sync_exit() has happened; requeue the callback to |
91 | * catch a later GP. | |
cc44ca84 | 92 | */ |
89da3b94 ON |
93 | WRITE_ONCE(rsp->gp_state, GP_EXIT); |
94 | rcu_sync_call(rsp); | |
cc44ca84 ON |
95 | } else { |
96 | /* | |
89da3b94 ON |
97 | * We're at least a GP after the last rcu_sync_exit(); eveybody |
98 | * will now have observed the write side critical section. | |
99 | * Let 'em rip!. | |
cc44ca84 | 100 | */ |
89da3b94 | 101 | WRITE_ONCE(rsp->gp_state, GP_IDLE); |
cc44ca84 ON |
102 | } |
103 | spin_unlock_irqrestore(&rsp->rss_lock, flags); | |
104 | } | |
105 | ||
106 | /** | |
89da3b94 ON |
107 | * rcu_sync_enter() - Force readers onto slowpath |
108 | * @rsp: Pointer to rcu_sync structure to use for synchronization | |
109 | * | |
110 | * This function is used by updaters who need readers to make use of | |
111 | * a slowpath during the update. After this function returns, all | |
112 | * subsequent calls to rcu_sync_is_idle() will return false, which | |
113 | * tells readers to stay off their fastpaths. A later call to | |
114 | * rcu_sync_exit() re-enables reader slowpaths. | |
115 | * | |
116 | * When called in isolation, rcu_sync_enter() must wait for a grace | |
117 | * period, however, closely spaced calls to rcu_sync_enter() can | |
118 | * optimize away the grace-period wait via a state machine implemented | |
119 | * by rcu_sync_enter(), rcu_sync_exit(), and rcu_sync_func(). | |
120 | */ | |
121 | void rcu_sync_enter(struct rcu_sync *rsp) | |
122 | { | |
123 | int gp_state; | |
124 | ||
125 | spin_lock_irq(&rsp->rss_lock); | |
126 | gp_state = rsp->gp_state; | |
127 | if (gp_state == GP_IDLE) { | |
128 | WRITE_ONCE(rsp->gp_state, GP_ENTER); | |
129 | WARN_ON_ONCE(rsp->gp_count); | |
130 | /* | |
131 | * Note that we could simply do rcu_sync_call(rsp) here and | |
132 | * avoid the "if (gp_state == GP_IDLE)" block below. | |
133 | * | |
134 | * However, synchronize_rcu() can be faster if rcu_expedited | |
135 | * or rcu_blocking_is_gp() is true. | |
136 | * | |
137 | * Another reason is that we can't wait for rcu callback if | |
138 | * we are called at early boot time but this shouldn't happen. | |
139 | */ | |
140 | } | |
141 | rsp->gp_count++; | |
142 | spin_unlock_irq(&rsp->rss_lock); | |
143 | ||
144 | if (gp_state == GP_IDLE) { | |
145 | /* | |
146 | * See the comment above, this simply does the "synchronous" | |
147 | * call_rcu(rcu_sync_func) which does GP_ENTER -> GP_PASSED. | |
148 | */ | |
149 | synchronize_rcu(); | |
150 | rcu_sync_func(&rsp->cb_head); | |
151 | /* Not really needed, wait_event() would see GP_PASSED. */ | |
152 | return; | |
153 | } | |
154 | ||
155 | wait_event(rsp->gp_wait, READ_ONCE(rsp->gp_state) >= GP_PASSED); | |
156 | } | |
157 | ||
158 | /** | |
159 | * rcu_sync_exit() - Allow readers back onto fast path after grace period | |
cc44ca84 ON |
160 | * @rsp: Pointer to rcu_sync structure to use for synchronization |
161 | * | |
162 | * This function is used by updaters who have completed, and can therefore | |
163 | * now allow readers to make use of their fastpaths after a grace period | |
164 | * has elapsed. After this grace period has completed, all subsequent | |
165 | * calls to rcu_sync_is_idle() will return true, which tells readers that | |
166 | * they can once again use their fastpaths. | |
167 | */ | |
168 | void rcu_sync_exit(struct rcu_sync *rsp) | |
169 | { | |
89da3b94 ON |
170 | WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_IDLE); |
171 | WARN_ON_ONCE(READ_ONCE(rsp->gp_count) == 0); | |
172 | ||
cc44ca84 ON |
173 | spin_lock_irq(&rsp->rss_lock); |
174 | if (!--rsp->gp_count) { | |
89da3b94 ON |
175 | if (rsp->gp_state == GP_PASSED) { |
176 | WRITE_ONCE(rsp->gp_state, GP_EXIT); | |
177 | rcu_sync_call(rsp); | |
178 | } else if (rsp->gp_state == GP_EXIT) { | |
179 | WRITE_ONCE(rsp->gp_state, GP_REPLAY); | |
cc44ca84 ON |
180 | } |
181 | } | |
182 | spin_unlock_irq(&rsp->rss_lock); | |
183 | } | |
07899a6e ON |
184 | |
185 | /** | |
186 | * rcu_sync_dtor() - Clean up an rcu_sync structure | |
187 | * @rsp: Pointer to rcu_sync structure to be cleaned up | |
188 | */ | |
189 | void rcu_sync_dtor(struct rcu_sync *rsp) | |
190 | { | |
89da3b94 | 191 | int gp_state; |
07899a6e | 192 | |
89da3b94 ON |
193 | WARN_ON_ONCE(READ_ONCE(rsp->gp_count)); |
194 | WARN_ON_ONCE(READ_ONCE(rsp->gp_state) == GP_PASSED); | |
07899a6e ON |
195 | |
196 | spin_lock_irq(&rsp->rss_lock); | |
89da3b94 ON |
197 | if (rsp->gp_state == GP_REPLAY) |
198 | WRITE_ONCE(rsp->gp_state, GP_EXIT); | |
199 | gp_state = rsp->gp_state; | |
07899a6e ON |
200 | spin_unlock_irq(&rsp->rss_lock); |
201 | ||
89da3b94 | 202 | if (gp_state != GP_IDLE) { |
95bf33b5 | 203 | rcu_barrier(); |
89da3b94 | 204 | WARN_ON_ONCE(rsp->gp_state != GP_IDLE); |
07899a6e ON |
205 | } |
206 | } |