]>
Commit | Line | Data |
---|---|---|
d8be8173 PM |
1 | /* |
2 | * Sleepable Read-Copy Update mechanism for mutual exclusion, | |
3 | * tiny version for non-preemptible single-CPU use. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License as published by | |
7 | * the Free Software Foundation; either version 2 of the License, or | |
8 | * (at your option) any later version. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License | |
16 | * along with this program; if not, you can access it online at | |
17 | * http://www.gnu.org/licenses/gpl-2.0.html. | |
18 | * | |
19 | * Copyright (C) IBM Corporation, 2017 | |
20 | * | |
21 | * Author: Paul McKenney <paulmck@us.ibm.com> | |
22 | */ | |
23 | ||
24 | #include <linux/export.h> | |
25 | #include <linux/mutex.h> | |
26 | #include <linux/preempt.h> | |
27 | #include <linux/rcupdate_wait.h> | |
28 | #include <linux/sched.h> | |
29 | #include <linux/delay.h> | |
30 | #include <linux/srcu.h> | |
31 | ||
32 | #include <linux/rcu_node_tree.h> | |
45753c5f | 33 | #include "rcu_segcblist.h" |
d8be8173 PM |
34 | #include "rcu.h" |
35 | ||
36 | static int init_srcu_struct_fields(struct srcu_struct *sp) | |
37 | { | |
38 | sp->srcu_lock_nesting[0] = 0; | |
39 | sp->srcu_lock_nesting[1] = 0; | |
40 | init_swait_queue_head(&sp->srcu_wq); | |
41 | sp->srcu_gp_seq = 0; | |
42 | rcu_segcblist_init(&sp->srcu_cblist); | |
43 | sp->srcu_gp_running = false; | |
44 | sp->srcu_gp_waiting = false; | |
45 | sp->srcu_idx = 0; | |
46 | INIT_WORK(&sp->srcu_work, srcu_drive_gp); | |
47 | return 0; | |
48 | } | |
49 | ||
50 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
51 | ||
52 | int __init_srcu_struct(struct srcu_struct *sp, const char *name, | |
53 | struct lock_class_key *key) | |
54 | { | |
55 | /* Don't re-initialize a lock while it is held. */ | |
56 | debug_check_no_locks_freed((void *)sp, sizeof(*sp)); | |
57 | lockdep_init_map(&sp->dep_map, name, key, 0); | |
58 | return init_srcu_struct_fields(sp); | |
59 | } | |
60 | EXPORT_SYMBOL_GPL(__init_srcu_struct); | |
61 | ||
62 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
63 | ||
64 | /* | |
65 | * init_srcu_struct - initialize a sleep-RCU structure | |
66 | * @sp: structure to initialize. | |
67 | * | |
68 | * Must invoke this on a given srcu_struct before passing that srcu_struct | |
69 | * to any other function. Each srcu_struct represents a separate domain | |
70 | * of SRCU protection. | |
71 | */ | |
72 | int init_srcu_struct(struct srcu_struct *sp) | |
73 | { | |
74 | return init_srcu_struct_fields(sp); | |
75 | } | |
76 | EXPORT_SYMBOL_GPL(init_srcu_struct); | |
77 | ||
78 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
79 | ||
80 | /* | |
81 | * cleanup_srcu_struct - deconstruct a sleep-RCU structure | |
82 | * @sp: structure to clean up. | |
83 | * | |
84 | * Must invoke this after you are finished using a given srcu_struct that | |
85 | * was initialized via init_srcu_struct(), else you leak memory. | |
86 | */ | |
87 | void cleanup_srcu_struct(struct srcu_struct *sp) | |
88 | { | |
89 | WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]); | |
90 | flush_work(&sp->srcu_work); | |
91 | WARN_ON(rcu_seq_state(sp->srcu_gp_seq)); | |
92 | WARN_ON(sp->srcu_gp_running); | |
93 | WARN_ON(sp->srcu_gp_waiting); | |
94 | WARN_ON(!rcu_segcblist_empty(&sp->srcu_cblist)); | |
95 | } | |
96 | EXPORT_SYMBOL_GPL(cleanup_srcu_struct); | |
97 | ||
98 | /* | |
99 | * Counts the new reader in the appropriate per-CPU element of the | |
100 | * srcu_struct. Must be called from process context. | |
101 | * Returns an index that must be passed to the matching srcu_read_unlock(). | |
102 | */ | |
103 | int __srcu_read_lock(struct srcu_struct *sp) | |
104 | { | |
105 | int idx; | |
106 | ||
107 | idx = READ_ONCE(sp->srcu_idx); | |
108 | WRITE_ONCE(sp->srcu_lock_nesting[idx], sp->srcu_lock_nesting[idx] + 1); | |
109 | return idx; | |
110 | } | |
111 | EXPORT_SYMBOL_GPL(__srcu_read_lock); | |
112 | ||
113 | /* | |
114 | * Removes the count for the old reader from the appropriate element of | |
115 | * the srcu_struct. Must be called from process context. | |
116 | */ | |
117 | void __srcu_read_unlock(struct srcu_struct *sp, int idx) | |
118 | { | |
119 | int newval = sp->srcu_lock_nesting[idx] - 1; | |
120 | ||
121 | WRITE_ONCE(sp->srcu_lock_nesting[idx], newval); | |
122 | if (!newval && READ_ONCE(sp->srcu_gp_waiting)) | |
123 | swake_up(&sp->srcu_wq); | |
124 | } | |
125 | EXPORT_SYMBOL_GPL(__srcu_read_unlock); | |
126 | ||
127 | /* | |
128 | * Workqueue handler to drive one grace period and invoke any callbacks | |
129 | * that become ready as a result. Single-CPU and !PREEMPT operation | |
130 | * means that we get away with murder on synchronization. ;-) | |
131 | */ | |
132 | void srcu_drive_gp(struct work_struct *wp) | |
133 | { | |
134 | int idx; | |
135 | struct rcu_cblist ready_cbs; | |
136 | struct srcu_struct *sp; | |
137 | struct rcu_head *rhp; | |
138 | ||
139 | sp = container_of(wp, struct srcu_struct, srcu_work); | |
140 | if (sp->srcu_gp_running || rcu_segcblist_empty(&sp->srcu_cblist)) | |
141 | return; /* Already running or nothing to do. */ | |
142 | ||
143 | /* Tag recently arrived callbacks and wait for readers. */ | |
144 | WRITE_ONCE(sp->srcu_gp_running, true); | |
145 | rcu_segcblist_accelerate(&sp->srcu_cblist, | |
146 | rcu_seq_snap(&sp->srcu_gp_seq)); | |
147 | rcu_seq_start(&sp->srcu_gp_seq); | |
148 | idx = sp->srcu_idx; | |
149 | WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx); | |
150 | WRITE_ONCE(sp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ | |
151 | swait_event(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx])); | |
152 | WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */ | |
153 | rcu_seq_end(&sp->srcu_gp_seq); | |
154 | ||
155 | /* Update callback list based on GP, and invoke ready callbacks. */ | |
156 | rcu_segcblist_advance(&sp->srcu_cblist, | |
157 | rcu_seq_current(&sp->srcu_gp_seq)); | |
158 | if (rcu_segcblist_ready_cbs(&sp->srcu_cblist)) { | |
159 | rcu_cblist_init(&ready_cbs); | |
160 | local_irq_disable(); | |
161 | rcu_segcblist_extract_done_cbs(&sp->srcu_cblist, &ready_cbs); | |
162 | local_irq_enable(); | |
163 | rhp = rcu_cblist_dequeue(&ready_cbs); | |
164 | for (; rhp != NULL; rhp = rcu_cblist_dequeue(&ready_cbs)) { | |
165 | local_bh_disable(); | |
166 | rhp->func(rhp); | |
167 | local_bh_enable(); | |
168 | } | |
169 | local_irq_disable(); | |
170 | rcu_segcblist_insert_count(&sp->srcu_cblist, &ready_cbs); | |
171 | local_irq_enable(); | |
172 | } | |
173 | WRITE_ONCE(sp->srcu_gp_running, false); | |
174 | ||
175 | /* | |
176 | * If more callbacks, reschedule ourselves. This can race with | |
177 | * a call_srcu() at interrupt level, but the ->srcu_gp_running | |
178 | * checks will straighten that out. | |
179 | */ | |
180 | if (!rcu_segcblist_empty(&sp->srcu_cblist)) | |
181 | schedule_work(&sp->srcu_work); | |
182 | } | |
183 | EXPORT_SYMBOL_GPL(srcu_drive_gp); | |
184 | ||
185 | /* | |
186 | * Enqueue an SRCU callback on the specified srcu_struct structure, | |
187 | * initiating grace-period processing if it is not already running. | |
188 | */ | |
189 | void call_srcu(struct srcu_struct *sp, struct rcu_head *head, | |
190 | rcu_callback_t func) | |
191 | { | |
192 | unsigned long flags; | |
193 | ||
194 | head->func = func; | |
195 | local_irq_save(flags); | |
196 | rcu_segcblist_enqueue(&sp->srcu_cblist, head, false); | |
197 | local_irq_restore(flags); | |
198 | if (!READ_ONCE(sp->srcu_gp_running)) | |
199 | schedule_work(&sp->srcu_work); | |
200 | } | |
201 | EXPORT_SYMBOL_GPL(call_srcu); | |
202 | ||
203 | /* | |
204 | * synchronize_srcu - wait for prior SRCU read-side critical-section completion | |
205 | */ | |
206 | void synchronize_srcu(struct srcu_struct *sp) | |
207 | { | |
208 | struct rcu_synchronize rs; | |
209 | ||
210 | init_rcu_head_on_stack(&rs.head); | |
211 | init_completion(&rs.completion); | |
212 | call_srcu(sp, &rs.head, wakeme_after_rcu); | |
213 | wait_for_completion(&rs.completion); | |
214 | destroy_rcu_head_on_stack(&rs.head); | |
215 | } | |
216 | EXPORT_SYMBOL_GPL(synchronize_srcu); |