]>
Commit | Line | Data |
---|---|---|
d8be8173 PM |
1 | /* |
2 | * Sleepable Read-Copy Update mechanism for mutual exclusion, | |
3 | * tiny version for non-preemptible single-CPU use. | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify | |
6 | * it under the terms of the GNU General Public License as published by | |
7 | * the Free Software Foundation; either version 2 of the License, or | |
8 | * (at your option) any later version. | |
9 | * | |
10 | * This program is distributed in the hope that it will be useful, | |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
13 | * GNU General Public License for more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License | |
16 | * along with this program; if not, you can access it online at | |
17 | * http://www.gnu.org/licenses/gpl-2.0.html. | |
18 | * | |
19 | * Copyright (C) IBM Corporation, 2017 | |
20 | * | |
21 | * Author: Paul McKenney <paulmck@us.ibm.com> | |
22 | */ | |
23 | ||
24 | #include <linux/export.h> | |
25 | #include <linux/mutex.h> | |
26 | #include <linux/preempt.h> | |
27 | #include <linux/rcupdate_wait.h> | |
28 | #include <linux/sched.h> | |
29 | #include <linux/delay.h> | |
30 | #include <linux/srcu.h> | |
31 | ||
32 | #include <linux/rcu_node_tree.h> | |
45753c5f | 33 | #include "rcu_segcblist.h" |
d8be8173 PM |
34 | #include "rcu.h" |
35 | ||
825c5bd2 PM |
36 | int rcu_scheduler_active __read_mostly; |
37 | ||
d8be8173 PM |
38 | static int init_srcu_struct_fields(struct srcu_struct *sp) |
39 | { | |
40 | sp->srcu_lock_nesting[0] = 0; | |
41 | sp->srcu_lock_nesting[1] = 0; | |
42 | init_swait_queue_head(&sp->srcu_wq); | |
2464dd94 PM |
43 | sp->srcu_cb_head = NULL; |
44 | sp->srcu_cb_tail = &sp->srcu_cb_head; | |
d8be8173 PM |
45 | sp->srcu_gp_running = false; |
46 | sp->srcu_gp_waiting = false; | |
47 | sp->srcu_idx = 0; | |
48 | INIT_WORK(&sp->srcu_work, srcu_drive_gp); | |
49 | return 0; | |
50 | } | |
51 | ||
52 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
53 | ||
54 | int __init_srcu_struct(struct srcu_struct *sp, const char *name, | |
55 | struct lock_class_key *key) | |
56 | { | |
57 | /* Don't re-initialize a lock while it is held. */ | |
58 | debug_check_no_locks_freed((void *)sp, sizeof(*sp)); | |
59 | lockdep_init_map(&sp->dep_map, name, key, 0); | |
60 | return init_srcu_struct_fields(sp); | |
61 | } | |
62 | EXPORT_SYMBOL_GPL(__init_srcu_struct); | |
63 | ||
64 | #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
65 | ||
66 | /* | |
67 | * init_srcu_struct - initialize a sleep-RCU structure | |
68 | * @sp: structure to initialize. | |
69 | * | |
70 | * Must invoke this on a given srcu_struct before passing that srcu_struct | |
71 | * to any other function. Each srcu_struct represents a separate domain | |
72 | * of SRCU protection. | |
73 | */ | |
74 | int init_srcu_struct(struct srcu_struct *sp) | |
75 | { | |
76 | return init_srcu_struct_fields(sp); | |
77 | } | |
78 | EXPORT_SYMBOL_GPL(init_srcu_struct); | |
79 | ||
80 | #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ | |
81 | ||
82 | /* | |
83 | * cleanup_srcu_struct - deconstruct a sleep-RCU structure | |
84 | * @sp: structure to clean up. | |
85 | * | |
86 | * Must invoke this after you are finished using a given srcu_struct that | |
87 | * was initialized via init_srcu_struct(), else you leak memory. | |
88 | */ | |
89 | void cleanup_srcu_struct(struct srcu_struct *sp) | |
90 | { | |
91 | WARN_ON(sp->srcu_lock_nesting[0] || sp->srcu_lock_nesting[1]); | |
92 | flush_work(&sp->srcu_work); | |
d8be8173 PM |
93 | WARN_ON(sp->srcu_gp_running); |
94 | WARN_ON(sp->srcu_gp_waiting); | |
2464dd94 PM |
95 | WARN_ON(sp->srcu_cb_head); |
96 | WARN_ON(&sp->srcu_cb_head != sp->srcu_cb_tail); | |
d8be8173 PM |
97 | } |
98 | EXPORT_SYMBOL_GPL(cleanup_srcu_struct); | |
99 | ||
d8be8173 PM |
100 | /* |
101 | * Removes the count for the old reader from the appropriate element of | |
cdf7abc4 | 102 | * the srcu_struct. |
d8be8173 PM |
103 | */ |
104 | void __srcu_read_unlock(struct srcu_struct *sp, int idx) | |
105 | { | |
106 | int newval = sp->srcu_lock_nesting[idx] - 1; | |
107 | ||
108 | WRITE_ONCE(sp->srcu_lock_nesting[idx], newval); | |
109 | if (!newval && READ_ONCE(sp->srcu_gp_waiting)) | |
110 | swake_up(&sp->srcu_wq); | |
111 | } | |
112 | EXPORT_SYMBOL_GPL(__srcu_read_unlock); | |
113 | ||
114 | /* | |
115 | * Workqueue handler to drive one grace period and invoke any callbacks | |
116 | * that become ready as a result. Single-CPU and !PREEMPT operation | |
117 | * means that we get away with murder on synchronization. ;-) | |
118 | */ | |
119 | void srcu_drive_gp(struct work_struct *wp) | |
120 | { | |
121 | int idx; | |
2464dd94 | 122 | struct rcu_head *lh; |
d8be8173 | 123 | struct rcu_head *rhp; |
2464dd94 | 124 | struct srcu_struct *sp; |
d8be8173 PM |
125 | |
126 | sp = container_of(wp, struct srcu_struct, srcu_work); | |
2464dd94 | 127 | if (sp->srcu_gp_running || !READ_ONCE(sp->srcu_cb_head)) |
d8be8173 PM |
128 | return; /* Already running or nothing to do. */ |
129 | ||
2464dd94 | 130 | /* Remove recently arrived callbacks and wait for readers. */ |
d8be8173 | 131 | WRITE_ONCE(sp->srcu_gp_running, true); |
2464dd94 PM |
132 | local_irq_disable(); |
133 | lh = sp->srcu_cb_head; | |
134 | sp->srcu_cb_head = NULL; | |
135 | sp->srcu_cb_tail = &sp->srcu_cb_head; | |
136 | local_irq_enable(); | |
d8be8173 PM |
137 | idx = sp->srcu_idx; |
138 | WRITE_ONCE(sp->srcu_idx, !sp->srcu_idx); | |
139 | WRITE_ONCE(sp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */ | |
140 | swait_event(sp->srcu_wq, !READ_ONCE(sp->srcu_lock_nesting[idx])); | |
141 | WRITE_ONCE(sp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */ | |
2464dd94 PM |
142 | |
143 | /* Invoke the callbacks we removed above. */ | |
144 | while (lh) { | |
145 | rhp = lh; | |
146 | lh = lh->next; | |
147 | local_bh_disable(); | |
148 | rhp->func(rhp); | |
149 | local_bh_enable(); | |
d8be8173 | 150 | } |
d8be8173 PM |
151 | |
152 | /* | |
2464dd94 PM |
153 | * Enable rescheduling, and if there are more callbacks, |
154 | * reschedule ourselves. This can race with a call_srcu() | |
155 | * at interrupt level, but the ->srcu_gp_running checks will | |
156 | * straighten that out. | |
d8be8173 | 157 | */ |
2464dd94 PM |
158 | WRITE_ONCE(sp->srcu_gp_running, false); |
159 | if (READ_ONCE(sp->srcu_cb_head)) | |
d8be8173 PM |
160 | schedule_work(&sp->srcu_work); |
161 | } | |
162 | EXPORT_SYMBOL_GPL(srcu_drive_gp); | |
163 | ||
164 | /* | |
165 | * Enqueue an SRCU callback on the specified srcu_struct structure, | |
166 | * initiating grace-period processing if it is not already running. | |
167 | */ | |
2464dd94 | 168 | void call_srcu(struct srcu_struct *sp, struct rcu_head *rhp, |
d8be8173 PM |
169 | rcu_callback_t func) |
170 | { | |
171 | unsigned long flags; | |
172 | ||
2464dd94 PM |
173 | rhp->func = func; |
174 | rhp->next = NULL; | |
d8be8173 | 175 | local_irq_save(flags); |
2464dd94 PM |
176 | *sp->srcu_cb_tail = rhp; |
177 | sp->srcu_cb_tail = &rhp->next; | |
d8be8173 PM |
178 | local_irq_restore(flags); |
179 | if (!READ_ONCE(sp->srcu_gp_running)) | |
180 | schedule_work(&sp->srcu_work); | |
181 | } | |
182 | EXPORT_SYMBOL_GPL(call_srcu); | |
183 | ||
184 | /* | |
185 | * synchronize_srcu - wait for prior SRCU read-side critical-section completion | |
186 | */ | |
187 | void synchronize_srcu(struct srcu_struct *sp) | |
188 | { | |
189 | struct rcu_synchronize rs; | |
190 | ||
191 | init_rcu_head_on_stack(&rs.head); | |
192 | init_completion(&rs.completion); | |
193 | call_srcu(sp, &rs.head, wakeme_after_rcu); | |
194 | wait_for_completion(&rs.completion); | |
195 | destroy_rcu_head_on_stack(&rs.head); | |
196 | } | |
197 | EXPORT_SYMBOL_GPL(synchronize_srcu); | |
825c5bd2 PM |
198 | |
199 | /* Lockdep diagnostics. */ | |
200 | void __init rcu_scheduler_starting(void) | |
201 | { | |
202 | rcu_scheduler_active = RCU_SCHEDULER_RUNNING; | |
203 | } |