]>
Commit | Line | Data |
---|---|---|
7911747b PB |
1 | #ifndef QEMU_RCU_H |
2 | #define QEMU_RCU_H | |
3 | ||
4 | /* | |
5 | * urcu-mb.h | |
6 | * | |
7 | * Userspace RCU header with explicit memory barrier. | |
8 | * | |
9 | * This library is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU Lesser General Public | |
11 | * License as published by the Free Software Foundation; either | |
12 | * version 2.1 of the License, or (at your option) any later version. | |
13 | * | |
14 | * This library is distributed in the hope that it will be useful, | |
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
17 | * Lesser General Public License for more details. | |
18 | * | |
19 | * You should have received a copy of the GNU Lesser General Public | |
20 | * License along with this library; if not, write to the Free Software | |
21 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
22 | * | |
23 | * IBM's contributions to this file may be relicensed under LGPLv2 or later. | |
24 | */ | |
25 | ||
7911747b | 26 | |
7911747b PB |
27 | #include "qemu/thread.h" |
28 | #include "qemu/queue.h" | |
29 | #include "qemu/atomic.h" | |
ef149763 | 30 | #include "qemu/notify.h" |
c8d3877e | 31 | #include "qemu/sys_membarrier.h" |
17c78154 | 32 | #include "qemu/coroutine-tls.h" |
7911747b PB |
33 | |
34 | #ifdef __cplusplus | |
35 | extern "C" { | |
36 | #endif | |
37 | ||
38 | /* | |
39 | * Important ! | |
40 | * | |
41 | * Each thread containing read-side critical sections must be registered | |
42 | * with rcu_register_thread() before calling rcu_read_lock(). | |
43 | * rcu_unregister_thread() should be called before the thread exits. | |
44 | */ | |
45 | ||
46 | #ifdef DEBUG_RCU | |
47 | #define rcu_assert(args...) assert(args) | |
48 | #else | |
49 | #define rcu_assert(args...) | |
50 | #endif | |
51 | ||
52 | /* | |
53 | * Global quiescent period counter with low-order bits unused. | |
54 | * Using a int rather than a char to eliminate false register dependencies | |
55 | * causing stalls on some architectures. | |
56 | */ | |
57 | extern unsigned long rcu_gp_ctr; | |
58 | ||
59 | extern QemuEvent rcu_gp_event; | |
60 | ||
61 | struct rcu_reader_data { | |
62 | /* Data used by both reader and synchronize_rcu() */ | |
63 | unsigned long ctr; | |
64 | bool waiting; | |
65 | ||
d62cb4f2 PB |
66 | /* Data used by reader only */ |
67 | unsigned depth; | |
68 | ||
492e1ca9 | 69 | /* Data used for registry, protected by rcu_registry_lock */ |
7911747b | 70 | QLIST_ENTRY(rcu_reader_data) node; |
ef149763 GK |
71 | |
72 | /* | |
73 | * NotifierList used to force an RCU grace period. Accessed under | |
74 | * rcu_registry_lock. Note that the notifier is called _outside_ | |
75 | * the thread! | |
76 | */ | |
77 | NotifierList force_rcu; | |
7911747b PB |
78 | }; |
79 | ||
17c78154 | 80 | QEMU_DECLARE_CO_TLS(struct rcu_reader_data, rcu_reader) |
7911747b PB |
81 | |
82 | static inline void rcu_read_lock(void) | |
83 | { | |
17c78154 | 84 | struct rcu_reader_data *p_rcu_reader = get_ptr_rcu_reader(); |
d62cb4f2 PB |
85 | unsigned ctr; |
86 | ||
87 | if (p_rcu_reader->depth++ > 0) { | |
88 | return; | |
89 | } | |
7911747b | 90 | |
d73415a3 SH |
91 | ctr = qatomic_read(&rcu_gp_ctr); |
92 | qatomic_set(&p_rcu_reader->ctr, ctr); | |
77a8b846 PB |
93 | |
94 | /* Write p_rcu_reader->ctr before reading RCU-protected pointers. */ | |
c8d3877e | 95 | smp_mb_placeholder(); |
7911747b PB |
96 | } |
97 | ||
98 | static inline void rcu_read_unlock(void) | |
99 | { | |
17c78154 | 100 | struct rcu_reader_data *p_rcu_reader = get_ptr_rcu_reader(); |
7911747b | 101 | |
d62cb4f2 PB |
102 | assert(p_rcu_reader->depth != 0); |
103 | if (--p_rcu_reader->depth > 0) { | |
104 | return; | |
105 | } | |
106 | ||
77a8b846 PB |
107 | /* Ensure that the critical section is seen to precede the |
108 | * store to p_rcu_reader->ctr. Together with the following | |
c8d3877e | 109 | * smp_mb_placeholder(), this ensures writes to p_rcu_reader->ctr |
77a8b846 PB |
110 | * are sequentially consistent. |
111 | */ | |
d73415a3 | 112 | qatomic_store_release(&p_rcu_reader->ctr, 0); |
77a8b846 PB |
113 | |
114 | /* Write p_rcu_reader->ctr before reading p_rcu_reader->waiting. */ | |
c8d3877e | 115 | smp_mb_placeholder(); |
d73415a3 SH |
116 | if (unlikely(qatomic_read(&p_rcu_reader->waiting))) { |
117 | qatomic_set(&p_rcu_reader->waiting, false); | |
7911747b PB |
118 | qemu_event_set(&rcu_gp_event); |
119 | } | |
120 | } | |
121 | ||
122 | extern void synchronize_rcu(void); | |
123 | ||
124 | /* | |
125 | * Reader thread registration. | |
126 | */ | |
127 | extern void rcu_register_thread(void); | |
128 | extern void rcu_unregister_thread(void); | |
73c6e401 PB |
129 | |
130 | /* | |
131 | * Support for fork(). fork() support is enabled at startup. | |
132 | */ | |
133 | extern void rcu_enable_atfork(void); | |
134 | extern void rcu_disable_atfork(void); | |
7911747b | 135 | |
26387f86 PB |
136 | struct rcu_head; |
137 | typedef void RCUCBFunc(struct rcu_head *head); | |
138 | ||
139 | struct rcu_head { | |
140 | struct rcu_head *next; | |
141 | RCUCBFunc *func; | |
142 | }; | |
143 | ||
144 | extern void call_rcu1(struct rcu_head *head, RCUCBFunc *func); | |
d816614c | 145 | extern void drain_call_rcu(void); |
26387f86 PB |
146 | |
147 | /* The operands of the minus operator must have the same type, | |
148 | * which must be the one that we specify in the cast. | |
149 | */ | |
150 | #define call_rcu(head, func, field) \ | |
151 | call_rcu1(({ \ | |
152 | char __attribute__((unused)) \ | |
153 | offset_must_be_zero[-offsetof(typeof(*(head)), field)], \ | |
154 | func_type_invalid = (func) - (void (*)(typeof(head)))(func); \ | |
155 | &(head)->field; \ | |
156 | }), \ | |
157 | (RCUCBFunc *)(func)) | |
158 | ||
439c5e02 PB |
159 | #define g_free_rcu(obj, field) \ |
160 | call_rcu1(({ \ | |
161 | char __attribute__((unused)) \ | |
162 | offset_must_be_zero[-offsetof(typeof(*(obj)), field)]; \ | |
163 | &(obj)->field; \ | |
164 | }), \ | |
165 | (RCUCBFunc *)g_free); | |
166 | ||
5626f8c6 DDAG |
167 | typedef void RCUReadAuto; |
168 | static inline RCUReadAuto *rcu_read_auto_lock(void) | |
169 | { | |
170 | rcu_read_lock(); | |
171 | /* Anything non-NULL causes the cleanup function to be called */ | |
172 | return (void *)(uintptr_t)0x1; | |
173 | } | |
174 | ||
175 | static inline void rcu_read_auto_unlock(RCUReadAuto *r) | |
176 | { | |
177 | rcu_read_unlock(); | |
178 | } | |
179 | ||
180 | G_DEFINE_AUTOPTR_CLEANUP_FUNC(RCUReadAuto, rcu_read_auto_unlock) | |
181 | ||
182 | #define WITH_RCU_READ_LOCK_GUARD() \ | |
56f21718 | 183 | WITH_RCU_READ_LOCK_GUARD_(glue(_rcu_read_auto, __COUNTER__)) |
5626f8c6 DDAG |
184 | |
185 | #define WITH_RCU_READ_LOCK_GUARD_(var) \ | |
186 | for (g_autoptr(RCUReadAuto) var = rcu_read_auto_lock(); \ | |
187 | (var); rcu_read_auto_unlock(var), (var) = NULL) | |
188 | ||
189 | #define RCU_READ_LOCK_GUARD() \ | |
190 | g_autoptr(RCUReadAuto) _rcu_read_auto __attribute__((unused)) = rcu_read_auto_lock() | |
191 | ||
ef149763 GK |
192 | /* |
193 | * Force-RCU notifiers tell readers that they should exit their | |
194 | * read-side critical section. | |
195 | */ | |
196 | void rcu_add_force_rcu_notifier(Notifier *n); | |
197 | void rcu_remove_force_rcu_notifier(Notifier *n); | |
198 | ||
7911747b PB |
199 | #ifdef __cplusplus |
200 | } | |
201 | #endif | |
202 | ||
203 | #endif /* QEMU_RCU_H */ |