]>
Commit | Line | Data |
---|---|---|
341774fe MD |
1 | #ifndef QEMU_RCU_QUEUE_H |
2 | #define QEMU_RCU_QUEUE_H | |
3 | ||
4 | /* | |
5 | * rcu_queue.h | |
6 | * | |
7 | * RCU-friendly versions of the queue.h primitives. | |
8 | * | |
9 | * This library is free software; you can redistribute it and/or | |
10 | * modify it under the terms of the GNU Lesser General Public | |
11 | * License as published by the Free Software Foundation; either | |
12 | * version 2.1 of the License, or (at your option) any later version. | |
13 | * | |
14 | * This library is distributed in the hope that it will be useful, | |
15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
17 | * Lesser General Public License for more details. | |
18 | * | |
19 | * You should have received a copy of the GNU Lesser General Public | |
20 | * License along with this library; if not, write to the Free Software | |
21 | * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | |
22 | * | |
23 | * Copyright (c) 2013 Mike D. Day, IBM Corporation. | |
24 | * | |
25 | * IBM's contributions to this file may be relicensed under LGPLv2 or later. | |
26 | */ | |
27 | ||
28 | #include "qemu/queue.h" | |
29 | #include "qemu/atomic.h" | |
30 | ||
341774fe MD |
31 | /* |
32 | * List access methods. | |
33 | */ | |
d73415a3 SH |
34 | #define QLIST_EMPTY_RCU(head) (qatomic_read(&(head)->lh_first) == NULL) |
35 | #define QLIST_FIRST_RCU(head) (qatomic_rcu_read(&(head)->lh_first)) | |
36 | #define QLIST_NEXT_RCU(elm, field) (qatomic_rcu_read(&(elm)->field.le_next)) | |
341774fe MD |
37 | |
38 | /* | |
39 | * List functions. | |
40 | */ | |
41 | ||
42 | ||
43 | /* | |
d73415a3 | 44 | * The difference between qatomic_read/set and qatomic_rcu_read/set |
341774fe MD |
45 | * is in the including of a read/write memory barrier to the volatile |
46 | * access. atomic_rcu_* macros include the memory barrier, the | |
47 | * plain atomic macros do not. Therefore, it should be correct to | |
48 | * issue a series of reads or writes to the same element using only | |
49 | * the atomic_* macro, until the last read or write, which should be | |
50 | * atomic_rcu_* to introduce a read or write memory barrier as | |
51 | * appropriate. | |
52 | */ | |
53 | ||
54 | /* Upon publication of the listelm->next value, list readers | |
55 | * will see the new node when following next pointers from | |
56 | * antecedent nodes, but may not see the new node when following | |
57 | * prev pointers from subsequent nodes until after the RCU grace | |
58 | * period expires. | |
59 | * see linux/include/rculist.h __list_add_rcu(new, prev, next) | |
60 | */ | |
61 | #define QLIST_INSERT_AFTER_RCU(listelm, elm, field) do { \ | |
62 | (elm)->field.le_next = (listelm)->field.le_next; \ | |
63 | (elm)->field.le_prev = &(listelm)->field.le_next; \ | |
d73415a3 | 64 | qatomic_rcu_set(&(listelm)->field.le_next, (elm)); \ |
341774fe MD |
65 | if ((elm)->field.le_next != NULL) { \ |
66 | (elm)->field.le_next->field.le_prev = \ | |
67 | &(elm)->field.le_next; \ | |
68 | } \ | |
69 | } while (/*CONSTCOND*/0) | |
70 | ||
71 | /* Upon publication of the listelm->prev->next value, list | |
72 | * readers will see the new element when following prev pointers | |
73 | * from subsequent elements, but may not see the new element | |
74 | * when following next pointers from antecedent elements | |
75 | * until after the RCU grace period expires. | |
76 | */ | |
77 | #define QLIST_INSERT_BEFORE_RCU(listelm, elm, field) do { \ | |
78 | (elm)->field.le_prev = (listelm)->field.le_prev; \ | |
79 | (elm)->field.le_next = (listelm); \ | |
d73415a3 | 80 | qatomic_rcu_set((listelm)->field.le_prev, (elm)); \ |
341774fe MD |
81 | (listelm)->field.le_prev = &(elm)->field.le_next; \ |
82 | } while (/*CONSTCOND*/0) | |
83 | ||
84 | /* Upon publication of the head->first value, list readers | |
85 | * will see the new element when following the head, but may | |
86 | * not see the new element when following prev pointers from | |
87 | * subsequent elements until after the RCU grace period has | |
88 | * expired. | |
89 | */ | |
90 | #define QLIST_INSERT_HEAD_RCU(head, elm, field) do { \ | |
91 | (elm)->field.le_prev = &(head)->lh_first; \ | |
92 | (elm)->field.le_next = (head)->lh_first; \ | |
d73415a3 | 93 | qatomic_rcu_set((&(head)->lh_first), (elm)); \ |
341774fe MD |
94 | if ((elm)->field.le_next != NULL) { \ |
95 | (elm)->field.le_next->field.le_prev = \ | |
96 | &(elm)->field.le_next; \ | |
97 | } \ | |
98 | } while (/*CONSTCOND*/0) | |
99 | ||
100 | ||
101 | /* prior to publication of the elm->prev->next value, some list | |
102 | * readers may still see the removed element when following | |
103 | * the antecedent's next pointer. | |
104 | */ | |
105 | #define QLIST_REMOVE_RCU(elm, field) do { \ | |
106 | if ((elm)->field.le_next != NULL) { \ | |
107 | (elm)->field.le_next->field.le_prev = \ | |
108 | (elm)->field.le_prev; \ | |
109 | } \ | |
d73415a3 | 110 | qatomic_set((elm)->field.le_prev, (elm)->field.le_next); \ |
341774fe MD |
111 | } while (/*CONSTCOND*/0) |
112 | ||
113 | /* List traversal must occur within an RCU critical section. */ | |
114 | #define QLIST_FOREACH_RCU(var, head, field) \ | |
d73415a3 | 115 | for ((var) = qatomic_rcu_read(&(head)->lh_first); \ |
341774fe | 116 | (var); \ |
d73415a3 | 117 | (var) = qatomic_rcu_read(&(var)->field.le_next)) |
341774fe MD |
118 | |
119 | /* List traversal must occur within an RCU critical section. */ | |
120 | #define QLIST_FOREACH_SAFE_RCU(var, head, field, next_var) \ | |
d73415a3 | 121 | for ((var) = (qatomic_rcu_read(&(head)->lh_first)); \ |
341774fe | 122 | (var) && \ |
d73415a3 | 123 | ((next_var) = qatomic_rcu_read(&(var)->field.le_next), 1); \ |
341774fe MD |
124 | (var) = (next_var)) |
125 | ||
13d8ef7d EC |
126 | /* |
127 | * RCU simple queue | |
128 | */ | |
129 | ||
130 | /* Simple queue access methods */ | |
d73415a3 SH |
131 | #define QSIMPLEQ_EMPTY_RCU(head) \ |
132 | (qatomic_read(&(head)->sqh_first) == NULL) | |
133 | #define QSIMPLEQ_FIRST_RCU(head) qatomic_rcu_read(&(head)->sqh_first) | |
134 | #define QSIMPLEQ_NEXT_RCU(elm, field) qatomic_rcu_read(&(elm)->field.sqe_next) | |
13d8ef7d EC |
135 | |
136 | /* Simple queue functions */ | |
137 | #define QSIMPLEQ_INSERT_HEAD_RCU(head, elm, field) do { \ | |
138 | (elm)->field.sqe_next = (head)->sqh_first; \ | |
139 | if ((elm)->field.sqe_next == NULL) { \ | |
140 | (head)->sqh_last = &(elm)->field.sqe_next; \ | |
141 | } \ | |
d73415a3 | 142 | qatomic_rcu_set(&(head)->sqh_first, (elm)); \ |
13d8ef7d EC |
143 | } while (/*CONSTCOND*/0) |
144 | ||
145 | #define QSIMPLEQ_INSERT_TAIL_RCU(head, elm, field) do { \ | |
146 | (elm)->field.sqe_next = NULL; \ | |
d73415a3 | 147 | qatomic_rcu_set((head)->sqh_last, (elm)); \ |
13d8ef7d EC |
148 | (head)->sqh_last = &(elm)->field.sqe_next; \ |
149 | } while (/*CONSTCOND*/0) | |
150 | ||
151 | #define QSIMPLEQ_INSERT_AFTER_RCU(head, listelm, elm, field) do { \ | |
152 | (elm)->field.sqe_next = (listelm)->field.sqe_next; \ | |
153 | if ((elm)->field.sqe_next == NULL) { \ | |
154 | (head)->sqh_last = &(elm)->field.sqe_next; \ | |
155 | } \ | |
d73415a3 | 156 | qatomic_rcu_set(&(listelm)->field.sqe_next, (elm)); \ |
13d8ef7d EC |
157 | } while (/*CONSTCOND*/0) |
158 | ||
159 | #define QSIMPLEQ_REMOVE_HEAD_RCU(head, field) do { \ | |
d73415a3 | 160 | qatomic_set(&(head)->sqh_first, (head)->sqh_first->field.sqe_next);\ |
13d8ef7d EC |
161 | if ((head)->sqh_first == NULL) { \ |
162 | (head)->sqh_last = &(head)->sqh_first; \ | |
163 | } \ | |
164 | } while (/*CONSTCOND*/0) | |
165 | ||
166 | #define QSIMPLEQ_REMOVE_RCU(head, elm, type, field) do { \ | |
167 | if ((head)->sqh_first == (elm)) { \ | |
168 | QSIMPLEQ_REMOVE_HEAD_RCU((head), field); \ | |
169 | } else { \ | |
170 | struct type *curr = (head)->sqh_first; \ | |
171 | while (curr->field.sqe_next != (elm)) { \ | |
172 | curr = curr->field.sqe_next; \ | |
173 | } \ | |
d73415a3 | 174 | qatomic_set(&curr->field.sqe_next, \ |
13d8ef7d EC |
175 | curr->field.sqe_next->field.sqe_next); \ |
176 | if (curr->field.sqe_next == NULL) { \ | |
177 | (head)->sqh_last = &(curr)->field.sqe_next; \ | |
178 | } \ | |
179 | } \ | |
180 | } while (/*CONSTCOND*/0) | |
181 | ||
182 | #define QSIMPLEQ_FOREACH_RCU(var, head, field) \ | |
d73415a3 | 183 | for ((var) = qatomic_rcu_read(&(head)->sqh_first); \ |
13d8ef7d | 184 | (var); \ |
d73415a3 | 185 | (var) = qatomic_rcu_read(&(var)->field.sqe_next)) |
13d8ef7d EC |
186 | |
187 | #define QSIMPLEQ_FOREACH_SAFE_RCU(var, head, field, next) \ | |
d73415a3 SH |
188 | for ((var) = qatomic_rcu_read(&(head)->sqh_first); \ |
189 | (var) && ((next) = qatomic_rcu_read(&(var)->field.sqe_next), 1);\ | |
13d8ef7d EC |
190 | (var) = (next)) |
191 | ||
945d9c75 EC |
192 | /* |
193 | * RCU tail queue | |
194 | */ | |
195 | ||
196 | /* Tail queue access methods */ | |
d73415a3 SH |
197 | #define QTAILQ_EMPTY_RCU(head) (qatomic_read(&(head)->tqh_first) == NULL) |
198 | #define QTAILQ_FIRST_RCU(head) qatomic_rcu_read(&(head)->tqh_first) | |
199 | #define QTAILQ_NEXT_RCU(elm, field) qatomic_rcu_read(&(elm)->field.tqe_next) | |
945d9c75 EC |
200 | |
201 | /* Tail queue functions */ | |
202 | #define QTAILQ_INSERT_HEAD_RCU(head, elm, field) do { \ | |
203 | (elm)->field.tqe_next = (head)->tqh_first; \ | |
204 | if ((elm)->field.tqe_next != NULL) { \ | |
7274f01b PB |
205 | (head)->tqh_first->field.tqe_circ.tql_prev = \ |
206 | &(elm)->field.tqe_circ; \ | |
945d9c75 | 207 | } else { \ |
7274f01b | 208 | (head)->tqh_circ.tql_prev = &(elm)->field.tqe_circ; \ |
945d9c75 | 209 | } \ |
d73415a3 | 210 | qatomic_rcu_set(&(head)->tqh_first, (elm)); \ |
7274f01b | 211 | (elm)->field.tqe_circ.tql_prev = &(head)->tqh_circ; \ |
945d9c75 EC |
212 | } while (/*CONSTCOND*/0) |
213 | ||
7274f01b PB |
214 | #define QTAILQ_INSERT_TAIL_RCU(head, elm, field) do { \ |
215 | (elm)->field.tqe_next = NULL; \ | |
216 | (elm)->field.tqe_circ.tql_prev = (head)->tqh_circ.tql_prev; \ | |
d73415a3 | 217 | qatomic_rcu_set(&(head)->tqh_circ.tql_prev->tql_next, (elm)); \ |
7274f01b | 218 | (head)->tqh_circ.tql_prev = &(elm)->field.tqe_circ; \ |
945d9c75 EC |
219 | } while (/*CONSTCOND*/0) |
220 | ||
221 | #define QTAILQ_INSERT_AFTER_RCU(head, listelm, elm, field) do { \ | |
222 | (elm)->field.tqe_next = (listelm)->field.tqe_next; \ | |
223 | if ((elm)->field.tqe_next != NULL) { \ | |
7274f01b PB |
224 | (elm)->field.tqe_next->field.tqe_circ.tql_prev = \ |
225 | &(elm)->field.tqe_circ; \ | |
945d9c75 | 226 | } else { \ |
7274f01b | 227 | (head)->tqh_circ.tql_prev = &(elm)->field.tqe_circ; \ |
945d9c75 | 228 | } \ |
d73415a3 | 229 | qatomic_rcu_set(&(listelm)->field.tqe_next, (elm)); \ |
7274f01b | 230 | (elm)->field.tqe_circ.tql_prev = &(listelm)->field.tqe_circ; \ |
945d9c75 EC |
231 | } while (/*CONSTCOND*/0) |
232 | ||
7274f01b PB |
233 | #define QTAILQ_INSERT_BEFORE_RCU(listelm, elm, field) do { \ |
234 | (elm)->field.tqe_circ.tql_prev = (listelm)->field.tqe_circ.tql_prev; \ | |
235 | (elm)->field.tqe_next = (listelm); \ | |
d73415a3 | 236 | qatomic_rcu_set(&(listelm)->field.tqe_circ.tql_prev->tql_next, (elm));\ |
7274f01b PB |
237 | (listelm)->field.tqe_circ.tql_prev = &(elm)->field.tqe_circ; \ |
238 | } while (/*CONSTCOND*/0) | |
945d9c75 EC |
239 | |
240 | #define QTAILQ_REMOVE_RCU(head, elm, field) do { \ | |
241 | if (((elm)->field.tqe_next) != NULL) { \ | |
7274f01b PB |
242 | (elm)->field.tqe_next->field.tqe_circ.tql_prev = \ |
243 | (elm)->field.tqe_circ.tql_prev; \ | |
945d9c75 | 244 | } else { \ |
7274f01b | 245 | (head)->tqh_circ.tql_prev = (elm)->field.tqe_circ.tql_prev; \ |
945d9c75 | 246 | } \ |
d73415a3 SH |
247 | qatomic_set(&(elm)->field.tqe_circ.tql_prev->tql_next, \ |
248 | (elm)->field.tqe_next); \ | |
7274f01b | 249 | (elm)->field.tqe_circ.tql_prev = NULL; \ |
945d9c75 EC |
250 | } while (/*CONSTCOND*/0) |
251 | ||
252 | #define QTAILQ_FOREACH_RCU(var, head, field) \ | |
d73415a3 | 253 | for ((var) = qatomic_rcu_read(&(head)->tqh_first); \ |
945d9c75 | 254 | (var); \ |
d73415a3 | 255 | (var) = qatomic_rcu_read(&(var)->field.tqe_next)) |
945d9c75 EC |
256 | |
257 | #define QTAILQ_FOREACH_SAFE_RCU(var, head, field, next) \ | |
d73415a3 SH |
258 | for ((var) = qatomic_rcu_read(&(head)->tqh_first); \ |
259 | (var) && ((next) = qatomic_rcu_read(&(var)->field.tqe_next), 1);\ | |
945d9c75 EC |
260 | (var) = (next)) |
261 | ||
8c3570e3 PB |
262 | /* |
263 | * RCU singly-linked list | |
264 | */ | |
265 | ||
266 | /* Singly-linked list access methods */ | |
d73415a3 SH |
267 | #define QSLIST_EMPTY_RCU(head) (qatomic_read(&(head)->slh_first) == NULL) |
268 | #define QSLIST_FIRST_RCU(head) qatomic_rcu_read(&(head)->slh_first) | |
269 | #define QSLIST_NEXT_RCU(elm, field) qatomic_rcu_read(&(elm)->field.sle_next) | |
8c3570e3 PB |
270 | |
271 | /* Singly-linked list functions */ | |
272 | #define QSLIST_INSERT_HEAD_RCU(head, elm, field) do { \ | |
273 | (elm)->field.sle_next = (head)->slh_first; \ | |
d73415a3 | 274 | qatomic_rcu_set(&(head)->slh_first, (elm)); \ |
8c3570e3 PB |
275 | } while (/*CONSTCOND*/0) |
276 | ||
277 | #define QSLIST_INSERT_AFTER_RCU(head, listelm, elm, field) do { \ | |
278 | (elm)->field.sle_next = (listelm)->field.sle_next; \ | |
d73415a3 | 279 | qatomic_rcu_set(&(listelm)->field.sle_next, (elm)); \ |
8c3570e3 PB |
280 | } while (/*CONSTCOND*/0) |
281 | ||
282 | #define QSLIST_REMOVE_HEAD_RCU(head, field) do { \ | |
d73415a3 | 283 | qatomic_set(&(head)->slh_first, (head)->slh_first->field.sle_next);\ |
8c3570e3 PB |
284 | } while (/*CONSTCOND*/0) |
285 | ||
286 | #define QSLIST_REMOVE_RCU(head, elm, type, field) do { \ | |
287 | if ((head)->slh_first == (elm)) { \ | |
288 | QSLIST_REMOVE_HEAD_RCU((head), field); \ | |
289 | } else { \ | |
290 | struct type *curr = (head)->slh_first; \ | |
291 | while (curr->field.sle_next != (elm)) { \ | |
292 | curr = curr->field.sle_next; \ | |
293 | } \ | |
d73415a3 | 294 | qatomic_set(&curr->field.sle_next, \ |
8c3570e3 PB |
295 | curr->field.sle_next->field.sle_next); \ |
296 | } \ | |
297 | } while (/*CONSTCOND*/0) | |
298 | ||
299 | #define QSLIST_FOREACH_RCU(var, head, field) \ | |
d73415a3 SH |
300 | for ((var) = qatomic_rcu_read(&(head)->slh_first); \ |
301 | (var); \ | |
302 | (var) = qatomic_rcu_read(&(var)->field.sle_next)) | |
8c3570e3 | 303 | |
d73415a3 SH |
304 | #define QSLIST_FOREACH_SAFE_RCU(var, head, field, next) \ |
305 | for ((var) = qatomic_rcu_read(&(head)->slh_first); \ | |
306 | (var) && ((next) = qatomic_rcu_read(&(var)->field.sle_next), 1); \ | |
8c3570e3 PB |
307 | (var) = (next)) |
308 | ||
175de524 | 309 | #endif /* QEMU_RCU_QUEUE_H */ |