include/qemu/rcu_queue.h

   1 #ifndef QEMU_RCU_QUEUE_H
   2 #define QEMU_RCU_QUEUE_H
   3
   4 /*
   5  * rcu_queue.h
   6  *
   7  * RCU-friendly versions of the queue.h primitives.
   8  *
   9  * This library is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU Lesser General Public
  11  * License as published by the Free Software Foundation; either
  12  * version 2.1 of the License, or (at your option) any later version.
  13  *
  14  * This library is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  * Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public
  20  * License along with this library; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  22  *
  23  * Copyright (c) 2013 Mike D. Day, IBM Corporation.
  24  *
  25  * IBM's contributions to this file may be relicensed under LGPLv2 or later.
  26  */
  27
  28 #include "qemu/queue.h"
  29 #include "qemu/atomic.h"
  30
  31 /*
  32  * List access methods.
  33  */
  34 #define QLIST_EMPTY_RCU(head) (qatomic_read(&(head)->lh_first) == NULL)
  35 #define QLIST_FIRST_RCU(head) (qatomic_rcu_read(&(head)->lh_first))
  36 #define QLIST_NEXT_RCU(elm, field) (qatomic_rcu_read(&(elm)->field.le_next))
  37
  38 /*
  39  * List functions.
  40  */
  41
  42
  43 /*
  44  *  The difference between qatomic_read/set and qatomic_rcu_read/set
  45  *  is in the including of a read/write memory barrier to the volatile
  46  *  access. atomic_rcu_* macros include the memory barrier, the
  47  *  plain atomic macros do not. Therefore, it should be correct to
  48  *  issue a series of reads or writes to the same element using only
  49  *  the atomic_* macro, until the last read or write, which should be
  50  *  atomic_rcu_* to introduce a read or write memory barrier as
  51  *  appropriate.
  52  */
  53
  54 /* Upon publication of the listelm->next value, list readers
  55  * will see the new node when following next pointers from
  56  * antecedent nodes, but may not see the new node when following
  57  * prev pointers from subsequent nodes until after the RCU grace
  58  * period expires.
  59  * see linux/include/rculist.h __list_add_rcu(new, prev, next)
  60  */
  61 #define QLIST_INSERT_AFTER_RCU(listelm, elm, field) do {    \
  62     (elm)->field.le_next = (listelm)->field.le_next;        \
  63     (elm)->field.le_prev = &(listelm)->field.le_next;       \
  64     qatomic_rcu_set(&(listelm)->field.le_next, (elm));      \
  65     if ((elm)->field.le_next != NULL) {                     \
  66        (elm)->field.le_next->field.le_prev =                \
  67         &(elm)->field.le_next;                              \
  68     }                                                       \
  69 } while (/*CONSTCOND*/0)
  70
  71 /* Upon publication of the listelm->prev->next value, list
  72  * readers will see the new element when following prev pointers
  73  * from subsequent elements, but may not see the new element
  74  * when following next pointers from antecedent elements
  75  * until after the RCU grace period expires.
  76  */
  77 #define QLIST_INSERT_BEFORE_RCU(listelm, elm, field) do {   \
  78     (elm)->field.le_prev = (listelm)->field.le_prev;        \
  79     (elm)->field.le_next = (listelm);                       \
  80     qatomic_rcu_set((listelm)->field.le_prev, (elm));       \
  81     (listelm)->field.le_prev = &(elm)->field.le_next;       \
  82 } while (/*CONSTCOND*/0)
  83
  84 /* Upon publication of the head->first value, list readers
  85  * will see the new element when following the head, but may
  86  * not see the new element when following prev pointers from
  87  * subsequent elements until after the RCU grace period has
  88  * expired.
  89  */
  90 #define QLIST_INSERT_HEAD_RCU(head, elm, field) do {    \
  91     (elm)->field.le_prev = &(head)->lh_first;           \
  92     (elm)->field.le_next = (head)->lh_first;            \
  93     qatomic_rcu_set((&(head)->lh_first), (elm));        \
  94     if ((elm)->field.le_next != NULL) {                 \
  95        (elm)->field.le_next->field.le_prev =            \
  96         &(elm)->field.le_next;                          \
  97     }                                                   \
  98 } while (/*CONSTCOND*/0)
  99
 100
 101 /* prior to publication of the elm->prev->next value, some list
 102  * readers may still see the removed element when following
 103  * the antecedent's next pointer.
 104  */
 105 #define QLIST_REMOVE_RCU(elm, field) do {           \
 106     if ((elm)->field.le_next != NULL) {             \
 107        (elm)->field.le_next->field.le_prev =        \
 108         (elm)->field.le_prev;                       \
 109     }                                               \
 110     qatomic_set((elm)->field.le_prev, (elm)->field.le_next); \
 111 } while (/*CONSTCOND*/0)
 112
 113 /* List traversal must occur within an RCU critical section.  */
 114 #define QLIST_FOREACH_RCU(var, head, field)                 \
 115         for ((var) = qatomic_rcu_read(&(head)->lh_first);   \
 116                 (var);                                      \
 117                 (var) = qatomic_rcu_read(&(var)->field.le_next))
 118
 119 /* List traversal must occur within an RCU critical section.  */
 120 #define QLIST_FOREACH_SAFE_RCU(var, head, field, next_var)           \
 121     for ((var) = (qatomic_rcu_read(&(head)->lh_first));              \
 122       (var) &&                                                       \
 123           ((next_var) = qatomic_rcu_read(&(var)->field.le_next), 1); \
 124            (var) = (next_var))
 125
 126 /*
 127  * RCU simple queue
 128  */
 129
 130 /* Simple queue access methods */
 131 #define QSIMPLEQ_EMPTY_RCU(head) \
 132     (qatomic_read(&(head)->sqh_first) == NULL)
 133 #define QSIMPLEQ_FIRST_RCU(head)       qatomic_rcu_read(&(head)->sqh_first)
 134 #define QSIMPLEQ_NEXT_RCU(elm, field)  qatomic_rcu_read(&(elm)->field.sqe_next)
 135
 136 /* Simple queue functions */
 137 #define QSIMPLEQ_INSERT_HEAD_RCU(head, elm, field) do {         \
 138     (elm)->field.sqe_next = (head)->sqh_first;                  \
 139     if ((elm)->field.sqe_next == NULL) {                        \
 140         (head)->sqh_last = &(elm)->field.sqe_next;              \
 141     }                                                           \
 142     qatomic_rcu_set(&(head)->sqh_first, (elm));                 \
 143 } while (/*CONSTCOND*/0)
 144
 145 #define QSIMPLEQ_INSERT_TAIL_RCU(head, elm, field) do {    \
 146     (elm)->field.sqe_next = NULL;                          \
 147     qatomic_rcu_set((head)->sqh_last, (elm));              \
 148     (head)->sqh_last = &(elm)->field.sqe_next;             \
 149 } while (/*CONSTCOND*/0)
 150
 151 #define QSIMPLEQ_INSERT_AFTER_RCU(head, listelm, elm, field) do {       \
 152     (elm)->field.sqe_next = (listelm)->field.sqe_next;                  \
 153     if ((elm)->field.sqe_next == NULL) {                                \
 154         (head)->sqh_last = &(elm)->field.sqe_next;                      \
 155     }                                                                   \
 156     qatomic_rcu_set(&(listelm)->field.sqe_next, (elm));                 \
 157 } while (/*CONSTCOND*/0)
 158
 159 #define QSIMPLEQ_REMOVE_HEAD_RCU(head, field) do {                     \
 160     qatomic_set(&(head)->sqh_first, (head)->sqh_first->field.sqe_next);\
 161     if ((head)->sqh_first == NULL) {                                   \
 162         (head)->sqh_last = &(head)->sqh_first;                         \
 163     }                                                                  \
 164 } while (/*CONSTCOND*/0)
 165
 166 #define QSIMPLEQ_REMOVE_RCU(head, elm, type, field) do {            \
 167     if ((head)->sqh_first == (elm)) {                               \
 168         QSIMPLEQ_REMOVE_HEAD_RCU((head), field);                    \
 169     } else {                                                        \
 170         struct type *curr = (head)->sqh_first;                      \
 171         while (curr->field.sqe_next != (elm)) {                     \
 172             curr = curr->field.sqe_next;                            \
 173         }                                                           \
 174         qatomic_set(&curr->field.sqe_next,                          \
 175                    curr->field.sqe_next->field.sqe_next);           \
 176         if (curr->field.sqe_next == NULL) {                         \
 177             (head)->sqh_last = &(curr)->field.sqe_next;             \
 178         }                                                           \
 179     }                                                               \
 180 } while (/*CONSTCOND*/0)
 181
 182 #define QSIMPLEQ_FOREACH_RCU(var, head, field)                          \
 183     for ((var) = qatomic_rcu_read(&(head)->sqh_first);                  \
 184          (var);                                                         \
 185          (var) = qatomic_rcu_read(&(var)->field.sqe_next))
 186
 187 #define QSIMPLEQ_FOREACH_SAFE_RCU(var, head, field, next)                \
 188     for ((var) = qatomic_rcu_read(&(head)->sqh_first);                   \
 189          (var) && ((next) = qatomic_rcu_read(&(var)->field.sqe_next), 1);\
 190          (var) = (next))
 191
 192 /*
 193  * RCU tail queue
 194  */
 195
 196 /* Tail queue access methods */
 197 #define QTAILQ_EMPTY_RCU(head)      (qatomic_read(&(head)->tqh_first) == NULL)
 198 #define QTAILQ_FIRST_RCU(head)       qatomic_rcu_read(&(head)->tqh_first)
 199 #define QTAILQ_NEXT_RCU(elm, field)  qatomic_rcu_read(&(elm)->field.tqe_next)
 200
 201 /* Tail queue functions */
 202 #define QTAILQ_INSERT_HEAD_RCU(head, elm, field) do {                   \
 203     (elm)->field.tqe_next = (head)->tqh_first;                          \
 204     if ((elm)->field.tqe_next != NULL) {                                \
 205         (head)->tqh_first->field.tqe_circ.tql_prev =                    \
 206             &(elm)->field.tqe_circ;                                     \
 207     } else {                                                            \
 208         (head)->tqh_circ.tql_prev = &(elm)->field.tqe_circ;             \
 209     }                                                                   \
 210     qatomic_rcu_set(&(head)->tqh_first, (elm));                         \
 211     (elm)->field.tqe_circ.tql_prev = &(head)->tqh_circ;                 \
 212 } while (/*CONSTCOND*/0)
 213
 214 #define QTAILQ_INSERT_TAIL_RCU(head, elm, field) do {                   \
 215     (elm)->field.tqe_next = NULL;                                       \
 216     (elm)->field.tqe_circ.tql_prev = (head)->tqh_circ.tql_prev;         \
 217     qatomic_rcu_set(&(head)->tqh_circ.tql_prev->tql_next, (elm));       \
 218     (head)->tqh_circ.tql_prev = &(elm)->field.tqe_circ;                 \
 219 } while (/*CONSTCOND*/0)
 220
 221 #define QTAILQ_INSERT_AFTER_RCU(head, listelm, elm, field) do {         \
 222     (elm)->field.tqe_next = (listelm)->field.tqe_next;                  \
 223     if ((elm)->field.tqe_next != NULL) {                                \
 224         (elm)->field.tqe_next->field.tqe_circ.tql_prev =                \
 225             &(elm)->field.tqe_circ;                                     \
 226     } else {                                                            \
 227         (head)->tqh_circ.tql_prev = &(elm)->field.tqe_circ;             \
 228     }                                                                   \
 229     qatomic_rcu_set(&(listelm)->field.tqe_next, (elm));                 \
 230     (elm)->field.tqe_circ.tql_prev = &(listelm)->field.tqe_circ;        \
 231 } while (/*CONSTCOND*/0)
 232
 233 #define QTAILQ_INSERT_BEFORE_RCU(listelm, elm, field) do {                \
 234     (elm)->field.tqe_circ.tql_prev = (listelm)->field.tqe_circ.tql_prev;  \
 235     (elm)->field.tqe_next = (listelm);                                    \
 236     qatomic_rcu_set(&(listelm)->field.tqe_circ.tql_prev->tql_next, (elm));\
 237     (listelm)->field.tqe_circ.tql_prev = &(elm)->field.tqe_circ;          \
 238 } while (/*CONSTCOND*/0)
 239
 240 #define QTAILQ_REMOVE_RCU(head, elm, field) do {                        \
 241     if (((elm)->field.tqe_next) != NULL) {                              \
 242         (elm)->field.tqe_next->field.tqe_circ.tql_prev =                \
 243             (elm)->field.tqe_circ.tql_prev;                             \
 244     } else {                                                            \
 245         (head)->tqh_circ.tql_prev = (elm)->field.tqe_circ.tql_prev;     \
 246     }                                                                   \
 247     qatomic_set(&(elm)->field.tqe_circ.tql_prev->tql_next,              \
 248                 (elm)->field.tqe_next);                                 \
 249     (elm)->field.tqe_circ.tql_prev = NULL;                              \
 250 } while (/*CONSTCOND*/0)
 251
 252 #define QTAILQ_FOREACH_RCU(var, head, field)                            \
 253     for ((var) = qatomic_rcu_read(&(head)->tqh_first);                  \
 254          (var);                                                         \
 255          (var) = qatomic_rcu_read(&(var)->field.tqe_next))
 256
 257 #define QTAILQ_FOREACH_SAFE_RCU(var, head, field, next)                  \
 258     for ((var) = qatomic_rcu_read(&(head)->tqh_first);                   \
 259          (var) && ((next) = qatomic_rcu_read(&(var)->field.tqe_next), 1);\
 260          (var) = (next))
 261
 262 /*
 263  * RCU singly-linked list
 264  */
 265
 266 /* Singly-linked list access methods */
 267 #define QSLIST_EMPTY_RCU(head)      (qatomic_read(&(head)->slh_first) == NULL)
 268 #define QSLIST_FIRST_RCU(head)       qatomic_rcu_read(&(head)->slh_first)
 269 #define QSLIST_NEXT_RCU(elm, field)  qatomic_rcu_read(&(elm)->field.sle_next)
 270
 271 /* Singly-linked list functions */
 272 #define QSLIST_INSERT_HEAD_RCU(head, elm, field) do {           \
 273     (elm)->field.sle_next = (head)->slh_first;                  \
 274     qatomic_rcu_set(&(head)->slh_first, (elm));                 \
 275 } while (/*CONSTCOND*/0)
 276
 277 #define QSLIST_INSERT_AFTER_RCU(head, listelm, elm, field) do {         \
 278     (elm)->field.sle_next = (listelm)->field.sle_next;                  \
 279     qatomic_rcu_set(&(listelm)->field.sle_next, (elm));                 \
 280 } while (/*CONSTCOND*/0)
 281
 282 #define QSLIST_REMOVE_HEAD_RCU(head, field) do {                       \
 283     qatomic_set(&(head)->slh_first, (head)->slh_first->field.sle_next);\
 284 } while (/*CONSTCOND*/0)
 285
 286 #define QSLIST_REMOVE_RCU(head, elm, type, field) do {              \
 287     if ((head)->slh_first == (elm)) {                               \
 288         QSLIST_REMOVE_HEAD_RCU((head), field);                      \
 289     } else {                                                        \
 290         struct type *curr = (head)->slh_first;                      \
 291         while (curr->field.sle_next != (elm)) {                     \
 292             curr = curr->field.sle_next;                            \
 293         }                                                           \
 294         qatomic_set(&curr->field.sle_next,                          \
 295                    curr->field.sle_next->field.sle_next);           \
 296     }                                                               \
 297 } while (/*CONSTCOND*/0)
 298
 299 #define QSLIST_FOREACH_RCU(var, head, field)                          \
 300     for ((var) = qatomic_rcu_read(&(head)->slh_first);                \
 301          (var);                                                       \
 302          (var) = qatomic_rcu_read(&(var)->field.sle_next))
 303
 304 #define QSLIST_FOREACH_SAFE_RCU(var, head, field, next)                   \
 305     for ((var) = qatomic_rcu_read(&(head)->slh_first);                    \
 306          (var) && ((next) = qatomic_rcu_read(&(var)->field.sle_next), 1); \
 307          (var) = (next))
 308
 309 #endif /* QEMU_RCU_QUEUE_H */