include/linux/swait.h

   1 #ifndef _LINUX_SWAIT_H
   2 #define _LINUX_SWAIT_H
   3
   4 #include <linux/list.h>
   5 #include <linux/stddef.h>
   6 #include <linux/spinlock.h>
   7 #include <asm/current.h>
   8
   9 /*
  10  * Simple wait queues
  11  *
  12  * While these are very similar to regular wait queues (wait.h) the most
  13  * important difference is that the simple waitqueue allows for deterministic
  14  * behaviour -- IOW it has strictly bounded IRQ and lock hold times.
  15  *
  16  * Mainly, this is accomplished by two things. Firstly not allowing swake_up_all
  17  * from IRQ disabled, and dropping the lock upon every wakeup, giving a higher
  18  * priority task a chance to run.
  19  *
  20  * Secondly, we had to drop a fair number of features of the other waitqueue
  21  * code; notably:
  22  *
  23  *  - mixing INTERRUPTIBLE and UNINTERRUPTIBLE sleeps on the same waitqueue;
  24  *    all wakeups are TASK_NORMAL in order to avoid O(n) lookups for the right
  25  *    sleeper state.
  26  *
  27  *  - the exclusive mode; because this requires preserving the list order
  28  *    and this is hard.
  29  *
  30  *  - custom wake callback functions; because you cannot give any guarantees
  31  *    about random code. This also allows swait to be used in RT, such that
  32  *    raw spinlock can be used for the swait queue head.
  33  *
  34  * As a side effect of these; the data structures are slimmer albeit more ad-hoc.
  35  * For all the above, note that simple wait queues should _only_ be used under
  36  * very specific realtime constraints -- it is best to stick with the regular
  37  * wait queues in most cases.
  38  */
  39
  40 struct task_struct;
  41
  42 struct swait_queue_head {
  43         raw_spinlock_t          lock;
  44         struct list_head        task_list;
  45 };
  46
  47 struct swait_queue {
  48         struct task_struct      *task;
  49         struct list_head        task_list;
  50 };
  51
  52 #define __SWAITQUEUE_INITIALIZER(name) {                                \
  53         .task           = current,                                      \
  54         .task_list      = LIST_HEAD_INIT((name).task_list),             \
  55 }
  56
  57 #define DECLARE_SWAITQUEUE(name)                                        \
  58         struct swait_queue name = __SWAITQUEUE_INITIALIZER(name)
  59
  60 #define __SWAIT_QUEUE_HEAD_INITIALIZER(name) {                          \
  61         .lock           = __RAW_SPIN_LOCK_UNLOCKED(name.lock),          \
  62         .task_list      = LIST_HEAD_INIT((name).task_list),             \
  63 }
  64
  65 #define DECLARE_SWAIT_QUEUE_HEAD(name)                                  \
  66         struct swait_queue_head name = __SWAIT_QUEUE_HEAD_INITIALIZER(name)
  67
  68 extern void __init_swait_queue_head(struct swait_queue_head *q, const char *name,
  69                                     struct lock_class_key *key);
  70
  71 #define init_swait_queue_head(q)                                \
  72         do {                                                    \
  73                 static struct lock_class_key __key;             \
  74                 __init_swait_queue_head((q), #q, &__key);       \
  75         } while (0)
  76
  77 #ifdef CONFIG_LOCKDEP
  78 # define __SWAIT_QUEUE_HEAD_INIT_ONSTACK(name)                  \
  79         ({ init_swait_queue_head(&name); name; })
  80 # define DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(name)                 \
  81         struct swait_queue_head name = __SWAIT_QUEUE_HEAD_INIT_ONSTACK(name)
  82 #else
  83 # define DECLARE_SWAIT_QUEUE_HEAD_ONSTACK(name)                 \
  84         DECLARE_SWAIT_QUEUE_HEAD(name)
  85 #endif
  86
  87 /**
  88  * swait_active -- locklessly test for waiters on the queue
  89  * @wq: the waitqueue to test for waiters
  90  *
  91  * returns true if the wait list is not empty
  92  *
  93  * NOTE: this function is lockless and requires care, incorrect usage _will_
  94  * lead to sporadic and non-obvious failure.
  95  *
  96  * NOTE2: this function has the same above implications as regular waitqueues.
  97  *
  98  * Use either while holding swait_queue_head::lock or when used for wakeups
  99  * with an extra smp_mb() like:
 100  *
 101  *      CPU0 - waker                    CPU1 - waiter
 102  *
 103  *                                      for (;;) {
 104  *      @cond = true;                     prepare_to_swait(&wq_head, &wait, state);
 105  *      smp_mb();                         // smp_mb() from set_current_state()
 106  *      if (swait_active(wq_head))        if (@cond)
 107  *        wake_up(wq_head);                      break;
 108  *                                        schedule();
 109  *                                      }
 110  *                                      finish_swait(&wq_head, &wait);
 111  *
 112  * Because without the explicit smp_mb() it's possible for the
 113  * swait_active() load to get hoisted over the @cond store such that we'll
 114  * observe an empty wait list while the waiter might not observe @cond.
 115  * This, in turn, can trigger missing wakeups.
 116  *
 117  * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
 118  * which (when the lock is uncontended) are of roughly equal cost.
 119  */
 120 static inline int swait_active(struct swait_queue_head *wq)
 121 {
 122         return !list_empty(&wq->task_list);
 123 }
 124
 125 /**
 126  * swq_has_sleeper - check if there are any waiting processes
 127  * @wq: the waitqueue to test for waiters
 128  *
 129  * Returns true if @wq has waiting processes
 130  *
 131  * Please refer to the comment for swait_active.
 132  */
 133 static inline bool swq_has_sleeper(struct swait_queue_head *wq)
 134 {
 135         /*
 136          * We need to be sure we are in sync with the list_add()
 137          * modifications to the wait queue (task_list).
 138          *
 139          * This memory barrier should be paired with one on the
 140          * waiting side.
 141          */
 142         smp_mb();
 143         return swait_active(wq);
 144 }
 145
 146 extern void swake_up(struct swait_queue_head *q);
 147 extern void swake_up_all(struct swait_queue_head *q);
 148 extern void swake_up_locked(struct swait_queue_head *q);
 149
 150 extern void __prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait);
 151 extern void prepare_to_swait(struct swait_queue_head *q, struct swait_queue *wait, int state);
 152 extern long prepare_to_swait_event(struct swait_queue_head *q, struct swait_queue *wait, int state);
 153
 154 extern void __finish_swait(struct swait_queue_head *q, struct swait_queue *wait);
 155 extern void finish_swait(struct swait_queue_head *q, struct swait_queue *wait);
 156
 157 /* as per ___wait_event() but for swait, therefore "exclusive == 0" */
 158 #define ___swait_event(wq, condition, state, ret, cmd)                  \
 159 ({                                                                      \
 160         struct swait_queue __wait;                                      \
 161         long __ret = ret;                                               \
 162                                                                         \
 163         INIT_LIST_HEAD(&__wait.task_list);                              \
 164         for (;;) {                                                      \
 165                 long __int = prepare_to_swait_event(&wq, &__wait, state);\
 166                                                                         \
 167                 if (condition)                                          \
 168                         break;                                          \
 169                                                                         \
 170                 if (___wait_is_interruptible(state) && __int) {         \
 171                         __ret = __int;                                  \
 172                         break;                                          \
 173                 }                                                       \
 174                                                                         \
 175                 cmd;                                                    \
 176         }                                                               \
 177         finish_swait(&wq, &__wait);                                     \
 178         __ret;                                                          \
 179 })
 180
 181 #define __swait_event(wq, condition)                                    \
 182         (void)___swait_event(wq, condition, TASK_UNINTERRUPTIBLE, 0,    \
 183                             schedule())
 184
 185 #define swait_event(wq, condition)                                      \
 186 do {                                                                    \
 187         if (condition)                                                  \
 188                 break;                                                  \
 189         __swait_event(wq, condition);                                   \
 190 } while (0)
 191
 192 #define __swait_event_timeout(wq, condition, timeout)                   \
 193         ___swait_event(wq, ___wait_cond_timeout(condition),             \
 194                       TASK_UNINTERRUPTIBLE, timeout,                    \
 195                       __ret = schedule_timeout(__ret))
 196
 197 #define swait_event_timeout(wq, condition, timeout)                     \
 198 ({                                                                      \
 199         long __ret = timeout;                                           \
 200         if (!___wait_cond_timeout(condition))                           \
 201                 __ret = __swait_event_timeout(wq, condition, timeout);  \
 202         __ret;                                                          \
 203 })
 204
 205 #define __swait_event_interruptible(wq, condition)                      \
 206         ___swait_event(wq, condition, TASK_INTERRUPTIBLE, 0,            \
 207                       schedule())
 208
 209 #define swait_event_interruptible(wq, condition)                        \
 210 ({                                                                      \
 211         int __ret = 0;                                                  \
 212         if (!(condition))                                               \
 213                 __ret = __swait_event_interruptible(wq, condition);     \
 214         __ret;                                                          \
 215 })
 216
 217 #define __swait_event_interruptible_timeout(wq, condition, timeout)     \
 218         ___swait_event(wq, ___wait_cond_timeout(condition),             \
 219                       TASK_INTERRUPTIBLE, timeout,                      \
 220                       __ret = schedule_timeout(__ret))
 221
 222 #define swait_event_interruptible_timeout(wq, condition, timeout)       \
 223 ({                                                                      \
 224         long __ret = timeout;                                           \
 225         if (!___wait_cond_timeout(condition))                           \
 226                 __ret = __swait_event_interruptible_timeout(wq,         \
 227                                                 condition, timeout);    \
 228         __ret;                                                          \
 229 })
 230
 231 #define __swait_event_idle(wq, condition)                               \
 232         (void)___swait_event(wq, condition, TASK_IDLE, 0, schedule())
 233
 234 /**
 235  * swait_event_idle - wait without system load contribution
 236  * @wq: the waitqueue to wait on
 237  * @condition: a C expression for the event to wait for
 238  *
 239  * The process is put to sleep (TASK_IDLE) until the @condition evaluates to
 240  * true. The @condition is checked each time the waitqueue @wq is woken up.
 241  *
 242  * This function is mostly used when a kthread or workqueue waits for some
 243  * condition and doesn't want to contribute to system load. Signals are
 244  * ignored.
 245  */
 246 #define swait_event_idle(wq, condition)                                 \
 247 do {                                                                    \
 248         if (condition)                                                  \
 249                 break;                                                  \
 250         __swait_event_idle(wq, condition);                              \
 251 } while (0)
 252
 253 #define __swait_event_idle_timeout(wq, condition, timeout)              \
 254         ___swait_event(wq, ___wait_cond_timeout(condition),             \
 255                        TASK_IDLE, timeout,                              \
 256                        __ret = schedule_timeout(__ret))
 257
 258 /**
 259  * swait_event_idle_timeout - wait up to timeout without load contribution
 260  * @wq: the waitqueue to wait on
 261  * @condition: a C expression for the event to wait for
 262  * @timeout: timeout at which we'll give up in jiffies
 263  *
 264  * The process is put to sleep (TASK_IDLE) until the @condition evaluates to
 265  * true. The @condition is checked each time the waitqueue @wq is woken up.
 266  *
 267  * This function is mostly used when a kthread or workqueue waits for some
 268  * condition and doesn't want to contribute to system load. Signals are
 269  * ignored.
 270  *
 271  * Returns:
 272  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 273  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 274  * or the remaining jiffies (at least 1) if the @condition evaluated
 275  * to %true before the @timeout elapsed.
 276  */
 277 #define swait_event_idle_timeout(wq, condition, timeout)                \
 278 ({                                                                      \
 279         long __ret = timeout;                                           \
 280         if (!___wait_cond_timeout(condition))                           \
 281                 __ret = __swait_event_idle_timeout(wq,                  \
 282                                                    condition, timeout); \
 283         __ret;                                                          \
 284 })
 285
 286 #endif /* _LINUX_SWAIT_H */