include/linux/wait.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _LINUX_WAIT_H
   3 #define _LINUX_WAIT_H
   4 /*
   5  * Linux wait queue related types and methods
   6  */
   7 #include <linux/list.h>
   8 #include <linux/stddef.h>
   9 #include <linux/spinlock.h>
  10
  11 #include <asm/current.h>
  12 #include <uapi/linux/wait.h>
  13
  14 typedef struct wait_queue_entry wait_queue_entry_t;
  15
  16 typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
  17 int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
  18
  19 /* wait_queue_entry::flags */
  20 #define WQ_FLAG_EXCLUSIVE       0x01
  21 #define WQ_FLAG_WOKEN           0x02
  22 #define WQ_FLAG_BOOKMARK        0x04
  23
  24 /*
  25  * A single wait-queue entry structure:
  26  */
  27 struct wait_queue_entry {
  28         unsigned int            flags;
  29         void                    *private;
  30         wait_queue_func_t       func;
  31         struct list_head        entry;
  32 };
  33
  34 struct wait_queue_head {
  35         spinlock_t              lock;
  36         struct list_head        head;
  37 };
  38 typedef struct wait_queue_head wait_queue_head_t;
  39
  40 struct task_struct;
  41
  42 /*
  43  * Macros for declaration and initialisaton of the datatypes
  44  */
  45
  46 #define __WAITQUEUE_INITIALIZER(name, tsk) {                                    \
  47         .private        = tsk,                                                  \
  48         .func           = default_wake_function,                                \
  49         .entry          = { NULL, NULL } }
  50
  51 #define DECLARE_WAITQUEUE(name, tsk)                                            \
  52         struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
  53
  54 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) {                                   \
  55         .lock           = __SPIN_LOCK_UNLOCKED(name.lock),                      \
  56         .head           = { &(name).head, &(name).head } }
  57
  58 #define DECLARE_WAIT_QUEUE_HEAD(name) \
  59         struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
  60
  61 extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
  62
  63 #define init_waitqueue_head(wq_head)                                            \
  64         do {                                                                    \
  65                 static struct lock_class_key __key;                             \
  66                                                                                 \
  67                 __init_waitqueue_head((wq_head), #wq_head, &__key);             \
  68         } while (0)
  69
  70 #ifdef CONFIG_LOCKDEP
  71 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
  72         ({ init_waitqueue_head(&name); name; })
  73 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
  74         struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
  75 #else
  76 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
  77 #endif
  78
  79 static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
  80 {
  81         wq_entry->flags         = 0;
  82         wq_entry->private       = p;
  83         wq_entry->func          = default_wake_function;
  84 }
  85
  86 static inline void
  87 init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
  88 {
  89         wq_entry->flags         = 0;
  90         wq_entry->private       = NULL;
  91         wq_entry->func          = func;
  92 }
  93
  94 /**
  95  * waitqueue_active -- locklessly test for waiters on the queue
  96  * @wq_head: the waitqueue to test for waiters
  97  *
  98  * returns true if the wait list is not empty
  99  *
 100  * NOTE: this function is lockless and requires care, incorrect usage _will_
 101  * lead to sporadic and non-obvious failure.
 102  *
 103  * Use either while holding wait_queue_head::lock or when used for wakeups
 104  * with an extra smp_mb() like:
 105  *
 106  *      CPU0 - waker                    CPU1 - waiter
 107  *
 108  *                                      for (;;) {
 109  *      @cond = true;                     prepare_to_wait(&wq_head, &wait, state);
 110  *      smp_mb();                         // smp_mb() from set_current_state()
 111  *      if (waitqueue_active(wq_head))         if (@cond)
 112  *        wake_up(wq_head);                      break;
 113  *                                        schedule();
 114  *                                      }
 115  *                                      finish_wait(&wq_head, &wait);
 116  *
 117  * Because without the explicit smp_mb() it's possible for the
 118  * waitqueue_active() load to get hoisted over the @cond store such that we'll
 119  * observe an empty wait list while the waiter might not observe @cond.
 120  *
 121  * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
 122  * which (when the lock is uncontended) are of roughly equal cost.
 123  */
 124 static inline int waitqueue_active(struct wait_queue_head *wq_head)
 125 {
 126         return !list_empty(&wq_head->head);
 127 }
 128
 129 /**
 130  * wq_has_sleeper - check if there are any waiting processes
 131  * @wq_head: wait queue head
 132  *
 133  * Returns true if wq_head has waiting processes
 134  *
 135  * Please refer to the comment for waitqueue_active.
 136  */
 137 static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
 138 {
 139         /*
 140          * We need to be sure we are in sync with the
 141          * add_wait_queue modifications to the wait queue.
 142          *
 143          * This memory barrier should be paired with one on the
 144          * waiting side.
 145          */
 146         smp_mb();
 147         return waitqueue_active(wq_head);
 148 }
 149
 150 extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 151 extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 152 extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 153
 154 static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 155 {
 156         list_add(&wq_entry->entry, &wq_head->head);
 157 }
 158
 159 /*
 160  * Used for wake-one threads:
 161  */
 162 static inline void
 163 __add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 164 {
 165         wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
 166         __add_wait_queue(wq_head, wq_entry);
 167 }
 168
 169 static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 170 {
 171         list_add_tail(&wq_entry->entry, &wq_head->head);
 172 }
 173
 174 static inline void
 175 __add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 176 {
 177         wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
 178         __add_wait_queue_entry_tail(wq_head, wq_entry);
 179 }
 180
 181 static inline void
 182 __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 183 {
 184         list_del(&wq_entry->entry);
 185 }
 186
 187 void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
 188 void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
 189 void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
 190                 unsigned int mode, void *key, wait_queue_entry_t *bookmark);
 191 void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
 192 void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
 193 void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode, int nr);
 194
 195 #define wake_up(x)                      __wake_up(x, TASK_NORMAL, 1, NULL)
 196 #define wake_up_nr(x, nr)               __wake_up(x, TASK_NORMAL, nr, NULL)
 197 #define wake_up_all(x)                  __wake_up(x, TASK_NORMAL, 0, NULL)
 198 #define wake_up_locked(x)               __wake_up_locked((x), TASK_NORMAL, 1)
 199 #define wake_up_all_locked(x)           __wake_up_locked((x), TASK_NORMAL, 0)
 200
 201 #define wake_up_interruptible(x)        __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
 202 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
 203 #define wake_up_interruptible_all(x)    __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
 204 #define wake_up_interruptible_sync(x)   __wake_up_sync((x), TASK_INTERRUPTIBLE, 1)
 205
 206 /*
 207  * Wakeup macros to be used to report events to the targets.
 208  */
 209 #define wake_up_poll(x, m)                                                      \
 210         __wake_up(x, TASK_NORMAL, 1, (void *) (m))
 211 #define wake_up_locked_poll(x, m)                                               \
 212         __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
 213 #define wake_up_interruptible_poll(x, m)                                        \
 214         __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
 215 #define wake_up_interruptible_sync_poll(x, m)                                   \
 216         __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
 217
 218 #define ___wait_cond_timeout(condition)                                         \
 219 ({                                                                              \
 220         bool __cond = (condition);                                              \
 221         if (__cond && !__ret)                                                   \
 222                 __ret = 1;                                                      \
 223         __cond || !__ret;                                                       \
 224 })
 225
 226 #define ___wait_is_interruptible(state)                                         \
 227         (!__builtin_constant_p(state) ||                                        \
 228                 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE)          \
 229
 230 extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
 231
 232 /*
 233  * The below macro ___wait_event() has an explicit shadow of the __ret
 234  * variable when used from the wait_event_*() macros.
 235  *
 236  * This is so that both can use the ___wait_cond_timeout() construct
 237  * to wrap the condition.
 238  *
 239  * The type inconsistency of the wait_event_*() __ret variable is also
 240  * on purpose; we use long where we can return timeout values and int
 241  * otherwise.
 242  */
 243
 244 #define ___wait_event(wq_head, condition, state, exclusive, ret, cmd)           \
 245 ({                                                                              \
 246         __label__ __out;                                                        \
 247         struct wait_queue_entry __wq_entry;                                     \
 248         long __ret = ret;       /* explicit shadow */                           \
 249                                                                                 \
 250         init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0);        \
 251         for (;;) {                                                              \
 252                 long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
 253                                                                                 \
 254                 if (condition)                                                  \
 255                         break;                                                  \
 256                                                                                 \
 257                 if (___wait_is_interruptible(state) && __int) {                 \
 258                         __ret = __int;                                          \
 259                         goto __out;                                             \
 260                 }                                                               \
 261                                                                                 \
 262                 cmd;                                                            \
 263         }                                                                       \
 264         finish_wait(&wq_head, &__wq_entry);                                     \
 265 __out:  __ret;                                                                  \
 266 })
 267
 268 #define __wait_event(wq_head, condition)                                        \
 269         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 270                             schedule())
 271
 272 /**
 273  * wait_event - sleep until a condition gets true
 274  * @wq_head: the waitqueue to wait on
 275  * @condition: a C expression for the event to wait for
 276  *
 277  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 278  * @condition evaluates to true. The @condition is checked each time
 279  * the waitqueue @wq_head is woken up.
 280  *
 281  * wake_up() has to be called after changing any variable that could
 282  * change the result of the wait condition.
 283  */
 284 #define wait_event(wq_head, condition)                                          \
 285 do {                                                                            \
 286         might_sleep();                                                          \
 287         if (condition)                                                          \
 288                 break;                                                          \
 289         __wait_event(wq_head, condition);                                       \
 290 } while (0)
 291
 292 #define __io_wait_event(wq_head, condition)                                     \
 293         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 294                             io_schedule())
 295
 296 /*
 297  * io_wait_event() -- like wait_event() but with io_schedule()
 298  */
 299 #define io_wait_event(wq_head, condition)                                       \
 300 do {                                                                            \
 301         might_sleep();                                                          \
 302         if (condition)                                                          \
 303                 break;                                                          \
 304         __io_wait_event(wq_head, condition);                                    \
 305 } while (0)
 306
 307 #define __wait_event_freezable(wq_head, condition)                              \
 308         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
 309                             schedule(); try_to_freeze())
 310
 311 /**
 312  * wait_event_freezable - sleep (or freeze) until a condition gets true
 313  * @wq_head: the waitqueue to wait on
 314  * @condition: a C expression for the event to wait for
 315  *
 316  * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
 317  * to system load) until the @condition evaluates to true. The
 318  * @condition is checked each time the waitqueue @wq_head is woken up.
 319  *
 320  * wake_up() has to be called after changing any variable that could
 321  * change the result of the wait condition.
 322  */
 323 #define wait_event_freezable(wq_head, condition)                                \
 324 ({                                                                              \
 325         int __ret = 0;                                                          \
 326         might_sleep();                                                          \
 327         if (!(condition))                                                       \
 328                 __ret = __wait_event_freezable(wq_head, condition);             \
 329         __ret;                                                                  \
 330 })
 331
 332 #define __wait_event_timeout(wq_head, condition, timeout)                       \
 333         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 334                       TASK_UNINTERRUPTIBLE, 0, timeout,                         \
 335                       __ret = schedule_timeout(__ret))
 336
 337 /**
 338  * wait_event_timeout - sleep until a condition gets true or a timeout elapses
 339  * @wq_head: the waitqueue to wait on
 340  * @condition: a C expression for the event to wait for
 341  * @timeout: timeout, in jiffies
 342  *
 343  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 344  * @condition evaluates to true. The @condition is checked each time
 345  * the waitqueue @wq_head is woken up.
 346  *
 347  * wake_up() has to be called after changing any variable that could
 348  * change the result of the wait condition.
 349  *
 350  * Returns:
 351  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 352  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 353  * or the remaining jiffies (at least 1) if the @condition evaluated
 354  * to %true before the @timeout elapsed.
 355  */
 356 #define wait_event_timeout(wq_head, condition, timeout)                         \
 357 ({                                                                              \
 358         long __ret = timeout;                                                   \
 359         might_sleep();                                                          \
 360         if (!___wait_cond_timeout(condition))                                   \
 361                 __ret = __wait_event_timeout(wq_head, condition, timeout);      \
 362         __ret;                                                                  \
 363 })
 364
 365 #define __wait_event_freezable_timeout(wq_head, condition, timeout)             \
 366         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 367                       TASK_INTERRUPTIBLE, 0, timeout,                           \
 368                       __ret = schedule_timeout(__ret); try_to_freeze())
 369
 370 /*
 371  * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
 372  * increasing load and is freezable.
 373  */
 374 #define wait_event_freezable_timeout(wq_head, condition, timeout)               \
 375 ({                                                                              \
 376         long __ret = timeout;                                                   \
 377         might_sleep();                                                          \
 378         if (!___wait_cond_timeout(condition))                                   \
 379                 __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
 380         __ret;                                                                  \
 381 })
 382
 383 #define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)              \
 384         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0,     \
 385                             cmd1; schedule(); cmd2)
 386 /*
 387  * Just like wait_event_cmd(), except it sets exclusive flag
 388  */
 389 #define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)                \
 390 do {                                                                            \
 391         if (condition)                                                          \
 392                 break;                                                          \
 393         __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2);             \
 394 } while (0)
 395
 396 #define __wait_event_cmd(wq_head, condition, cmd1, cmd2)                        \
 397         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 398                             cmd1; schedule(); cmd2)
 399
 400 /**
 401  * wait_event_cmd - sleep until a condition gets true
 402  * @wq_head: the waitqueue to wait on
 403  * @condition: a C expression for the event to wait for
 404  * @cmd1: the command will be executed before sleep
 405  * @cmd2: the command will be executed after sleep
 406  *
 407  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 408  * @condition evaluates to true. The @condition is checked each time
 409  * the waitqueue @wq_head is woken up.
 410  *
 411  * wake_up() has to be called after changing any variable that could
 412  * change the result of the wait condition.
 413  */
 414 #define wait_event_cmd(wq_head, condition, cmd1, cmd2)                          \
 415 do {                                                                            \
 416         if (condition)                                                          \
 417                 break;                                                          \
 418         __wait_event_cmd(wq_head, condition, cmd1, cmd2);                       \
 419 } while (0)
 420
 421 #define __wait_event_interruptible(wq_head, condition)                          \
 422         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
 423                       schedule())
 424
 425 /**
 426  * wait_event_interruptible - sleep until a condition gets true
 427  * @wq_head: the waitqueue to wait on
 428  * @condition: a C expression for the event to wait for
 429  *
 430  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 431  * @condition evaluates to true or a signal is received.
 432  * The @condition is checked each time the waitqueue @wq_head is woken up.
 433  *
 434  * wake_up() has to be called after changing any variable that could
 435  * change the result of the wait condition.
 436  *
 437  * The function will return -ERESTARTSYS if it was interrupted by a
 438  * signal and 0 if @condition evaluated to true.
 439  */
 440 #define wait_event_interruptible(wq_head, condition)                            \
 441 ({                                                                              \
 442         int __ret = 0;                                                          \
 443         might_sleep();                                                          \
 444         if (!(condition))                                                       \
 445                 __ret = __wait_event_interruptible(wq_head, condition);         \
 446         __ret;                                                                  \
 447 })
 448
 449 #define __wait_event_interruptible_timeout(wq_head, condition, timeout)         \
 450         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 451                       TASK_INTERRUPTIBLE, 0, timeout,                           \
 452                       __ret = schedule_timeout(__ret))
 453
 454 /**
 455  * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
 456  * @wq_head: the waitqueue to wait on
 457  * @condition: a C expression for the event to wait for
 458  * @timeout: timeout, in jiffies
 459  *
 460  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 461  * @condition evaluates to true or a signal is received.
 462  * The @condition is checked each time the waitqueue @wq_head is woken up.
 463  *
 464  * wake_up() has to be called after changing any variable that could
 465  * change the result of the wait condition.
 466  *
 467  * Returns:
 468  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 469  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 470  * the remaining jiffies (at least 1) if the @condition evaluated
 471  * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 472  * interrupted by a signal.
 473  */
 474 #define wait_event_interruptible_timeout(wq_head, condition, timeout)           \
 475 ({                                                                              \
 476         long __ret = timeout;                                                   \
 477         might_sleep();                                                          \
 478         if (!___wait_cond_timeout(condition))                                   \
 479                 __ret = __wait_event_interruptible_timeout(wq_head,             \
 480                                                 condition, timeout);            \
 481         __ret;                                                                  \
 482 })
 483
 484 #define __wait_event_hrtimeout(wq_head, condition, timeout, state)              \
 485 ({                                                                              \
 486         int __ret = 0;                                                          \
 487         struct hrtimer_sleeper __t;                                             \
 488                                                                                 \
 489         hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);   \
 490         hrtimer_init_sleeper(&__t, current);                                    \
 491         if ((timeout) != KTIME_MAX)                                             \
 492                 hrtimer_start_range_ns(&__t.timer, timeout,                     \
 493                                        current->timer_slack_ns,                 \
 494                                        HRTIMER_MODE_REL);                       \
 495                                                                                 \
 496         __ret = ___wait_event(wq_head, condition, state, 0, 0,                  \
 497                 if (!__t.task) {                                                \
 498                         __ret = -ETIME;                                         \
 499                         break;                                                  \
 500                 }                                                               \
 501                 schedule());                                                    \
 502                                                                                 \
 503         hrtimer_cancel(&__t.timer);                                             \
 504         destroy_hrtimer_on_stack(&__t.timer);                                   \
 505         __ret;                                                                  \
 506 })
 507
 508 /**
 509  * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
 510  * @wq_head: the waitqueue to wait on
 511  * @condition: a C expression for the event to wait for
 512  * @timeout: timeout, as a ktime_t
 513  *
 514  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 515  * @condition evaluates to true or a signal is received.
 516  * The @condition is checked each time the waitqueue @wq_head is woken up.
 517  *
 518  * wake_up() has to be called after changing any variable that could
 519  * change the result of the wait condition.
 520  *
 521  * The function returns 0 if @condition became true, or -ETIME if the timeout
 522  * elapsed.
 523  */
 524 #define wait_event_hrtimeout(wq_head, condition, timeout)                       \
 525 ({                                                                              \
 526         int __ret = 0;                                                          \
 527         might_sleep();                                                          \
 528         if (!(condition))                                                       \
 529                 __ret = __wait_event_hrtimeout(wq_head, condition, timeout,     \
 530                                                TASK_UNINTERRUPTIBLE);           \
 531         __ret;                                                                  \
 532 })
 533
 534 /**
 535  * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
 536  * @wq: the waitqueue to wait on
 537  * @condition: a C expression for the event to wait for
 538  * @timeout: timeout, as a ktime_t
 539  *
 540  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 541  * @condition evaluates to true or a signal is received.
 542  * The @condition is checked each time the waitqueue @wq is woken up.
 543  *
 544  * wake_up() has to be called after changing any variable that could
 545  * change the result of the wait condition.
 546  *
 547  * The function returns 0 if @condition became true, -ERESTARTSYS if it was
 548  * interrupted by a signal, or -ETIME if the timeout elapsed.
 549  */
 550 #define wait_event_interruptible_hrtimeout(wq, condition, timeout)              \
 551 ({                                                                              \
 552         long __ret = 0;                                                         \
 553         might_sleep();                                                          \
 554         if (!(condition))                                                       \
 555                 __ret = __wait_event_hrtimeout(wq, condition, timeout,          \
 556                                                TASK_INTERRUPTIBLE);             \
 557         __ret;                                                                  \
 558 })
 559
 560 #define __wait_event_interruptible_exclusive(wq, condition)                     \
 561         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                  \
 562                       schedule())
 563
 564 #define wait_event_interruptible_exclusive(wq, condition)                       \
 565 ({                                                                              \
 566         int __ret = 0;                                                          \
 567         might_sleep();                                                          \
 568         if (!(condition))                                                       \
 569                 __ret = __wait_event_interruptible_exclusive(wq, condition);    \
 570         __ret;                                                                  \
 571 })
 572
 573 #define __wait_event_killable_exclusive(wq, condition)                          \
 574         ___wait_event(wq, condition, TASK_KILLABLE, 1, 0,                       \
 575                       schedule())
 576
 577 #define wait_event_killable_exclusive(wq, condition)                            \
 578 ({                                                                              \
 579         int __ret = 0;                                                          \
 580         might_sleep();                                                          \
 581         if (!(condition))                                                       \
 582                 __ret = __wait_event_killable_exclusive(wq, condition);         \
 583         __ret;                                                                  \
 584 })
 585
 586
 587 #define __wait_event_freezable_exclusive(wq, condition)                         \
 588         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                  \
 589                         schedule(); try_to_freeze())
 590
 591 #define wait_event_freezable_exclusive(wq, condition)                           \
 592 ({                                                                              \
 593         int __ret = 0;                                                          \
 594         might_sleep();                                                          \
 595         if (!(condition))                                                       \
 596                 __ret = __wait_event_freezable_exclusive(wq, condition);        \
 597         __ret;                                                                  \
 598 })
 599
 600 extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
 601 extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
 602
 603 #define __wait_event_interruptible_locked(wq, condition, exclusive, fn)         \
 604 ({                                                                              \
 605         int __ret;                                                              \
 606         DEFINE_WAIT(__wait);                                                    \
 607         if (exclusive)                                                          \
 608                 __wait.flags |= WQ_FLAG_EXCLUSIVE;                              \
 609         do {                                                                    \
 610                 __ret = fn(&(wq), &__wait);                                     \
 611                 if (__ret)                                                      \
 612                         break;                                                  \
 613         } while (!(condition));                                                 \
 614         __remove_wait_queue(&(wq), &__wait);                                    \
 615         __set_current_state(TASK_RUNNING);                                      \
 616         __ret;                                                                  \
 617 })
 618
 619
 620 /**
 621  * wait_event_interruptible_locked - sleep until a condition gets true
 622  * @wq: the waitqueue to wait on
 623  * @condition: a C expression for the event to wait for
 624  *
 625  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 626  * @condition evaluates to true or a signal is received.
 627  * The @condition is checked each time the waitqueue @wq is woken up.
 628  *
 629  * It must be called with wq.lock being held.  This spinlock is
 630  * unlocked while sleeping but @condition testing is done while lock
 631  * is held and when this macro exits the lock is held.
 632  *
 633  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 634  * functions which must match the way they are locked/unlocked outside
 635  * of this macro.
 636  *
 637  * wake_up_locked() has to be called after changing any variable that could
 638  * change the result of the wait condition.
 639  *
 640  * The function will return -ERESTARTSYS if it was interrupted by a
 641  * signal and 0 if @condition evaluated to true.
 642  */
 643 #define wait_event_interruptible_locked(wq, condition)                          \
 644         ((condition)                                                            \
 645          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
 646
 647 /**
 648  * wait_event_interruptible_locked_irq - sleep until a condition gets true
 649  * @wq: the waitqueue to wait on
 650  * @condition: a C expression for the event to wait for
 651  *
 652  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 653  * @condition evaluates to true or a signal is received.
 654  * The @condition is checked each time the waitqueue @wq is woken up.
 655  *
 656  * It must be called with wq.lock being held.  This spinlock is
 657  * unlocked while sleeping but @condition testing is done while lock
 658  * is held and when this macro exits the lock is held.
 659  *
 660  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 661  * functions which must match the way they are locked/unlocked outside
 662  * of this macro.
 663  *
 664  * wake_up_locked() has to be called after changing any variable that could
 665  * change the result of the wait condition.
 666  *
 667  * The function will return -ERESTARTSYS if it was interrupted by a
 668  * signal and 0 if @condition evaluated to true.
 669  */
 670 #define wait_event_interruptible_locked_irq(wq, condition)                      \
 671         ((condition)                                                            \
 672          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
 673
 674 /**
 675  * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
 676  * @wq: the waitqueue to wait on
 677  * @condition: a C expression for the event to wait for
 678  *
 679  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 680  * @condition evaluates to true or a signal is received.
 681  * The @condition is checked each time the waitqueue @wq is woken up.
 682  *
 683  * It must be called with wq.lock being held.  This spinlock is
 684  * unlocked while sleeping but @condition testing is done while lock
 685  * is held and when this macro exits the lock is held.
 686  *
 687  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 688  * functions which must match the way they are locked/unlocked outside
 689  * of this macro.
 690  *
 691  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 692  * set thus when other process waits process on the list if this
 693  * process is awaken further processes are not considered.
 694  *
 695  * wake_up_locked() has to be called after changing any variable that could
 696  * change the result of the wait condition.
 697  *
 698  * The function will return -ERESTARTSYS if it was interrupted by a
 699  * signal and 0 if @condition evaluated to true.
 700  */
 701 #define wait_event_interruptible_exclusive_locked(wq, condition)                \
 702         ((condition)                                                            \
 703          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
 704
 705 /**
 706  * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
 707  * @wq: the waitqueue to wait on
 708  * @condition: a C expression for the event to wait for
 709  *
 710  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 711  * @condition evaluates to true or a signal is received.
 712  * The @condition is checked each time the waitqueue @wq is woken up.
 713  *
 714  * It must be called with wq.lock being held.  This spinlock is
 715  * unlocked while sleeping but @condition testing is done while lock
 716  * is held and when this macro exits the lock is held.
 717  *
 718  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 719  * functions which must match the way they are locked/unlocked outside
 720  * of this macro.
 721  *
 722  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 723  * set thus when other process waits process on the list if this
 724  * process is awaken further processes are not considered.
 725  *
 726  * wake_up_locked() has to be called after changing any variable that could
 727  * change the result of the wait condition.
 728  *
 729  * The function will return -ERESTARTSYS if it was interrupted by a
 730  * signal and 0 if @condition evaluated to true.
 731  */
 732 #define wait_event_interruptible_exclusive_locked_irq(wq, condition)            \
 733         ((condition)                                                            \
 734          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
 735
 736
 737 #define __wait_event_killable(wq, condition)                                    \
 738         ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
 739
 740 /**
 741  * wait_event_killable - sleep until a condition gets true
 742  * @wq_head: the waitqueue to wait on
 743  * @condition: a C expression for the event to wait for
 744  *
 745  * The process is put to sleep (TASK_KILLABLE) until the
 746  * @condition evaluates to true or a signal is received.
 747  * The @condition is checked each time the waitqueue @wq_head is woken up.
 748  *
 749  * wake_up() has to be called after changing any variable that could
 750  * change the result of the wait condition.
 751  *
 752  * The function will return -ERESTARTSYS if it was interrupted by a
 753  * signal and 0 if @condition evaluated to true.
 754  */
 755 #define wait_event_killable(wq_head, condition)                                 \
 756 ({                                                                              \
 757         int __ret = 0;                                                          \
 758         might_sleep();                                                          \
 759         if (!(condition))                                                       \
 760                 __ret = __wait_event_killable(wq_head, condition);              \
 761         __ret;                                                                  \
 762 })
 763
 764 #define __wait_event_killable_timeout(wq_head, condition, timeout)              \
 765         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 766                       TASK_KILLABLE, 0, timeout,                                \
 767                       __ret = schedule_timeout(__ret))
 768
 769 /**
 770  * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
 771  * @wq_head: the waitqueue to wait on
 772  * @condition: a C expression for the event to wait for
 773  * @timeout: timeout, in jiffies
 774  *
 775  * The process is put to sleep (TASK_KILLABLE) until the
 776  * @condition evaluates to true or a kill signal is received.
 777  * The @condition is checked each time the waitqueue @wq_head is woken up.
 778  *
 779  * wake_up() has to be called after changing any variable that could
 780  * change the result of the wait condition.
 781  *
 782  * Returns:
 783  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 784  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 785  * the remaining jiffies (at least 1) if the @condition evaluated
 786  * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 787  * interrupted by a kill signal.
 788  *
 789  * Only kill signals interrupt this process.
 790  */
 791 #define wait_event_killable_timeout(wq_head, condition, timeout)                \
 792 ({                                                                              \
 793         long __ret = timeout;                                                   \
 794         might_sleep();                                                          \
 795         if (!___wait_cond_timeout(condition))                                   \
 796                 __ret = __wait_event_killable_timeout(wq_head,                  \
 797                                                 condition, timeout);            \
 798         __ret;                                                                  \
 799 })
 800
 801
 802 #define __wait_event_lock_irq(wq_head, condition, lock, cmd)                    \
 803         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 804                             spin_unlock_irq(&lock);                             \
 805                             cmd;                                                \
 806                             schedule();                                         \
 807                             spin_lock_irq(&lock))
 808
 809 /**
 810  * wait_event_lock_irq_cmd - sleep until a condition gets true. The
 811  *                           condition is checked under the lock. This
 812  *                           is expected to be called with the lock
 813  *                           taken.
 814  * @wq_head: the waitqueue to wait on
 815  * @condition: a C expression for the event to wait for
 816  * @lock: a locked spinlock_t, which will be released before cmd
 817  *        and schedule() and reacquired afterwards.
 818  * @cmd: a command which is invoked outside the critical section before
 819  *       sleep
 820  *
 821  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 822  * @condition evaluates to true. The @condition is checked each time
 823  * the waitqueue @wq_head is woken up.
 824  *
 825  * wake_up() has to be called after changing any variable that could
 826  * change the result of the wait condition.
 827  *
 828  * This is supposed to be called while holding the lock. The lock is
 829  * dropped before invoking the cmd and going to sleep and is reacquired
 830  * afterwards.
 831  */
 832 #define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd)                  \
 833 do {                                                                            \
 834         if (condition)                                                          \
 835                 break;                                                          \
 836         __wait_event_lock_irq(wq_head, condition, lock, cmd);                   \
 837 } while (0)
 838
 839 /**
 840  * wait_event_lock_irq - sleep until a condition gets true. The
 841  *                       condition is checked under the lock. This
 842  *                       is expected to be called with the lock
 843  *                       taken.
 844  * @wq_head: the waitqueue to wait on
 845  * @condition: a C expression for the event to wait for
 846  * @lock: a locked spinlock_t, which will be released before schedule()
 847  *        and reacquired afterwards.
 848  *
 849  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 850  * @condition evaluates to true. The @condition is checked each time
 851  * the waitqueue @wq_head is woken up.
 852  *
 853  * wake_up() has to be called after changing any variable that could
 854  * change the result of the wait condition.
 855  *
 856  * This is supposed to be called while holding the lock. The lock is
 857  * dropped before going to sleep and is reacquired afterwards.
 858  */
 859 #define wait_event_lock_irq(wq_head, condition, lock)                           \
 860 do {                                                                            \
 861         if (condition)                                                          \
 862                 break;                                                          \
 863         __wait_event_lock_irq(wq_head, condition, lock, );                      \
 864 } while (0)
 865
 866
 867 #define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd)      \
 868         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
 869                       spin_unlock_irq(&lock);                                   \
 870                       cmd;                                                      \
 871                       schedule();                                               \
 872                       spin_lock_irq(&lock))
 873
 874 /**
 875  * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
 876  *              The condition is checked under the lock. This is expected to
 877  *              be called with the lock taken.
 878  * @wq_head: the waitqueue to wait on
 879  * @condition: a C expression for the event to wait for
 880  * @lock: a locked spinlock_t, which will be released before cmd and
 881  *        schedule() and reacquired afterwards.
 882  * @cmd: a command which is invoked outside the critical section before
 883  *       sleep
 884  *
 885  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 886  * @condition evaluates to true or a signal is received. The @condition is
 887  * checked each time the waitqueue @wq_head is woken up.
 888  *
 889  * wake_up() has to be called after changing any variable that could
 890  * change the result of the wait condition.
 891  *
 892  * This is supposed to be called while holding the lock. The lock is
 893  * dropped before invoking the cmd and going to sleep and is reacquired
 894  * afterwards.
 895  *
 896  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 897  * and 0 if @condition evaluated to true.
 898  */
 899 #define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd)    \
 900 ({                                                                              \
 901         int __ret = 0;                                                          \
 902         if (!(condition))                                                       \
 903                 __ret = __wait_event_interruptible_lock_irq(wq_head,            \
 904                                                 condition, lock, cmd);          \
 905         __ret;                                                                  \
 906 })
 907
 908 /**
 909  * wait_event_interruptible_lock_irq - sleep until a condition gets true.
 910  *              The condition is checked under the lock. This is expected
 911  *              to be called with the lock taken.
 912  * @wq_head: the waitqueue to wait on
 913  * @condition: a C expression for the event to wait for
 914  * @lock: a locked spinlock_t, which will be released before schedule()
 915  *        and reacquired afterwards.
 916  *
 917  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 918  * @condition evaluates to true or signal is received. The @condition is
 919  * checked each time the waitqueue @wq_head is woken up.
 920  *
 921  * wake_up() has to be called after changing any variable that could
 922  * change the result of the wait condition.
 923  *
 924  * This is supposed to be called while holding the lock. The lock is
 925  * dropped before going to sleep and is reacquired afterwards.
 926  *
 927  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 928  * and 0 if @condition evaluated to true.
 929  */
 930 #define wait_event_interruptible_lock_irq(wq_head, condition, lock)             \
 931 ({                                                                              \
 932         int __ret = 0;                                                          \
 933         if (!(condition))                                                       \
 934                 __ret = __wait_event_interruptible_lock_irq(wq_head,            \
 935                                                 condition, lock,);              \
 936         __ret;                                                                  \
 937 })
 938
 939 #define __wait_event_interruptible_lock_irq_timeout(wq_head, condition,         \
 940                                                     lock, timeout)              \
 941         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 942                       TASK_INTERRUPTIBLE, 0, timeout,                           \
 943                       spin_unlock_irq(&lock);                                   \
 944                       __ret = schedule_timeout(__ret);                          \
 945                       spin_lock_irq(&lock));
 946
 947 /**
 948  * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
 949  *              true or a timeout elapses. The condition is checked under
 950  *              the lock. This is expected to be called with the lock taken.
 951  * @wq_head: the waitqueue to wait on
 952  * @condition: a C expression for the event to wait for
 953  * @lock: a locked spinlock_t, which will be released before schedule()
 954  *        and reacquired afterwards.
 955  * @timeout: timeout, in jiffies
 956  *
 957  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 958  * @condition evaluates to true or signal is received. The @condition is
 959  * checked each time the waitqueue @wq_head is woken up.
 960  *
 961  * wake_up() has to be called after changing any variable that could
 962  * change the result of the wait condition.
 963  *
 964  * This is supposed to be called while holding the lock. The lock is
 965  * dropped before going to sleep and is reacquired afterwards.
 966  *
 967  * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
 968  * was interrupted by a signal, and the remaining jiffies otherwise
 969  * if the condition evaluated to true before the timeout elapsed.
 970  */
 971 #define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock,     \
 972                                                   timeout)                      \
 973 ({                                                                              \
 974         long __ret = timeout;                                                   \
 975         if (!___wait_cond_timeout(condition))                                   \
 976                 __ret = __wait_event_interruptible_lock_irq_timeout(            \
 977                                         wq_head, condition, lock, timeout);     \
 978         __ret;                                                                  \
 979 })
 980
 981 /*
 982  * Waitqueues which are removed from the waitqueue_head at wakeup time
 983  */
 984 void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
 985 void prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
 986 long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
 987 void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 988 long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
 989 int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
 990 int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
 991
 992 #define DEFINE_WAIT_FUNC(name, function)                                        \
 993         struct wait_queue_entry name = {                                        \
 994                 .private        = current,                                      \
 995                 .func           = function,                                     \
 996                 .entry          = LIST_HEAD_INIT((name).entry),                 \
 997         }
 998
 999 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
1000
1001 #define init_wait(wait)                                                         \
1002         do {                                                                    \
1003                 (wait)->private = current;                                      \
1004                 (wait)->func = autoremove_wake_function;                        \
1005                 INIT_LIST_HEAD(&(wait)->entry);                                 \
1006                 (wait)->flags = 0;                                              \
1007         } while (0)
1008
1009 #endif /* _LINUX_WAIT_H */